Merge branch 'main' into dependabot/pip/httpx-0.28.0

Author: jcadam14

locust-load-test/locust_scripts/leis.py

@@ -12,7 +12,7 @@ def get_leis():
         with httpx.Client() as client:
             url = os.getenv(
                 "LEI_REPO",
-                "https://raw.githubusercontent.com/cfpb/sbl-test-data/test_leis/",
+                "https://raw.githubusercontent.com/cfpb/sbl-test-data/refs/heads/main/test_leis/",
             )
             full_path = url + os.getenv("LEI_FILE", "test_leis.json")
             response = client.get(full_path)

poetry.lock

@@ -15,7 +15,7 @@ regtech-api-commons = {git = "https://github.com/cfpb/regtech-api-commons.git"}
 regtech-data-validator = {git = "https://github.com/cfpb/regtech-data-validator.git"}
 regtech-regex = {git = "https://github.com/cfpb/regtech-regex.git"}
 boto3 = "~1.34.0"
-python-multipart = "^0.0.18"
+python-multipart = "^0.0.19"
 alembic = "^1.14.0"
 async-lru = "^2.0.4"
 ujson = "^5.10.0"
@@ -31,7 +31,7 @@ aiosqlite = "^0.20.0"
 
 
 [tool.poetry.group.linters.dependencies]
-ruff = "0.7.4"
+ruff = "0.8.3"
 black = "24.8.0"
 
 
@@ -78,7 +78,9 @@ env = [
   "FS_UPLOAD_CONFIG__MKDIR=true",
   "FS_DOWNLOAD_CONFIG__PROTOCOL=file",
   "ENV=TEST",
-  "MAIL_API_URL=http://mail-api:8765/internal/confirmation/send"
+  "MAIL_API_URL=http://mail-api:8765/internal/confirmation/send",
+  'REQUEST_VALIDATORS__SIGN_AND_SUBMIT=["valid_lei_status","valid_lei_tin","valid_filing_exists","valid_sub_accepted","valid_voluntary_filer","valid_contact_info"]',
+  'REQUEST_VALIDATORS__FILING_CREATE=["valid_period_exists", "valid_no_filing_exists"]'
 ]
 testpaths = ["tests"]
 

pyproject.toml

@@ -20,4 +20,6 @@ FS_UPLOAD_CONFIG__PROTOCOL="file"
 FS_UPLOAD_CONFIG__ROOT="../upload"
 EXPIRED_SUBMISSION_CHECK_SECS=120
 SERVER_CONFIG__RELOAD="true"
-MAIL_API_URL=http://mail-api:8765/internal/confirmation/send
+MAIL_API_URL=http://mail-api:8765/internal/confirmation/send
+REQUEST_VALIDATORS__SIGN_AND_SUBMIT=["valid_lei_status","valid_lei_tin","valid_filing_exists","valid_sub_accepted","valid_voluntary_filer","valid_contact_info"]
+REQUEST_VALIDATORS__FILING_CREATE=["valid_period_exists", "valid_no_filing_exists"]

src/.env.local

@@ -17,4 +17,5 @@ CERTS_URL=${KC_REALM_URL}/protocol/openid-connect/certs
 FS_UPLOAD_CONFIG__PROTOCOL="file"
 FS_UPLOAD_CONFIG__ROOT="../upload"
 USER_FI_API_URL=http://localhost:8881/v1/institutions/
-EXPIRED_SUBMISSION_CHECK_SECS=120
+EXPIRED_SUBMISSION_CHECK_SECS=120
+REQUEST_VALIDATORS__SIGN_AND_SUBMIT=["check_lei_status","check_lei_tin","check_filing_exists","check_sub_accepted","check_voluntary_filer","check_contact_info"]

src/.env.template

@@ -1,7 +1,7 @@
 from enum import StrEnum
 import os
 from urllib import parse
-from typing import Any
+from typing import Any, Set
 
 from pydantic import field_validator, ValidationInfo, BaseModel
 from pydantic.networks import PostgresDsn
@@ -83,8 +83,25 @@ def build_postgres_dsn(cls, postgres_dsn, info: ValidationInfo) -> Any:
     model_config = SettingsConfigDict(env_file=env_files_to_load, extra="allow", env_nested_delimiter="__")
 
 
+class RequestActionValidations(BaseSettings):
+    sign_and_submit: Set[str] = {
+        "valid_lei_status",
+        "valid_lei_tin",
+        "valid_filing_exists",
+        "valid_sub_accepted",
+        "valid_voluntary_filer",
+        "valid_contact_info",
+    }
+
+    filing_create: Set[str] = {"valid_period_exists", "valid_no_filing_exists"}
+
+    model_config = SettingsConfigDict(env_prefix="request_validators__", env_file=env_files_to_load, extra="allow")
+
+
 settings = Settings()
 
+request_action_validations = RequestActionValidations()
+
 kc_settings = KeycloakSettings(_env_file=env_files_to_load)
 
 regex_configs = RegexConfigs.instance()

src/sbl_filing_api/config.py

@@ -1,7 +1,7 @@
 from sbl_filing_api.entities.models.model_enums import FilingType, FilingTaskState, SubmissionState, UserActionType
 from datetime import datetime
 from typing import Any, List
-from sqlalchemy import Enum as SAEnum, String
+from sqlalchemy import Enum as SAEnum, String, desc
 from sqlalchemy import ForeignKey, func, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, relationship
 from sqlalchemy.ext.asyncio import AsyncAttrs
@@ -114,7 +114,10 @@ class FilingDAO(Base):
     lei: Mapped[str]
     tasks: Mapped[List[FilingTaskProgressDAO] | None] = relationship(lazy="selectin", cascade="all, delete-orphan")
     institution_snapshot_id: Mapped[str] = mapped_column(nullable=True)
-    contact_info: Mapped[ContactInfoDAO] = relationship("ContactInfoDAO", lazy="joined")
+    contact_info: Mapped[ContactInfoDAO | None] = relationship("ContactInfoDAO", lazy="joined")
+    submissions: Mapped[List[SubmissionDAO] | None] = relationship(
+        "SubmissionDAO", lazy="select", order_by=desc(SubmissionDAO.submission_time)
+    )
     signatures: Mapped[List[UserActionDAO] | None] = relationship(
         "UserActionDAO", secondary="filing_signature", lazy="selectin", order_by="desc(UserActionDAO.timestamp)"
     )

src/sbl_filing_api/entities/models/dao.py

@@ -7,8 +7,6 @@
 
 from regtech_api_commons.models.auth import AuthenticatedUser
 
-from copy import deepcopy
-
 from async_lru import alru_cache
 
 from sbl_filing_api.entities.models.dao import (
@@ -64,23 +62,17 @@ async def get_submission_by_counter(session: AsyncSession, lei: str, filing_peri
 
 async def get_filing(session: AsyncSession, lei: str, filing_period: str) -> FilingDAO:
     result = await query_helper(session, FilingDAO, lei=lei, filing_period=filing_period)
-    if result:
-        result = await populate_missing_tasks(session, result)
     return result[0] if result else None
 
 
 async def get_filings(session: AsyncSession, leis: list[str], filing_period: str) -> list[FilingDAO]:
     stmt = select(FilingDAO).filter(FilingDAO.lei.in_(leis), FilingDAO.filing_period == filing_period)
     result = (await session.scalars(stmt)).all()
-    if result:
-        result = await populate_missing_tasks(session, result)
     return result if result else []
 
 
 async def get_period_filings(session: AsyncSession, filing_period: str) -> List[FilingDAO]:
     filings = await query_helper(session, FilingDAO, filing_period=filing_period)
-    if filings:
-        filings = await populate_missing_tasks(session, filings)
     return filings
 
 
@@ -148,9 +140,7 @@ async def upsert_filing(session: AsyncSession, filing: FilingDTO) -> FilingDAO:
 
 async def create_new_filing(session: AsyncSession, lei: str, filing_period: str, creator_id: int) -> FilingDAO:
     new_filing = FilingDAO(filing_period=filing_period, lei=lei, creator_id=creator_id)
-    new_filing = await upsert_helper(session, new_filing, FilingDAO)
-    new_filing = await populate_missing_tasks(session, [new_filing])
-    return new_filing[0]
+    return await upsert_helper(session, new_filing, FilingDAO)
 
 
 async def update_task_state(
@@ -171,7 +161,12 @@ async def update_contact_info(
     session: AsyncSession, lei: str, filing_period: str, new_contact_info: ContactInfoDTO
 ) -> FilingDAO:
     filing = await get_filing(session, lei=lei, filing_period=filing_period)
-    filing.contact_info = ContactInfoDAO(**new_contact_info.__dict__.copy(), filing=filing.id)
+    if filing.contact_info:
+        for key, value in new_contact_info.__dict__.items():
+            if key != "id":
+                setattr(filing.contact_info, key, value)
+    else:
+        filing.contact_info = ContactInfoDAO(**new_contact_info.__dict__.copy(), filing=filing.id)
     return await upsert_helper(session, filing, FilingDAO)
 
 
@@ -202,23 +197,3 @@ async def query_helper(session: AsyncSession, table_obj: T, **filter_args) -> Li
     if filter_args:
         stmt = stmt.filter_by(**filter_args)
     return (await session.scalars(stmt)).all()
-
-
-async def populate_missing_tasks(session: AsyncSession, filings: List[FilingDAO]):
-    filing_tasks = await get_filing_tasks(session)
-    filings_copy = deepcopy(filings)
-    for f in filings_copy:
-        tasks = [t.task.name for t in f.tasks]
-        missing_tasks = [t for t in filing_tasks if t.name not in tasks]
-        for mt in missing_tasks:
-            f.tasks.append(
-                FilingTaskProgressDAO(
-                    filing=f.id,
-                    task_name=mt.name,
-                    task=mt,
-                    state=FilingTaskState.NOT_STARTED,
-                    user="",
-                )
-            )
-
-    return filings_copy

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -11,9 +11,11 @@
 from regtech_api_commons.api.exceptions import RegTechHttpException
 from regtech_api_commons.models.auth import AuthenticatedUser
 
+from sbl_filing_api.entities.models.dao import FilingDAO
 from sbl_filing_api.entities.models.model_enums import UserActionType
 from sbl_filing_api.services import submission_processor
 from sbl_filing_api.services.multithread_handler import handle_submission
+from sbl_filing_api.config import request_action_validations
 from typing import Annotated, List
 
 from sbl_filing_api.entities.engine.engine import get_session
@@ -39,6 +41,8 @@
 
 from sbl_filing_api.services.request_handler import send_confirmation_email
 
+from sbl_filing_api.services.request_action_validator import UserActionContext, validate_user_action, set_context
+
 logger = logging.getLogger(__name__)
 
 
@@ -72,92 +76,57 @@ async def get_filings(request: Request, period_code: str):
     return await repo.get_filings(request.state.db_session, user.institutions, period_code)
 
 
-@router.post("/institutions/{lei}/filings/{period_code}", response_model=FilingDTO)
+@router.post(
+    "/institutions/{lei}/filings/{period_code}",
+    response_model=FilingDTO,
+    dependencies=[
+        Depends(set_context({UserActionContext.PERIOD, UserActionContext.FILING})),
+        Depends(validate_user_action(request_action_validations.filing_create, "Filing Create Forbidden")),
+    ],
+)
 @requires("authenticated")
 async def post_filing(request: Request, lei: str, period_code: str):
-    period = await repo.get_filing_period(request.state.db_session, filing_period=period_code)
-
-    if period:
-        filing = await repo.get_filing(request.state.db_session, lei, period_code)
-        if filing:
-            raise RegTechHttpException(
-                status_code=status.HTTP_409_CONFLICT,
-                name="Filing Creation Conflict",
-                detail=f"Filing already exists for Filing Period {period_code} and LEI {lei}",
-            )
-        creator = None
-        try:
-            creator = await repo.add_user_action(
-                request.state.db_session,
-                UserActionDTO(
-                    user_id=request.user.id,
-                    user_name=request.user.name,
-                    user_email=request.user.email,
-                    action_type=UserActionType.CREATE,
-                ),
-            )
-        except Exception:
-            logger.exception("Error while trying to create the filing.creator UserAction.")
-            raise RegTechHttpException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                name="Filing.creator UserAction error",
-                detail="Error while trying to create the filing.creator UserAction.",
-            )
-
-        try:
-            return await repo.create_new_filing(request.state.db_session, lei, period_code, creator_id=creator.id)
-        except Exception:
-            raise RegTechHttpException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                name="Filing Creation Error",
-                detail=f"An error occurred while creating a filing for LEI {lei} and Filing Period {period_code}.",
-            )
+    creator = None
+    try:
+        creator = await repo.add_user_action(
+            request.state.db_session,
+            UserActionDTO(
+                user_id=request.user.id,
+                user_name=request.user.name,
+                user_email=request.user.email,
+                action_type=UserActionType.CREATE,
+            ),
+        )
+    except Exception:
+        logger.exception("Error while trying to create the filing.creator UserAction.")
+        raise RegTechHttpException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            name="Filing.creator UserAction error",
+            detail="Error while trying to create the filing.creator UserAction.",
+        )
 
-    else:
+    try:
+        return await repo.create_new_filing(request.state.db_session, lei, period_code, creator_id=creator.id)
+    except Exception:
         raise RegTechHttpException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            name="Filing Period Not Found",
-            detail=f"The period ({period_code}) does not exist, therefore a Filing can not be created for this period.",
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            name="Filing Creation Error",
+            detail=f"An error occurred while creating a filing for LEI {lei} and Filing Period {period_code}.",
         )
 
 
-@router.put("/institutions/{lei}/filings/{period_code}/sign", response_model=FilingDTO)
+@router.put(
+    "/institutions/{lei}/filings/{period_code}/sign",
+    response_model=FilingDTO,
+    dependencies=[
+        Depends(set_context({UserActionContext.INSTITUTION, UserActionContext.FILING})),
+        Depends(validate_user_action(request_action_validations.sign_and_submit, "Filing Action Forbidden")),
+    ],
+)
 @requires("authenticated")
 async def sign_filing(request: Request, lei: str, period_code: str):
-    filing = await repo.get_filing(request.state.db_session, lei, period_code)
-    if not filing:
-        raise RegTechHttpException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            name="Filing Not Found",
-            detail=f"There is no Filing for LEI {lei} in period {period_code}, unable to sign a non-existent Filing.",
-        )
-    latest_sub = await repo.get_latest_submission(request.state.db_session, lei, period_code)
-    if not latest_sub or latest_sub.state != SubmissionState.SUBMISSION_ACCEPTED:
-        raise RegTechHttpException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            name="Filing Action Forbidden",
-            detail=f"Cannot sign filing. Filing for {lei} for period {period_code} does not have a latest submission the SUBMISSION_ACCEPTED state.",
-        )
-    if filing.is_voluntary is None:
-        raise RegTechHttpException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            name="Filing Action Forbidden",
-            detail=f"Cannot sign filing. Filing for {lei} for period {period_code} does not have a selection of is_voluntary defined.",
-        )
-    if not filing.contact_info:
-        raise RegTechHttpException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            name="Filing Action Forbidden",
-            detail=f"Cannot sign filing. Filing for {lei} for period {period_code} does not have contact info defined.",
-        )
-    """
-    if not filing.institution_snapshot_id:
-        return JSONResponse(
-            status_code=status.HTTP_403_FORBIDDEN,
-            content=f"Cannot sign filing. Filing for {lei} for period {period_code} does not have institution snapshot id defined.",
-        )
-    """
-
+    filing: FilingDAO = request.state.context["filing"]
+    latest_sub = (await filing.awaitable_attrs.submissions)[0]
     sig = await repo.add_user_action(
         request.state.db_session,
         UserActionDTO(