[Issue #2791] Create database table ExtractMetadata to store extract files

api/Dockerfile

@@ -17,10 +17,10 @@ RUN apt-get update \
     # https://pythonspeed.com/articles/security-updates-in-docker/
     && apt-get upgrade --yes \
     && apt-get install --no-install-recommends --yes \
-        build-essential \
-        libpq-dev \
-        postgresql \
-        wget \
+    build-essential \
+    libpq-dev \
+    postgresql \
+    wget \
     # Reduce the image size by clear apt cached lists
     # Complies with https://github.com/codacy/codacy-hadolint/blob/master/codacy-hadolint/docs/description/DL3009.md
     && rm -fr /var/lib/apt/lists/* \

api/src/constants/lookup_constants.py

@@ -118,6 +118,11 @@ class AgencySubmissionNotificationSetting(StrEnum):
     ALWAYS = "always"
 
 
+class ExtractType(StrEnum):
+    OPPORTUNITIES_XML = "opportunities_xml"
+    OPPORTUNITIES_CSV = "opportunities_csv"
+
+
 class OpportunityAttachmentType(StrEnum):
     NOTICE_OF_FUNDING_OPPORTUNITY = "notice_of_funding_opportunity"
     OTHER = "other"

api/src/db/migrations/versions/2024_11_15_add_metadata_extract_table.py

@@ -0,0 +1,75 @@
+"""Add metadata extract table
+
+Revision ID: 7346f6b52c3d
+Revises: 65e962033cc6
+Create Date: 2024-11-15 20:06:40.422630
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7346f6b52c3d"
+down_revision = "65e962033cc6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "lk_extract_type",
+        sa.Column("extract_type_id", sa.Integer(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("extract_type_id", name=op.f("lk_extract_type_pkey")),
+        schema="api",
+    )
+
+    op.create_table(
+        "extract_metadata",
+        sa.Column("extract_metadata_id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("extract_type_id", sa.Integer(), nullable=False),
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("file_path", sa.String(), nullable=False),
+        sa.Column("file_size_bytes", sa.BigInteger(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["extract_type_id"],
+            ["api.lk_extract_type.extract_type_id"],
+            name=op.f("extract_metadata_extract_type_id_lk_extract_type_fkey"),
+        ),
+        sa.PrimaryKeyConstraint("extract_metadata_id", name=op.f("extract_metadata_pkey")),
+        schema="api",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("lk_extract_type", schema="api")
+    op.drop_table("extract_metadata", schema="api")
+    # ### end Alembic commands ###

api/src/db/models/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-from . import agency_models, base, lookup_models, opportunity_models, user_models
+from . import agency_models, base, extract_models, lookup_models, opportunity_models, user_models
 from .transfer import topportunity_models
 
 logger = logging.getLogger(__name__)
@@ -16,4 +16,5 @@
     "topportunity_models",
     "agency_models",
     "user_models",
+    "extract_models",
 ]

api/src/db/models/extract_models.py

@@ -0,0 +1,21 @@
+from sqlalchemy import BigInteger, Column, ForeignKey, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.adapters.db.type_decorators.postgres_type_decorators import LookupColumn
+from src.constants.lookup_constants import ExtractType
+from src.db.models.base import ApiSchemaTable, TimestampMixin
+from src.db.models.lookup_models import LkExtractType
+
+
+class ExtractMetadata(ApiSchemaTable, TimestampMixin):
+    __tablename__ = "extract_metadata"
+
+    extract_metadata_id = Column(BigInteger, primary_key=True, autoincrement=True)
+    extract_type: Mapped[ExtractType] = mapped_column(
+        "extract_type_id",
+        LookupColumn(LkExtractType),
+        ForeignKey(LkExtractType.extract_type_id),
+    )
+    file_name = Column(String, nullable=False)
+    file_path = Column(String, nullable=False)
+    file_size_bytes = Column(BigInteger, nullable=False)

api/src/db/models/lookup_models.py

@@ -5,6 +5,7 @@
     AgencySubmissionNotificationSetting,
     ApplicantType,
     ExternalUserType,
+    ExtractType,
     FundingCategory,
     FundingInstrument,
     OpportunityAttachmentType,
@@ -117,6 +118,13 @@
 
 EXTERNAL_USER_TYPE_CONFIG = LookupConfig([LookupStr(ExternalUserType.LOGIN_GOV, 1)])
 
+EXTRACT_TYPE_CONFIG = LookupConfig(
+    [
+        LookupStr(ExtractType.OPPORTUNITIES_XML, 1),
+        LookupStr(ExtractType.OPPORTUNITIES_CSV, 2),
+    ]
+)
+
 
 @LookupRegistry.register_lookup(OPPORTUNITY_CATEGORY_CONFIG)
 class LkOpportunityCategory(LookupTable, TimestampMixin):
@@ -244,3 +252,17 @@ def from_lookup(cls, lookup: Lookup) -> "LkExternalUserType":
         return LkExternalUserType(
             external_user_type_id=lookup.lookup_val, description=lookup.get_description()
         )
+
+
+@LookupRegistry.register_lookup(EXTRACT_TYPE_CONFIG)
+class LkExtractType(LookupTable, TimestampMixin):
+    __tablename__ = "lk_extract_type"
+
+    extract_type_id: Mapped[int] = mapped_column(primary_key=True)
+    description: Mapped[str]
+
+    @classmethod
+    def from_lookup(cls, lookup: Lookup) -> "LkExtractType":
+        return LkExtractType(
+            extract_type_id=lookup.lookup_val, description=lookup.get_description()
+        )