[WIP] upgrade psycopg3

README.md

@@ -140,7 +140,7 @@ createdb cfdm_test
 Set the environment variable SQLA_SAMPLE_DB_CONN to point to this database, using:
 
 ```
-export SQLA_SAMPLE_DB_CONN="postgresql://<username>:<password>@localhost:<port - default is 5432>/cfdm_test"
+export SQLA_SAMPLE_DB_CONN="postgresql+psycopg://<username>:<password>@localhost:<port - default is 5432>/cfdm_test"
 ```
 
 Load our sample data into the development database (`cfdm_test`) by running:

requirements.txt

@@ -17,7 +17,7 @@ kombu==5.5.3 # Starting from celery 5.x release, the minimum required version is
 networkx==2.6.2
 prance[osv]==0.22.11.4.0
 pre-commit==2.21.0 #client-side hook triggered by operations like git commit
-psycopg2-binary==2.9.1
+psycopg==3.2.10
 python-dateutil==2.8.1
 python-dotenv>=0.20.0 # Sets variable defaults in .flaskenv file
 requests==2.32.4

tasks.py

@@ -262,9 +262,10 @@ def create_sample_db(ctx):
     """
 
     print("Loading schema...")
-    db_conn = os.getenv('SQLA_SAMPLE_DB_CONN')
+    db_conn = os.getenv('SQLA_SAMPLE_DB_CONN1')
+    db_conn2 = os.getenv('SQLA_SAMPLE_DB_CONN2')
     if not db_conn:
-        print("Error: SQLA_SAMPLE_DB_CONN env var must be set")
+        print("Error: SQLA_SAMPLE_DB_CONN1 env var must be set")
         return
     jdbc_url = to_jdbc_url(db_conn)
     result = run_migrations(ctx, jdbc_url)
@@ -281,7 +282,7 @@ def create_sample_db(ctx):
     print("Sample data loaded")
 
     print("Refreshing materialized views...")
-    os.environ["SQLA_CONN"] = db_conn  # SQLA_CONN is used by manage.py tasks
+    os.environ["SQLA_CONN"] = db_conn2  # SQLA_CONN is used by manage.py tasks
     subprocess.check_call(['python', 'cli.py', 'refresh_materialized'])
     print("Materialized views refreshed")
 

tests/test_filings.py

@@ -83,7 +83,7 @@ def test_filings_filters(self):
             factories.FilingsFactory(committee_id='C00000005'),
             factories.FilingsFactory(candidate_id='H00000001'),
             factories.FilingsFactory(amendment_indicator='A'),
-            factories.FilingsFactory(beginning_image_number=123456789021234567),
+            factories.FilingsFactory(beginning_image_number='123456789021234567'),
             factories.FilingsFactory(committee_type='P'),
             factories.FilingsFactory(cycle=2000),
             factories.FilingsFactory(document_type='X'),
@@ -104,7 +104,7 @@ def test_filings_filters(self):
 
         filter_fields = (
             ('amendment_indicator', 'A'),
-            ('beginning_image_number', 123456789021234567),
+            ('beginning_image_number', '123456789021234567'),
             ('committee_type', 'P'),
             ('cycle', 2000),
             ('document_type', 'X'),

tests/test_legal/test_ao_load.py

@@ -458,12 +458,13 @@ def create_document(self, ao_id, document, filename='201801_C.pdf'):
             text("""
             INSERT INTO aouser.document
             (document_id, ao_id, category, ocrtext, fileimage, description, document_date, filename)
-            VALUES (:docid, :id, :category, :text, :text, :descr, :date, :filename)"""),
+            VALUES (:docid, :id, :category, :text, :fileimage, :descr, :date, :filename)"""),
             {
                 "docid": document["document_id"],
                 "id": ao_id,
                 "category": document["category"],
                 "text": document["text"],
+                "fileimage": document.get("fileimage", document["text"].encode("utf-8")),
                 "descr": document["description"],
                 "date": document["date"],
                 "filename": filename

webservices/args.py

@@ -83,42 +83,58 @@ def _deserialize(self, value, attr, data, **kwargs):
         return '{0:0>2}'.format(value)
 
 
-class Date(fields.Str):
+class Date(fields.Date):
+    """Accepts both 'YYYY-MM-DD' and 'MM/DD/YYYY' formats."""
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        if isinstance(value, datetime.date):
+            return value
+
+        if not isinstance(value, str):
+            raise exceptions.ApiError(
+                    exceptions.DATE_ERROR,
+                    status_code=422)
 
-    def _validate(self, value):
         value = value.strip()
-        super()._validate(value)
-        try:
-            datetime.datetime.strptime(value, '%Y-%m-%d')
-        except (TypeError, ValueError):
+
+        for fmt in ("%Y-%m-%d", "%m/%d/%Y"):
             try:
-                datetime.datetime.strptime(value, '%m/%d/%Y')
-            except (TypeError, ValueError):
+                return datetime.datetime.strptime(value, fmt).date()
+            except ValueError:
                 raise exceptions.ApiError(
-                    exceptions.DATE_ERROR,
-                    status_code=422)
+                            exceptions.DATE_ERROR,
+                            status_code=422)
 
 
-class FullDate(fields.Str):
+class FullDate(fields.DateTime):
+    """Accepts ISO 8601, 'YYYY-MM-DD', or 'MM/DD/YYYY' and returns datetime."""
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        if isinstance(value, datetime.datetime):
+            return value
+
+        if not isinstance(value, str):
+            raise exceptions.ApiError(
+                        exceptions.FULLDATE_ERROR,
+                        status_code=422)
 
-    def _validate(self, value):
         value = value.strip()
-        super()._validate(value)
+
+        if value.endswith("Z"):
+            value = value[:-1] + "+00:00"
+
         try:
-            if value.endswith("Z"):
-                value = value[:-1] + "+00:00"
-            # Try parsing ISO 8601 format
-            datetime.datetime.fromisoformat(value)
-        except (TypeError, ValueError):
+            return datetime.datetime.fromisoformat(value)
+        except ValueError:
+            pass
+
+        for fmt in ("%Y-%m-%d", "%m/%d/%Y"):
             try:
-                # Try parsing YYYY-MM-DD
-                datetime.datetime.strptime(value, '%Y-%m-%d')
-            except (TypeError, ValueError):
-                try:
-                    # Try parsing MM/DD/YYYY
-                    datetime.datetime.strptime(value, '%m/%d/%Y')
-                except (TypeError, ValueError):
-                    raise exceptions.ApiError(
+                return datetime.datetime.strptime(value, fmt)
+            except ValueError:
+                continue
+
+        raise exceptions.ApiError(
                         exceptions.FULLDATE_ERROR,
                         status_code=422)
 
@@ -139,7 +155,23 @@ def _validate(self, value):
                 status_code=422)
 
 
-class FileNumber(fields.Str):
+class ImageNumberInt(fields.Int):
+
+    def _validate(self, value):
+        super()._validate(value)
+        try:
+            value = int(value)
+        except (TypeError, ValueError):
+            raise exceptions.ApiError(
+                exceptions.IMAGE_NUMBER_ERROR,
+                status_code=422)
+        if value < 0:
+            raise exceptions.ApiError(
+                exceptions.IMAGE_NUMBER_ERROR,
+                status_code=422)
+
+
+class FileNumber(fields.Int):
 
     def _validate(self, value):
         super()._validate(value)
@@ -572,7 +604,7 @@ def make_seek_args(field=fields.Int, description=None):
     'report_type': fields.List(IStr, metadata={'description': docs.REPORT_TYPE}),
     'request_type': fields.List(IStr, metadata={'description': docs.REQUEST_TYPE}),
     'document_type': fields.List(IStr, metadata={'description': docs.DOC_TYPE}),
-    'beginning_image_number': fields.List(ImageNumber, metadata={'description': docs.BEGINNING_IMAGE_NUMBER}),
+    'beginning_image_number': fields.List(ImageNumberInt, metadata={'description': docs.BEGINNING_IMAGE_NUMBER}),
     'report_year': fields.List(fields.Int, metadata={'description': docs.REPORT_YEAR}),
     'min_receipt_date': Date(metadata={'description': docs.MIN_RECEIPT_DATE}),
     'max_receipt_date': Date(metadata={'description': docs.MAX_RECEIPT_DATE}),
@@ -626,13 +658,13 @@ def make_seek_args(field=fields.Int, description=None):
     'election_state': fields.List(IStr, metadata={'description': docs.ELECTION_STATE}),
     'candidate_office': fields.List(fields.Str(
         validate=validate.OneOf(['', 'H', 'S', 'P'])),
-        metadata={'description': docs.OFFICE}),
-    'candidate_district': fields.List(IStr, metadata={'description': docs.ELECTION_DISTRICT}),
-    'candidate_party': fields.List(IStr, metadata={'description': docs.PARTY}),
-    'image_number': fields.List(ImageNumber, metadata={'description': docs.IMAGE_NUMBER}),
-    'min_load_timestamp': Date(metadata={'description': docs.LOAD_DATE}),
-    'max_load_timestamp': Date(metadata={'description': docs.LOAD_DATE}),
-    'committee_type': fields.List(fields.Str, metadata={'description': docs.COMMITTEE_TYPE}),
+        description=docs.OFFICE),
+    'candidate_district': fields.List(IStr, description=docs.ELECTION_DISTRICT),
+    'candidate_party': fields.List(IStr, description=docs.PARTY),
+    'image_number': fields.List(ImageNumber, description=docs.IMAGE_NUMBER),
+    'min_load_timestamp': Date(description=docs.LOAD_DATE),
+    'max_load_timestamp': Date(description=docs.LOAD_DATE),
+    'committee_type': fields.List(fields.Str, description=docs.COMMITTEE_TYPE),
     'organization_type': fields.List(
         IStr(validate=validate.OneOf(['', 'C', 'L', 'M', 'T', 'V', 'W', 'H', 'I'])),
         metadata={'description': docs.ORGANIZATION_TYPE},

webservices/common/models/costs.py

@@ -7,8 +7,8 @@
 class CommunicationCost(db.Model):
     __tablename__ = 'ofec_communication_cost_mv'
 
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer, index=True)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger, index=True)
     candidate_id = db.Column('cand_id', db.String, index=True)
     committee_id = db.Column('cmte_id', db.String, index=True)
     committee_name = db.Column(db.String)
@@ -61,7 +61,7 @@ class Electioneering(db.Model):
     candidate_state = db.Column('cand_office_st', db.String, index=True)
     beginning_image_number = db.Column('f9_begin_image_num', db.String, index=True)
     sb_image_num = db.Column(db.String, index=True)
-    sub_id = db.Column(db.Integer, doc=docs.EC_SUB_ID)
+    sub_id = db.Column(db.BigInteger, doc=docs.EC_SUB_ID)
     link_id = db.Column(db.Integer)
     sb_link_id = db.Column(db.String)
     number_of_candidates = db.Column(db.Numeric)

webservices/common/models/itemized.py

@@ -168,8 +168,8 @@ class ScheduleA(BaseItemized):
     schedule_type_full = db.Column('schedule_type_desc', db.String)
     increased_limit = db.Column(db.String)
     load_date = db.Column('pg_date', db.DateTime)
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     back_reference_transaction_id = db.Column('back_ref_tran_id', db.String)
     back_reference_schedule_name = db.Column('back_ref_sched_nm', db.String)
     pdf_url = db.Column(db.String)
@@ -365,8 +365,8 @@ class ScheduleB(BaseItemized):
     schedule_type = db.Column('schedule_type', db.String)
     schedule_type_full = db.Column('schedule_type_desc', db.String)
     load_date = db.Column('pg_date', db.DateTime)
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     back_reference_transaction_id = db.Column('back_ref_tran_id', db.String)
     back_reference_schedule_id = db.Column('back_ref_sched_id', db.String)
     semi_annual_bundled_refund = db.Column('semi_an_bundled_refund', db.Numeric(30, 2))
@@ -464,8 +464,8 @@ class ScheduleC(PdfMixin, BaseItemized):
         )''',
         lazy='joined',
     )
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     incurred_date = db.Column('incurred_dt', db.Date)
     loan_source_prefix = db.Column('loan_src_prefix', db.String)
     loan_source_first_name = db.Column('loan_src_f_nm', db.String)
@@ -532,8 +532,8 @@ def form_line_number(self):
 class ScheduleD(PdfMixin, BaseItemized):
     __tablename__ = 'ofec_sched_d_mv'
 
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     committee_name = db.Column('cmte_nm', db.String, doc=docs.COMMITTEE_NAME)
     creditor_debtor_name = db.Column('cred_dbtr_nm', db.String)
     creditor_debtor_last_name = db.Column('cred_dbtr_l_nm', db.String)
@@ -588,7 +588,7 @@ def form_line_number(self):
 class ScheduleE(PdfMixin, BaseItemized):
     __tablename__ = 'ofec_sched_e_mv'
 
-    sub_id = db.Column(db.String, primary_key=True)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
     # Payee info
     payee_prefix = db.Column(db.String)
     payee_name = db.Column('pye_nm', db.String)
@@ -657,7 +657,7 @@ class ScheduleE(PdfMixin, BaseItemized):
     filer_last_name = db.Column('filer_l_nm', db.String)
     filer_suffix = db.Column(db.String)
     transaction_id = db.Column('tran_id', db.String)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     action_code = db.Column('action_cd', db.String)
     action_code_full = db.Column('action_cd_desc', db.String)
     # Auxiliary fields
@@ -772,8 +772,8 @@ class ScheduleF(PdfMixin, BaseItemized):
         lazy='joined',
     )
 
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     committee_designated_coordinated_expenditure_indicator = db.Column('cmte_desg_coord_exp_ind', db.String)
     committee_name = db.Column('cmte_nm', db.String)
     entity_type = db.Column('entity_tp', db.String)
@@ -888,8 +888,8 @@ class ScheduleH4(BaseItemized):
     disbursement_amount = db.Column('disbursement_amount', db.Numeric(30, 2), index=True)
     schedule_type = db.Column('schedule_type', db.String)
     schedule_type_full = db.Column('schedule_type_desc', db.String)
-    sub_id = db.Column(db.Integer, primary_key=True)
-    original_sub_id = db.Column('orig_sub_id', db.Integer)
+    sub_id = db.Column(db.BigInteger, primary_key=True)
+    original_sub_id = db.Column('orig_sub_id', db.BigInteger)
     federal_share = db.Column('fed_share', db.Numeric(14, 2))
     nonfederal_share = db.Column('nonfed_share', db.Numeric(14, 2))
     administrative_voter_drive_activity_indicator = db.Column('admin_voter_drive_acty_ind', db.String)

webservices/common/models/links.py

@@ -30,7 +30,7 @@ class CandidateCommitteeAlternateLink(db.Model):
     __table_args__ = {"schema": "disclosure"}
     __tablename__ = "cand_cmte_linkage_alternate"
 
-    sub_id = db.Column(db.Integer, primary_key=True, doc=docs.SUB_ID)
+    sub_id = db.Column(db.BigInteger, primary_key=True, doc=docs.SUB_ID)
     candidate_id = db.Column(
         "cand_id",
         db.String,

webservices/common/models/national_party.py

@@ -82,7 +82,7 @@ class NationalParty_ScheduleA(db.Model):
     memo_cd_desc = db.Column(db.String)
     memo_text = db.Column(db.String)
     national_cmte_nonfed_acct = db.Column(db.String)
-    orig_sub_id = db.Column(db.Integer)
+    orig_sub_id = db.Column(db.BigInteger)
     party = db.Column(db.String)
     party_account_type = db.Column('party_account', db.String)
     party_full = db.Column(db.String)
@@ -96,7 +96,7 @@ class NationalParty_ScheduleA(db.Model):
     schedule_type_desc = db.Column(db.String)
     state = db.Column(db.String)
     state_full = db.Column(db.String)
-    sub_id = db.Column(db.Integer, primary_key=True, index=True)
+    sub_id = db.Column(db.BigInteger, primary_key=True, index=True)
     tran_id = db.Column(db.String)
     treasurer_name = db.Column(db.String)
     two_year_transaction_period = db.Column(db.Integer)
@@ -152,7 +152,7 @@ class NationalParty_ScheduleB(db.Model):
     memo_cd_desc = db.Column(db.String)
     memo_text = db.Column(db.String)
     national_cmte_nonfed_acct = db.Column('national_cmte_nonfed_acct', db.String)
-    orig_sub_id = db.Column(db.Integer)
+    orig_sub_id = db.Column(db.BigInteger)
     party = db.Column(db.String)
     party_account = db.Column('party_account', db.String)
     party_full = db.Column('party_full', db.String)
@@ -195,7 +195,7 @@ class NationalParty_ScheduleB(db.Model):
     spender_committee_type_full = db.Column('committee_type_full', db.String)
     state = db.Column(db.String)
     state_full = db.Column(db.String)
-    sub_id = db.Column(db.Integer, primary_key=True, index=True)
+    sub_id = db.Column(db.BigInteger, primary_key=True, index=True)
     tran_id = db.Column(db.String)
     treasurer_name = db.Column(db.String)
     two_year_transaction_period = db.Column(db.Integer)

webservices/common/models/operations_log.py

@@ -5,8 +5,8 @@
 class OperationsLog(db.Model):
     __tablename__ = 'fec_operations_log_vw'
 
-    sub_id = db.Column(db.Integer, primary_key=True, doc=docs.SUB_ID)
-    status_num = db.Column(db.Integer, doc=docs.STATUS_NUM)
+    sub_id = db.Column(db.BigInteger, primary_key=True, doc=docs.SUB_ID)
+    status_num = db.Column(db.String, doc=docs.STATUS_NUM)
     form_type = db.Column('form_tp', db.String, doc=docs.FORM_TYPE)
     report_year = db.Column('rpt_yr', db.Integer, doc=docs.REPORT_YEAR)
     candidate_committee_id = db.Column('cand_cmte_id', db.String, doc=docs.CAND_CMTE_ID)