OpenRecords v3.5 - Develop

.env.example

@@ -110,8 +110,10 @@ LDAP_BASE_DN=<LDAP SEARCH BASE>
 USE_LOCAL_AUTH=<USE LOCAL AUTHENTICATION>
 
 # ReCaptcha
-RECAPTCHA_SITE_KEY_V3=<SITE KEY>
-RECAPTCHA_SECRET_KEY_V3=<SECRET KEY>
+RECAPTCHA_ENABLED=
+RECAPTCHA_PUBLIC_KEY=
+RECAPTCHA_PRIVATE_KEY=
+RECAPTCHA_THRESHOLD=
 
 # Sentry
 SENTRY_DSN=<SENTRY DSN>

app/agency/api/views.py

@@ -90,9 +90,7 @@ def get_custom_request_form_options(agency_ein):
                                                                   CustomRequestForms.repeatable,
                                                                   CustomRequestForms.category,
                                                                   CustomRequestForms.minimum_required).filter_by(
-        agency_ein=agency_ein).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.id)).all()
-    # Convert the results of with_entities back to tuple format so that jsonify can be used
-    custom_request_forms = [tuple(form) for form in custom_request_forms]
+        agency_ein=agency_ein, is_active=True).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.order)).all()
     return jsonify(custom_request_forms), 200
 
 
@@ -177,12 +175,19 @@ def get_request_types(agency_ein):
     if current_user.is_agency_active(agency_ein):
         agency = Agencies.query.filter_by(ein=agency_ein).one_or_none()
         if agency is not None and agency.agency_features['custom_request_forms']['enabled']:
-            request_types = [
+            active_request_types = [
                 (custom_request_form.form_name, custom_request_form.form_name)
                 for custom_request_form in CustomRequestForms.query.filter_by(
-                    agency_ein=agency_ein
-                ).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.id)).all()
+                    agency_ein=agency_ein, is_active=True
+                ).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.order)).all()
             ]
+            inactive_request_types = [
+                (custom_request_form.form_name, "(Inactive) " + custom_request_form.form_name)
+                for custom_request_form in CustomRequestForms.query.filter_by(
+                    agency_ein=agency_ein, is_active=False
+                ).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.order)).all()
+            ]
+            request_types = active_request_types + inactive_request_types
             request_types.insert(0, ("", "All"))
             return jsonify({"request_types": request_types}), 200
 

app/jobs.py

@@ -7,13 +7,13 @@
 from psycopg2 import OperationalError
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import calendar, db, store
-from app.constants import OPENRECORDS_DL_EMAIL, request_status
+from app import calendar, sentry, store, db
+from app.constants import OPENRECORDS_DL_EMAIL, request_status, determination_type
 from app.constants.event_type import EMAIL_NOTIFICATION_SENT, REQ_STATUS_CHANGED
 from app.constants.response_privacy import PRIVATE
 from app.lib.db_utils import create_object, update_object
 from app.lib.email_utils import send_email
-from app.models import Agencies, Emails, Events, Requests, Users
+from app.models import Agencies, Emails, Events, Requests, Responses, Determinations, Users
 
 # NOTE: (For Future Reference)
 # If we find ourselves in need of a request context, app.test_request_context() might come in handy.
@@ -45,81 +45,104 @@ def _update_request_statuses():
     due_soon_date = calendar.addbusdays(
         now, current_app.config['DUE_SOON_DAYS_THRESHOLD']
     ).replace(hour=23, minute=59, second=59)  # the entire day
+    request_errors = []
 
     agencies = Agencies.query.with_entities(Agencies.ein).filter_by(is_active=True).all()
     for agency_ein, in agencies:
+        # Overdue requests
         requests_overdue = Requests.query.filter(
             Requests.due_date < now,
             Requests.status != request_status.CLOSED,
             Requests.agency_ein == agency_ein
         ).order_by(
-            Requests.due_date.asc()
+            Requests.id.asc()
         ).all()
 
+        # Query for all acknowledged overdue requests
+        agency_requests_overdue = Requests.query.join(Responses, Determinations).filter(
+            Requests.due_date < now,
+            Requests.status != request_status.CLOSED,
+            Requests.agency_ein == agency_ein,
+            Determinations.dtype == determination_type.ACKNOWLEDGMENT
+        ).order_by(
+            Requests.id.asc()
+        ).all()
+
+        # Get the difference for all unacknowledged overdue requests
+        agency_acknowledgments_overdue = list(set(requests_overdue) - set(agency_requests_overdue))
+        agency_acknowledgments_overdue.sort(key=lambda x: x.id)
+
+        # Due soon requests
         requests_due_soon = Requests.query.filter(
             Requests.due_date > now,
             Requests.due_date <= due_soon_date,
             Requests.status != request_status.CLOSED,
             Requests.agency_ein == agency_ein
         ).order_by(
-            Requests.due_date.asc()
+            Requests.id.asc()
+        ).all()
+
+        # Query for all acknowledged due soon requests
+        agency_requests_due_soon = Requests.query.join(Responses, Determinations).filter(
+            Requests.due_date > now,
+            Requests.due_date <= due_soon_date,
+            Requests.status != request_status.CLOSED,
+            Requests.agency_ein == agency_ein,
+            Determinations.dtype == determination_type.ACKNOWLEDGMENT
+        ).order_by(
+            Requests.id.asc()
         ).all()
 
+        # Get the difference for all unacknowledged due soon requests
+        agency_acknowledgments_due_soon = list(set(requests_due_soon) - set(agency_requests_due_soon))
+        agency_acknowledgments_due_soon.sort(key=lambda x: x.id)
+
         if not requests_overdue and not requests_due_soon:
             continue
 
-        agency_requests_overdue = []
-        agency_acknowledgments_overdue = []
-        agency_requests_due_soon = []
-        agency_acknowledgments_due_soon = []
-
         # OVERDUE
         for request in requests_overdue:
-
-            if request.was_acknowledged:
-                agency_requests_overdue.append(request)
-            else:
-                agency_acknowledgments_overdue.append(request)
-
             if request.status != request_status.OVERDUE:
-                create_object(
-                    Events(
-                        request.id,
-                        user_guid=None,
-                        type_=REQ_STATUS_CHANGED,
-                        previous_value={"status": request.status},
-                        new_value={"status": request_status.OVERDUE},
-                        response_id=None,
+                try:
+                    update_object(
+                        {"status": request_status.OVERDUE},
+                        Requests,
+                        request.id)
+                    create_object(
+                        Events(
+                            request.id,
+                            user_guid=None,
+                            type_=REQ_STATUS_CHANGED,
+                            previous_value={"status": request.status},
+                            new_value={"status": request_status.OVERDUE},
+                            response_id=None,
+                        )
                     )
-                )
-                update_object(
-                    {"status": request_status.OVERDUE},
-                    Requests,
-                    request.id)
+                except Exception:
+                    request_errors.append(
+                        (request.id, traceback.format_exc().replace("\n", "<br/>").replace(" ", "&nbsp;")))
 
         # DUE SOON
         for request in requests_due_soon:
-
-            if request.was_acknowledged:
-                agency_requests_due_soon.append(request)
-            else:
-                agency_acknowledgments_due_soon.append(request)
-
             if request.status != request_status.DUE_SOON:
-                create_object(
-                    Events(
-                        request.id,
-                        user_guid=None,
-                        type_=REQ_STATUS_CHANGED,
-                        previous_value={"status": request.status},
-                        new_value={"status": request_status.DUE_SOON},
-                        response_id=None,
+                try:
+                    update_object(
+                        {"status": request_status.DUE_SOON},
+                        Requests,
+                        request.id)
+                    create_object(
+                        Events(
+                            request.id,
+                            user_guid=None,
+                            type_=REQ_STATUS_CHANGED,
+                            previous_value={"status": request.status},
+                            new_value={"status": request_status.DUE_SOON},
+                            response_id=None,
+                        )
                     )
-                )
-                update_object(
-                    {"status": request_status.DUE_SOON},
-                    Requests,
-                    request.id)
+                except Exception:
+                    request_errors.append(
+                        (request.id, traceback.format_exc().replace("\n", "<br/>").replace(" ", "&nbsp;")))
 
         # mail to agency admins for each agency
         user_emails = list(set(admin.notification_email or admin.email for admin
@@ -161,6 +184,13 @@ def _update_request_statuses():
                 timestamp=datetime.utcnow()
             )
         )
+    send_email(
+        'Update Request Statuses Job Finished',
+        to=[OPENRECORDS_DL_EMAIL],
+        template='email_templates/email_update_request_statuses_job_finished',
+        timestamp=str(datetime.utcnow()),
+        request_errors=request_errors
+    )
 
 
 @app.task(autoretry_for=(OperationalError, SQLAlchemyError,), retry_kwargs={'max_retries': 5}, retry_backoff=True)

app/lib/__init__.py

@@ -46,6 +46,15 @@ def _populate(self, year):
         elif year >= 1888:
             self[date(year, 5, 30)] = "Memorial Day"
 
+        # Juneteenth Day
+        if year > 2021:
+            name = "Juneteenth Day"
+            self[date(year, 6, 19)] = name
+            if self.observed and date(year, 6, 19).weekday() == 5:
+                self[date(year, 6, 19) + rd(days=-1)] = name + " (Observed)"
+            elif self.observed and date(year, 6, 19).weekday() == 6:
+                self[date(year, 6, 19) + rd(days=+1)] = name + " (Observed)"
+
         # Independence Day
         if year > 1870:
             name = "Independence Day"

app/lib/email_utils.py

@@ -35,8 +35,8 @@ def send_async_email(msg):
         current_app.logger.exception("Failed to Send Email {} : {}".format(msg, e))
 
 
-def send_contact_email(subject, recipients, body, sender):
-    msg = Message(subject, recipients, body, sender=sender)
+def send_contact_email(subject, recipients, body, sender, reply_to):
+    msg = Message(subject, recipients, body, sender=sender, reply_to=reply_to)
     send_async_email.delay(msg)