Merge pull request #527 from CityOfNewYork/develop

OpenRecords v3.4.5

app/report/forms.py

@@ -3,7 +3,7 @@
 
     :synopsis: Defines forms used for report statistics.
 """
-from datetime import date
+from datetime import date, timedelta
 
 from flask_login import current_user
 from flask_wtf import Form
@@ -74,3 +74,29 @@ def __init__(self):
             years_active.append((str(year), str(year)))
         self.year.choices = years_active
         self.year.choices.insert(0, ('', ''))
+
+
+class OpenDataReportForm(Form):
+    """Form to generate a report with Open Data compliance data."""
+    date_from = DateField('Date From (required)', id='open-data-date-from', format='%m/%d/%Y', validators=[DataRequired()])
+    date_to = DateField('Date To (required)', id='open-data-date-to', format='%m/%d/%Y', validators=[DataRequired()])
+    submit_field = SubmitField('Generate Report')
+
+    def validate(self):
+        if not super().validate():
+            return False
+        is_valid = True
+        for field in [self.date_from, self.date_to]:
+            if field.data > date.today():
+                field.errors.append('The {} cannot be greater than today.'.format(field.label.text))
+                is_valid = False
+        if self.date_to.data < self.date_from.data:
+            field.errors.append('Date To cannot be before Date From.')
+            is_valid = False
+        if self.date_from.data == self.date_to.data:
+            field.errors.append('The dates cannot be the same.')
+            is_valid = False
+        if self.date_from.data + timedelta(days=365) < self.date_to.data:
+            field.errors.append('Date From and Date To must be within one year.')
+            is_valid = False
+        return is_valid

app/report/utils.py

@@ -1,16 +1,18 @@
 from datetime import datetime, timedelta
 
 import tablib
-from flask import current_app
-from sqlalchemy import asc, func, Date
+from flask import current_app, url_for, request as flask_request
+from sqlalchemy import asc, func, Date, or_
 from sqlalchemy.orm import joinedload
+from urllib.parse import urljoin
 
-from app import celery
+from app import celery, db
 from app.constants.event_type import REQ_ACKNOWLEDGED, REQ_CREATED, REQ_CLOSED, REQ_DENIED
+from app.constants.response_privacy import PRIVATE, RELEASE_AND_PRIVATE, RELEASE_AND_PUBLIC
 from app.constants.request_status import OPEN, IN_PROGRESS, DUE_SOON, OVERDUE, CLOSED
 from app.lib.date_utils import local_to_utc, utc_to_local
 from app.lib.email_utils import send_email
-from app.models import Agencies, Emails, Events, Requests, Responses, Users
+from app.models import Agencies, Emails, Events, Requests, Responses, Users, Files
 
 
 @celery.task(bind=True, name='app.report.utils.generate_acknowledgment_report')
@@ -495,3 +497,166 @@ def generate_monthly_metrics_report(self, agency_ein: str, date_from: str, date_
                attachment=excel_spreadsheet.export('xls'),
                filename='FOIL_monthly_metrics_report_{}_{}.xls'.format(date_from, date_to),
                mimetype='application/octet-stream')
+
+
+def generate_open_data_report(agency_ein: str, date_from: datetime, date_to: datetime):
+    """Generates a report of Open Data compliance.
+
+    Generates a report of requests in a time frame with the following tabs:
+    1) All request responses that could contain possible data sets.
+    2) All requests submitted during that given time frame.
+
+    Args:
+        agency_ein: Agency EIN
+        date_from: Date to filter from
+        date_to: Date to filter to
+    """
+
+    # Query for all responses that are possible data sets in the given date range
+    possible_data_sets = db.session.query(Requests,
+                                          Responses,
+                                          Files).join(Responses,
+                                                      Requests.id == Responses.request_id).join(Files,
+                                                                                                Responses.id == Files.id).with_entities(
+        Requests.id,
+        func.to_char(Requests.date_submitted, 'MM/DD/YYYY'),
+        func.to_char(Requests.date_closed, 'MM/DD/YYYY'),
+        Requests.title,
+        Requests.description,
+        Requests.custom_metadata,
+        func.to_char(Responses.date_modified, 'MM/DD/YYYY'),
+        Responses.privacy,
+        func.to_char(Responses.release_date, 'MM/DD/YYYY'),
+        Responses.id).filter(
+        Requests.agency_ein == agency_ein,
+        Requests.date_submitted.between(date_from, date_to),
+        Responses.privacy != PRIVATE,
+        or_(Files.name.ilike('%xlsx'),
+            Files.name.ilike('%csv'),
+            Files.name.ilike('%txt'),
+            Files.name.ilike('%xls'),
+            Files.name.ilike('%data%'),
+            Files.title.ilike('%data%'))).all()
+
+    # Process requests for the spreadsheet
+    possible_data_sets_processed = []
+    for request in possible_data_sets:
+        request = list(request)
+
+        # Change privacy value text
+        if request[7] == RELEASE_AND_PRIVATE:
+            request[7] = 'Release and Private - Agency and Requester Only'
+        elif request[7] == RELEASE_AND_PUBLIC:
+            request[7] = 'Public'
+
+        # Check if custom_metadata exists
+        if request[5] == {} or request[5] is None:
+            # Remove custom_metadata from normal requests
+            del request[5]
+        else:
+            # Process custom metadata
+            custom_metadata = request[5]
+            custom_metadata_text = ''
+            for form_number, form_values in sorted(custom_metadata.items()):
+                custom_metadata_text = custom_metadata_text + 'Request Type: ' + form_values['form_name'] + '\n\n'
+                for field_number, field_values in sorted(form_values['form_fields'].items()):
+                    # Make sure field_value exists otherwise give it a default value
+                    field_value = field_values.get('field_value', '')
+                    # Truncate field_value to 5000 characters for Excel limitations
+                    field_value = field_value[:5000]
+                    # Set field_value to empty string if None (used for select multiple empty value)
+                    if field_value is None:
+                        field_value = ''
+                    if isinstance(field_value, list):
+                        custom_metadata_text = custom_metadata_text + field_values[
+                            'field_name'] + ':\n' + ', '.join(field_values.get('field_value', '')) + '\n\n'
+                    else:
+                        custom_metadata_text = custom_metadata_text + field_values['field_name'] + ':\n' + field_value + '\n\n'
+                custom_metadata_text = custom_metadata_text + '\n'
+            # Replace normal request description with processed metadata string
+            request[4] = custom_metadata_text
+            del request[5]
+
+        # Add URL for response
+        response_id = request[8]
+        del request[8]
+        request.append(urljoin(flask_request.host_url, url_for('response.get_response_content', response_id=response_id)))
+        possible_data_sets_processed.append(request)
+
+    # Create "Possible Data Sets" data set
+    possible_data_sets_headers = ('Request ID',
+                                  'Request - Date Submitted',
+                                  'Request - Date Closed',
+                                  'Request - Title',
+                                  'Request - Description',
+                                  'Response - Date Added',
+                                  'Privacy / Visibility',
+                                  'Response - Publish Date',
+                                  'URL')
+    possible_data_sets_dataset = tablib.Dataset(*possible_data_sets_processed,
+                                                headers=possible_data_sets_headers,
+                                                title='Possible Data Sets')
+
+    # Query for all requests submitted in the given date range
+    all_requests = Requests.query.with_entities(
+        Requests.id,
+        func.to_char(Requests.date_submitted, 'MM/DD/YYYY'),
+        func.to_char(Requests.date_closed, 'MM/DD/YYYY'),
+        Requests.title,
+        Requests.description,
+        Requests.custom_metadata
+    ).filter(
+        Requests.date_submitted.between(date_from, date_to),
+        Requests.agency_ein == agency_ein,
+    ).order_by(asc(Requests.date_submitted)).all()
+
+    # Process requests for the spreadsheet
+    all_requests_processed = []
+    for request in all_requests:
+        request = list(request)
+
+        # Check if custom_metadata exists
+        if request[5] == {} or request[5] is None:
+            del request[5]
+        else:
+            # Process custom metadata
+            custom_metadata = request[5]
+            custom_metadata_text = ''
+            for form_number, form_values in sorted(custom_metadata.items()):
+                custom_metadata_text = custom_metadata_text + 'Request Type: ' + form_values['form_name'] + '\n\n'
+                for field_number, field_values in sorted(form_values['form_fields'].items()):
+                    field_value = field_values.get('field_value', '')
+                    field_value = field_value[:5000]
+                    if field_value is None:
+                        field_value = ''
+                    if isinstance(field_value, list):
+                        custom_metadata_text = custom_metadata_text + field_values[
+                            'field_name'] + ':\n' + ', '.join(field_values.get('field_value', '')) + '\n\n'
+                    else:
+                        custom_metadata_text = custom_metadata_text + field_values['field_name'] + ':\n' + field_value + '\n\n'
+                custom_metadata_text = custom_metadata_text + '\n'
+            request[4] = custom_metadata_text
+            del request[5]
+
+        # Add URL to request
+        request.append(urljoin(flask_request.host_url, url_for('request.view', request_id=request[0])))
+        all_requests_processed.append(request)
+
+    # Create "All Requests" data set
+    all_requests_headers = ('Request ID',
+                            'Request - Date Submitted',
+                            'Request - Date Closed',
+                            'Request - Title',
+                            'Request - Description',
+                            'URL')
+    all_requests_dataset = tablib.Dataset(*all_requests_processed,
+                                          headers=all_requests_headers,
+                                          title='All Requests')
+
+    # Create Databook from Datasets
+    excel_spreadsheet = tablib.Databook((
+        possible_data_sets_dataset,
+        all_requests_dataset,
+    ))
+
+    return excel_spreadsheet.export('xls')

app/report/views.py

@@ -3,8 +3,9 @@
 
    :synopsis: Handles the report URL endpoints for the OpenRecords application
 """
-from datetime import datetime
+from datetime import datetime, timedelta
 from calendar import monthrange
+from io import BytesIO
 
 from flask import (
     current_app,
@@ -14,6 +15,7 @@
     redirect,
     request,
     url_for,
+    send_file
 )
 from flask_login import current_user, login_required
 
@@ -27,8 +29,17 @@
     UserRequests
 )
 from app.report import report
-from app.report.forms import AcknowledgmentForm, ReportFilterForm, MonthlyMetricsReportForm
-from app.report.utils import generate_acknowledgment_report, generate_monthly_metrics_report
+from app.report.forms import (
+    AcknowledgmentForm,
+    ReportFilterForm,
+    MonthlyMetricsReportForm,
+    OpenDataReportForm
+)
+from app.report.utils import (
+    generate_acknowledgment_report,
+    generate_monthly_metrics_report,
+    generate_open_data_report
+)
 
 
 @report.route('/show', methods=['GET'])
@@ -41,7 +52,8 @@ def show_report():
     return render_template('report/reports.html',
                            acknowledgment_form=AcknowledgmentForm(),
                            monthly_report_form=MonthlyMetricsReportForm(),
-                           report_filter_form=ReportFilterForm())
+                           report_filter_form=ReportFilterForm(),
+                           open_data_report_form=OpenDataReportForm())
 
 
 @report.route('/', methods=['GET'])
@@ -187,3 +199,41 @@ def monthly_metrics_report():
         for field, _ in monthly_report_form.errors.items():
             flash(monthly_report_form.errors[field][0], category='danger')
     return redirect(url_for("report.show_report"))
+
+
+@report.route('/open-data-report', methods=['POST'])
+@login_required
+def open_data_report():
+    """Generates the Open Data Compliance report.
+
+    Returns:
+        Template with context.
+
+    """
+    open_data_report_form = OpenDataReportForm()
+    if open_data_report_form.validate_on_submit():
+        # Only agency administrators can access endpoint
+        if not current_user.is_agency_admin:
+            return jsonify({
+                'error': 'Only Agency Administrators can access this endpoint.'
+            }), 403
+
+        date_from = local_to_utc(datetime.strptime(request.form['date_from'], '%m/%d/%Y'),
+                                 current_app.config['APP_TIMEZONE'])
+        date_to = local_to_utc(datetime.strptime(request.form['date_to'], '%m/%d/%Y'),
+                               current_app.config['APP_TIMEZONE']) + timedelta(days=1)
+
+        open_data_report_spreadsheet = generate_open_data_report(current_user.default_agency_ein,
+                                                                 date_from,
+                                                                 date_to)
+        date_from_string = date_from.strftime('%Y%m%d')
+        date_to_string = date_to.strftime('%Y%m%d')
+        return send_file(
+            BytesIO(open_data_report_spreadsheet),
+            attachment_filename='open_data_compliance_report_{}_{}.xls'.format(date_from_string, date_to_string),
+            as_attachment=True
+        )
+    else:
+        for field, _ in open_data_report_form.errors.items():
+            flash(open_data_report_form.errors[field][0], category='danger')
+    return redirect(url_for('report.show_report'))

app/templates/report/reports.html

@@ -34,7 +34,7 @@ <h1 class="text-center">FOIL Request Stats</h1>
             <button id="submit-button" class="btn btn-success">Submit</button>
             <button id="clear-filter-button" class="btn btn-primary">Clear Filter</button>
         </div>
-        {% if current_user.is_agency_admin %}
+        {% if current_user.is_agency_admin() %}
             <br>
             <hr>
             <div class="container">
@@ -77,6 +77,21 @@ <h3>Monthly FOIL Metrics</h3>
                 </div>
                 {{ monthly_report_form.submit_field(class="btn btn-success") }}
             </form>
+            <br>
+            <hr>
+            <h3>Open Data Compliance Report</h3>
+            <form id="open-data-report-form" action="/report/open-data-report" method="POST" target="_blank" rel="noreferrer" data-parsley-validate>
+                {{ open_data_report_form.csrf_token }}
+                <div class="form-group">
+                    {{ open_data_report_form.date_from.label }}
+                    {{ open_data_report_form.date_from }}
+                </div>
+                <div class="form-group">
+                    {{ open_data_report_form.date_to.label }}
+                    {{ open_data_report_form.date_to }}
+                </div>
+                {{ open_data_report_form.submit_field(class="btn btn-success") }}
+            </form>
         {% endif %}
         </div>
     </div>

app/templates/report/reports.js.html

@@ -17,6 +17,9 @@
     var monthly_report_month = $("#monthly-report-month");
     var monthly_report_year = $("#monthly-report-year");
 
+    var open_data_date_from = $("#open-data-date-from");
+    var open_data_date_to = $("#open-data-date-to");
+
     function drawChart(labels, values) {
         reportChart = new Chart(ctx, {
             type: 'bar',
@@ -279,4 +282,33 @@
     monthly_report_year.attr('data-parsley-required-message',
     '<span class=\"glyphicon glyphicon-exclamation-sign\"></span>&nbsp;' +
     '<strong>Error, Year is required.</strong> Please select a year from the drop-down menu.');
+
+    open_data_date_from.attr('data-parsley-valid-date', '');
+    open_data_date_to.attr('data-parsley-valid-date', '');
+    open_data_date_from.attr("data-parsley-required-message",
+        '<span class=\"glyphicon glyphicon-exclamation-sign\"></span>&nbsp;' +
+        '<strong>Error, date from is required.</strong> Please select a date from the datepicker.');
+    open_data_date_to.attr("data-parsley-required-message",
+        '<span class=\"glyphicon glyphicon-exclamation-sign\"></span>&nbsp;' +
+        '<strong>Error, date to is required.</strong> Please select a date from the datepicker.');
+
+    open_data_date_from.datepicker({
+        dateFormat: "mm/dd/yy",
+        maxDate: 0
+    })
+        .mask("99/99/9999")
+        .keydown(function (e) {
+            // prevent keyboard input
+            e.preventDefault();
+        });
+
+    open_data_date_to.datepicker({
+        dateFormat: "mm/dd/yy",
+        maxDate: 0
+    })
+        .mask("99/99/9999")
+        .keydown(function (e) {
+            // prevent keyboard input
+            e.preventDefault();
+        });
 </script>