Skip to content

Commit

Permalink
Merge pull request #532 from CityOfNewYork/develop
Browse files Browse the repository at this point in the history
OpenRecords v3.4.7
  • Loading branch information
joelbcastillo authored Aug 25, 2020
2 parents 799c35c + c2ad730 commit 340f14c
Show file tree
Hide file tree
Showing 14 changed files with 182 additions and 13 deletions.
4 changes: 2 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ LDAP_BASE_DN=<LDAP SEARCH BASE>
USE_LOCAL_AUTH=<USE LOCAL AUTHENTICATION>

# ReCaptcha
RECAPTCHA_SITE_KEY=<SITE KEY>
RECAPTCHA_SECRET_KEY=<SECRET KEY>
RECAPTCHA_SITE_KEY_V3=<SITE KEY>
RECAPTCHA_SECRET_KEY_V3=<SECRET KEY>

# Sentry
SENTRY_DSN=<SENTRY DSN>
Expand Down
152 changes: 152 additions & 0 deletions app/lib/recaptcha_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
import logging

import requests
from flask import Markup, current_app, json, request
from wtforms import ValidationError
from wtforms.fields import HiddenField
from wtforms.widgets import HiddenInput

logger = logging.getLogger(__name__)

JSONEncoder = json.JSONEncoder

RECAPTCHA_TEMPLATE = """
<script src='https://www.google.com/recaptcha/api.js?render={public_key}&onload=executeRecaptcha{action}' async defer></script>
<script>
var executeRecaptcha{action} = function() {{
console.log("grecaptcha is ready!");
grecaptcha.execute('{public_key}', {{action: '{action}'}}).then(function(token) {{
console.log(token);
document.getElementById("{field_name}").value = token;
}});
}};
</script>
<input type="hidden" id="{field_name}" name="{field_name}">
"""

RECAPTCHA_TEMPLATE_MANUAL = """
<script src='https://www.google.com/recaptcha/api.js?render={public_key}' async defer></script>
<script>
var executeRecaptcha{action} = function() {{
console.log("executeRecaptcha{action}() is called!");
grecaptcha.ready(function() {{
console.log("grecaptcha is ready!");
grecaptcha.execute('{public_key}', {{action: '{action}'}}).then(function(token) {{
console.log(token);
document.getElementById("{field_name}").value = token;
}});
}});
}};
</script>
<input type="hidden" id="{field_name}" name="{field_name}">
"""

RECAPTCHA_VERIFY_SERVER = "https://www.google.com/recaptcha/api/siteverify"
RECAPTCHA_ERROR_CODES = {
"missing-input-secret": "The secret parameter is missing.",
"invalid-input-secret": "The secret parameter is invalid or malformed.",
"missing-input-response": "The response parameter is missing.",
"invalid-input-response": "The response parameter is invalid or malformed.",
}


class Recaptcha3Validator(object):
"""Validates a ReCaptcha."""

def __init__(self, message=None):
if message is None:
message = "Please verify that you are not a robot."
self.message = message

def __call__(self, form, field):
if current_app.testing:
return True

token = field.data
if not token:
logger.warning(
"Token is not ready or incorrect configuration (check JavaScript error log)."
)
raise ValidationError(field.gettext(self.message))

remote_ip = request.remote_addr
if not Recaptcha3Validator._validate_recaptcha(field, token, remote_ip):
field.recaptcha_error = "incorrect-captcha-sol"
raise ValidationError(field.gettext(self.message))

@staticmethod
def _validate_recaptcha(field, response, remote_addr):
"""Performs the actual validation."""
try:
private_key = current_app.config["RECAPTCHA3_PRIVATE_KEY"]
except KeyError:
raise RuntimeError("RECAPTCHA3_PRIVATE_KEY is not set in app config.")

data = {"secret": private_key, "remoteip": remote_addr, "response": response}

http_response = requests.post(RECAPTCHA_VERIFY_SERVER, data)
if http_response.status_code != 200:
return False

json_resp = http_response.json()
if (
json_resp["success"]
and json_resp["action"] == field.action
and json_resp["score"] > field.score_threshold
):
logger.info(json_resp)
return True
else:
logger.warning(json_resp)

for error in json_resp.get("error-codes", []):
if error in RECAPTCHA_ERROR_CODES:
raise ValidationError(RECAPTCHA_ERROR_CODES[error])

return False


class Recaptcha3Widget(HiddenInput):
def __call__(self, field, **kwargs):
"""Returns the recaptcha input HTML."""
public_key_name = "RECAPTCHA3_PUBLIC_KEY"
try:
public_key = current_app.config[public_key_name]
except KeyError:
raise RuntimeError("{public_key_name} is not set in app config.")

return Markup(
(
RECAPTCHA_TEMPLATE
if field.execute_on_load
else RECAPTCHA_TEMPLATE_MANUAL
).format(public_key=public_key, action=field.action, field_name=field.name)
)


class Recaptcha3Field(HiddenField):
widget = Recaptcha3Widget()

# error message if recaptcha validation fails
recaptcha_error = None

def __init__(
self,
action,
score_threshold=0.5,
execute_on_load=True,
validators=None,
**kwargs
):
"""If execute_on_load is False, recaptcha.execute needs to be manually bound to an event to obtain token,
the JavaScript function to call is executeRecaptcha{action}, e.g. onsubmit="executeRecaptchaSignIn" """
if not action:
# TODO: more validation on action, see https://developers.google.com/recaptcha/docs/v3#actions
# "actions may only contain alphanumeric characters and slashes, and must not be user-specific"
raise RuntimeError("action must not be none or empty.")

self.action = action
self.execute_on_load = execute_on_load
self.score_threshold = score_threshold
validators = validators or [Recaptcha3Validator()]
super(Recaptcha3Field, self).__init__(validators=validators, **kwargs)
5 changes: 4 additions & 1 deletion app/main/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from app.lib.db_utils import create_object, update_object
from app.lib.email_utils import send_contact_email
from app.models import Emails, Users
from app.request.forms import TechnicalSupportForm
from . import main


Expand Down Expand Up @@ -46,6 +47,8 @@ def status():
@main.route('/contact', methods=['GET', 'POST'])
@main.route('/technical-support', methods=['GET', 'POST'])
def technical_support():
form = TechnicalSupportForm()

if request.method == 'POST':
name = request.form.get('name')
email = request.form.get('email')
Expand Down Expand Up @@ -74,7 +77,7 @@ def technical_support():
else:
flash('Cannot send email.', category='danger')
error_id = request.args.get('error_id', '')
return render_template('main/contact.html', error_id=error_id)
return render_template('main/contact.html', error_id=error_id, form=form)


@main.route('/faq', methods=['GET'])
Expand Down
10 changes: 10 additions & 0 deletions app/request/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
)
from app.lib.db_utils import get_agency_choices
from app.models import Reasons, LetterTemplates, EnvelopeTemplates, CustomRequestForms
from app.lib.recaptcha_utils import Recaptcha3Field


class PublicUserRequestForm(Form):
Expand All @@ -53,6 +54,8 @@ class PublicUserRequestForm(Form):
# File Upload
request_file = FileField("Upload File (optional, must be less than 20 Mb)")

recaptcha = Recaptcha3Field(action="TestAction", execute_on_load=True)

# Submit Button
submit = SubmitField("Submit Request")

Expand Down Expand Up @@ -119,6 +122,8 @@ class AgencyUserRequestForm(Form):
# File Upload
request_file = FileField("Upload File (optional, must be less than 20 Mb)")

recaptcha = Recaptcha3Field(action="TestAction", execute_on_load=True)

# Submit Button
submit = SubmitField("Submit Request")

Expand Down Expand Up @@ -175,6 +180,7 @@ class AnonymousRequestForm(Form):
# File Upload
request_file = FileField("Upload File (optional, must be less than 20 Mb)")

recaptcha = Recaptcha3Field(action="TestAction", execute_on_load=True)
submit = SubmitField("Submit Request")

def __init__(self):
Expand Down Expand Up @@ -523,3 +529,7 @@ def __init__(self, request):
request.requester.notification_email or request.requester.email
)
self.subject.data = "Inquiry about {}".format(request.id)


class TechnicalSupportForm(Form):
recaptcha = Recaptcha3Field(action="TestAction", execute_on_load=True)
5 changes: 1 addition & 4 deletions app/request/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ def new():
:return: redirect to homepage on successful form validation
if form fields are missing or has improper values, backend error messages (WTForms) will appear
"""
site_key = current_app.config["RECAPTCHA_SITE_KEY"]

kiosk_mode = eval_request_bool(escape(flask_request.args.get("kiosk_mode", False)))
category = str(escape(flask_request.args.get("category", None)))
agency = str(escape(flask_request.args.get("agency", None)))
Expand Down Expand Up @@ -105,7 +103,7 @@ def new():
upload_path = handle_upload_no_id(form.request_file)
if form.request_file.errors:
return render_template(
new_request_template, form=form, site_key=site_key
new_request_template, form=form
)

custom_metadata = json.loads(
Expand Down Expand Up @@ -203,7 +201,6 @@ def new():
return render_template(
new_request_template,
form=form,
site_key=site_key,
kiosk_mode=kiosk_mode,
category=category,
agency=agency,
Expand Down
1 change: 1 addition & 0 deletions app/templates/main/contact.html
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ <h1 class="text-center">Technical Support</h1>
<p id="message-character-count" class="character-counter">5000 characters remaining</p>

<br>
{{ form.recaptcha }}
<input id="submit" name="submit" type="submit" value="Send" aria-label="Submit button">
</form>
</div>
Expand Down
3 changes: 3 additions & 0 deletions app/templates/main/home.html
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
{% endblock %}
{% block content %}
<div class="container col-sm-12">
<div class="alert alert-info home-page-description">Don’t forget the census. Take a few minutes to help NYC get its fair share of
federal funds. Go to <a href="https://my2020census.gov/">my2020census.gov</a> to complete the census.
</div>
<div class=" col-sm-8" role="main">
<h1 class="home-page-header">Welcome to NYC Government’s home for filing Freedom of Information Law (FOIL)
requests.
Expand Down
2 changes: 1 addition & 1 deletion app/templates/report/reports.html
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ <h1 class="text-center">FOIL Request Stats</h1>
<button id="submit-button" class="btn btn-success">Submit</button>
<button id="clear-filter-button" class="btn btn-primary">Clear Filter</button>
</div>
{% if current_user.is_agency_admin() %}
{% if not current_user.is_anonymous and current_user.is_agency_admin() %}
<br>
<hr>
<div class="container">
Expand Down
1 change: 1 addition & 0 deletions app/templates/request/new_request_agency.html
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ <h3>Address</h3>
placeholder="12345") }}<br>
</div>
<input type="hidden" name="tz-name">
{{ form.recaptcha }}
{{ form.submit(id="submit", class="btn-primary") }}
{% include "request/_pii_warning_modal.html" %}
<span id="processing-submission" hidden>
Expand Down
1 change: 1 addition & 0 deletions app/templates/request/new_request_anon.html
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ <h3>Address</h3>
</div>
</div>
<input type="hidden" name="tz-name">
{{ form.recaptcha }}
{{ form.submit(id="submit", class="btn-primary") }}
{% include 'request/_pii_warning_modal.html' %}
<span id="processing-submission" hidden>
Expand Down
1 change: 1 addition & 0 deletions app/templates/request/new_request_user.html
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ <h1>Request a Record</h1>
</div>
<br><br>
<input type="hidden" name="tz-name">
{{ form.recaptcha }}
{{ form.submit(id="submit", class="btn-primary") }}
{% include "request/_pii_warning_modal.html" %}
<span id="processing-submission" hidden>
Expand Down
4 changes: 2 additions & 2 deletions build_scripts/default/nginx_conf/sites/openrecords_v2_0.conf
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ server {
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/; font-src 'self' maxcdn.bootstrapcdn.com; object-src 'self' blob ajax.googleapis.com";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/ https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/; font-src 'self' maxcdn.bootstrapcdn.com; object-src 'self' blob ajax.googleapis.com; frame-src 'self' https://www.google.com;";

client_max_body_size 20M;

Expand Down Expand Up @@ -109,7 +109,7 @@ server {
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/; font-src maxcdn.bootstrapcdn.com; object-src 'self' ajax.googleapis.com";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/ https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/; font-src 'self' maxcdn.bootstrapcdn.com; object-src 'self' blob ajax.googleapis.com; frame-src 'self' https://www.google.com;";

client_max_body_size 20M;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ server {
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/; font-src 'self' maxcdn.bootstrapcdn.com; object-src 'self' ajax.googleapis.com";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' www.gstatic.com www.google.com translate.googleapis.com http://www1.nyc.gov/ data:; style-src 'self' 'unsafe-inline' code.jquery.com ajax.googleapis.com maxcdn.bootstrapcdn.com translate.googleapis.com http://www1.nyc.gov/; script-src 'self' 'unsafe-inline' 'unsafe-eval' translate.google.com translate.googleapis.com ajax.googleapis.com code.jquery.com http://www1.nyc.gov/ https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/; font-src 'self' maxcdn.bootstrapcdn.com; object-src 'self' blob ajax.googleapis.com; frame-src 'self' https://www.google.com;";

client_max_body_size 20M;

Expand Down
4 changes: 2 additions & 2 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ class Config:
os.path.join(os.path.abspath(os.path.dirname(__file__)), 'magic'))

# ReCaptcha
RECAPTCHA_SITE_KEY = os.environ.get('RECAPTCHA_SITE_KEY')
RECAPTCHA_SECRET_KEY = os.environ.get('RECAPTCHA_SECRET_KEY')
RECAPTCHA3_PUBLIC_KEY = os.environ.get("RECAPTCHA_SITE_KEY_V3", "")
RECAPTCHA3_PRIVATE_KEY = os.environ.get("RECAPTCHA_SECRET_KEY_V3", "")

# ElasticSearch settings
ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or "localhost:9200"
Expand Down

0 comments on commit 340f14c

Please sign in to comment.