[Python3 and Django2.2 upgrade] part 6 - Settings files refactoring

dependencies/pip/dev.txt

@@ -5,6 +5,7 @@
 #    pip-compile dependencies/pip/dev.in
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
@@ -15,8 +16,10 @@ backcall==0.2.0           # via ipython
 backports.csv==1.0.7      # via formpack
 begins==0.9               # via formpack
 billiard==3.6.3.0         # via celery
+boto3==1.14.44            # via -r dependencies/pip/requirements.in
+botocore==1.17.44         # via boto3, s3transfer
 celery[redis]==4.4.6      # via -r dependencies/pip/requirements.in, django-celery-beat
-certifi==2020.6.20        # via requests
+certifi==2020.6.20        # via requests, sentry-sdk
 chardet==3.0.4            # via requests
 cssselect==1.1.0          # via pyquery
 decorator==4.4.2          # via ipython, traitlets
@@ -29,14 +32,13 @@ django-db-readonly==0.6.0  # via -r dependencies/pip/requirements.in
 django-extensions==3.0.3  # via -r dependencies/pip/requirements.in
 django-filter==2.3.0      # via -r dependencies/pip/requirements.in
 django-guardian==2.3.0    # via -r dependencies/pip/requirements.in, djangorestframework-guardian
-django-nose==1.4.6        # via -r dependencies/pip/requirements.in
+django-nose==1.4.7        # via -r dependencies/pip/dev.in
 django-oauth-toolkit==1.3.2  # via -r dependencies/pip/requirements.in
 django-pure-pagination==0.3.0  # via -r dependencies/pip/requirements.in
 django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
-django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
@@ -46,7 +48,7 @@ djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
 djangorestframework-xml==2.0.0  # via -r dependencies/pip/requirements.in
 djangorestframework==3.11.0  # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian
-docutils==0.16            # via statistics
+docutils==0.15.2          # via botocore, statistics
 dpath==2.0.1              # via -r dependencies/pip/requirements.in
 ecdsa==0.15               # via tlslite-ng
 elaphe3==0.2.0            # via -r dependencies/pip/requirements.in
@@ -65,6 +67,7 @@ ipython-genutils==0.2.0   # via traitlets
 ipython==7.16.1           # via -r dependencies/pip/dev.in, ipdb
 jdcal==1.4.1              # via openpyxl
 jedi==0.17.2              # via ipython
+jmespath==0.10.0          # via boto3, botocore
 jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
 jsonschema==3.2.0         # via formpack
 kombu==4.6.11             # via celery
@@ -101,15 +104,16 @@ pytest-django==3.9.0      # via -r dependencies/pip/dev.in
 pytest-env==0.6.2         # via -r dependencies/pip/dev.in
 pytest==5.4.3             # via -r dependencies/pip/dev.in, pytest-django, pytest-env
 python-crontab==2.5.1     # via django-celery-beat
-python-dateutil==2.8.1    # via pandas, python-crontab
+python-dateutil==2.8.1    # via botocore, pandas, python-crontab
 python-slugify==4.0.1     # via transifex-client
 pytz==2020.1              # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas
 pyxform==0.15.1           # via -r dependencies/pip/requirements.in, formpack
-raven==6.10.0             # via -r dependencies/pip/requirements.in
 redis==3.5.3              # via -r dependencies/pip/requirements.in, celery, django-redis-sessions
 requests==2.24.0          # via django-oauth-toolkit, httmock, transifex-client
+s3transfer==0.3.3         # via boto3
 https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter  # via -r dependencies/pip/requirements.in
 sentinels==1.0.0          # via mongomock
+sentry-sdk==0.16.5        # via -r dependencies/pip/requirements.in
 shell-command==0.1        # via -r dependencies/pip/dev.in
 simplejson==3.17.2        # via -r dependencies/pip/requirements.in
 six==1.15.0               # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, mongomock, packaging, pyrsistent, python-dateutil, ssrf-protect, traitlets, transifex-client, unittest2
@@ -123,7 +127,7 @@ traitlets==4.3.3          # via ipython
 transifex-client==0.13.11  # via -r dependencies/pip/requirements.in
 unicodecsv==0.14.1        # via djangorestframework-csv, pyxform
 unittest2==1.1.0          # via pyxform
-urllib3==1.25.9           # via requests, transifex-client
+urllib3==1.25.9           # via botocore, requests, sentry-sdk, transifex-client
 vine==1.3.0               # via amqp, celery
 wcwidth==0.2.5            # via prompt-toolkit, pytest
 werkzeug==1.0.1           # via -r dependencies/pip/dev.in

dependencies/pip/prod.txt

@@ -5,6 +5,7 @@
 #    pip-compile dependencies/pip/prod.in
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
@@ -14,8 +15,10 @@ attrs==19.3.0             # via jsonschema
 backports.csv==1.0.7      # via formpack
 begins==0.9               # via formpack
 billiard==3.6.3.0         # via celery
+boto3==1.14.44            # via -r dependencies/pip/requirements.in
+botocore==1.17.44         # via boto3, s3transfer
 celery[redis]==4.4.6      # via -r dependencies/pip/requirements.in, django-celery-beat
-certifi==2020.6.20        # via requests
+certifi==2020.6.20        # via requests, sentry-sdk
 chardet==3.0.4            # via requests
 cssselect==1.1.0          # via pyquery
 defusedxml==0.6.0         # via djangorestframework-xml
@@ -33,7 +36,6 @@ django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
-django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
@@ -43,7 +45,7 @@ djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
 djangorestframework-xml==2.0.0  # via -r dependencies/pip/requirements.in
 djangorestframework==3.11.0  # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian
-docutils==0.16            # via statistics
+docutils==0.15.2          # via botocore, statistics
 dpath==2.0.1              # via -r dependencies/pip/requirements.in
 ecdsa==0.15               # via tlslite-ng
 elaphe3==0.2.0            # via -r dependencies/pip/requirements.in
@@ -57,6 +59,7 @@ gitdb==4.0.5              # via gitpython
 gitpython==3.1.7          # via transifex-client
 idna==2.10                # via requests
 jdcal==1.4.1              # via openpyxl
+jmespath==0.10.0          # via boto3, botocore
 jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
 jsonschema==3.2.0         # via formpack
 kombu==4.6.11             # via celery
@@ -76,14 +79,15 @@ pymongo==3.10.1           # via -r dependencies/pip/requirements.in
 pyquery==1.4.1            # via formpack
 pyrsistent==0.16.0        # via jsonschema
 python-crontab==2.5.1     # via django-celery-beat
-python-dateutil==2.8.1    # via pandas, python-crontab
+python-dateutil==2.8.1    # via botocore, pandas, python-crontab
 python-slugify==4.0.1     # via transifex-client
 pytz==2020.1              # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas
 pyxform==0.15.1           # via -r dependencies/pip/requirements.in, formpack
-raven==6.10.0             # via -r dependencies/pip/requirements.in
 redis==3.5.3              # via -r dependencies/pip/requirements.in, celery, django-redis-sessions
 requests==2.24.0          # via django-oauth-toolkit, transifex-client
+s3transfer==0.3.3         # via boto3
 https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter  # via -r dependencies/pip/requirements.in
+sentry-sdk==0.16.5        # via -r dependencies/pip/requirements.in
 simplejson==3.17.2        # via -r dependencies/pip/requirements.in
 six==1.15.0               # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2
 smmap==3.0.4              # via gitdb
@@ -95,7 +99,7 @@ traceback2==1.4.0         # via unittest2
 transifex-client==0.13.11  # via -r dependencies/pip/requirements.in
 unicodecsv==0.14.1        # via djangorestframework-csv, pyxform
 unittest2==1.1.0          # via pyxform
-urllib3==1.25.9           # via requests, transifex-client
+urllib3==1.25.9           # via botocore, requests, sentry-sdk, transifex-client
 uwsgi==2.0.19.1           # via -r dependencies/pip/prod.in
 vine==1.3.0               # via amqp, celery
 xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform

dependencies/pip/requirements.in

@@ -45,7 +45,6 @@ openpyxl
 celery>=4.0,<5.0
 celery[redis]
 amqp
-raven
 # new export code relies on
 pandas>=0.12.0
 elaphe3
@@ -94,3 +93,6 @@ djangorestframework-guardian
 
 # Django Storage AWS
 boto3
+
+# Sentry
+sentry-sdk

dependencies/pip/requirements.txt

@@ -5,6 +5,7 @@
 #    pip-compile dependencies/pip/requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
@@ -14,8 +15,10 @@ attrs==19.3.0             # via jsonschema
 backports.csv==1.0.7      # via formpack
 begins==0.9               # via formpack
 billiard==3.6.3.0         # via celery
+boto3==1.14.44            # via -r dependencies/pip/requirements.in
+botocore==1.17.44         # via boto3, s3transfer
 celery[redis]==4.4.6      # via -r dependencies/pip/requirements.in, django-celery-beat
-certifi==2020.6.20        # via requests
+certifi==2020.6.20        # via requests, sentry-sdk
 chardet==3.0.4            # via requests
 cssselect==1.1.0          # via pyquery
 defusedxml==0.6.0         # via djangorestframework-xml
@@ -33,7 +36,6 @@ django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
-django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
@@ -43,7 +45,7 @@ djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
 djangorestframework-xml==2.0.0  # via -r dependencies/pip/requirements.in
 djangorestframework==3.11.0  # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian
-docutils==0.16            # via statistics
+docutils==0.15.2          # via botocore, statistics
 dpath==2.0.1              # via -r dependencies/pip/requirements.in
 ecdsa==0.15               # via tlslite-ng
 elaphe3==0.2.0            # via -r dependencies/pip/requirements.in
@@ -57,6 +59,7 @@ gitdb==4.0.5              # via gitpython
 gitpython==3.1.7          # via transifex-client
 idna==2.10                # via requests
 jdcal==1.4.1              # via openpyxl
+jmespath==0.10.0          # via boto3, botocore
 jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
 jsonschema==3.2.0         # via formpack
 kombu==4.6.11             # via celery
@@ -76,14 +79,15 @@ pymongo==3.10.1           # via -r dependencies/pip/requirements.in
 pyquery==1.4.1            # via formpack
 pyrsistent==0.16.0        # via jsonschema
 python-crontab==2.5.1     # via django-celery-beat
-python-dateutil==2.8.1    # via pandas, python-crontab
+python-dateutil==2.8.1    # via botocore, pandas, python-crontab
 python-slugify==4.0.1     # via transifex-client
 pytz==2020.1              # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas
 pyxform==0.15.1           # via -r dependencies/pip/requirements.in, formpack
-raven==6.10.0             # via -r dependencies/pip/requirements.in
 redis==3.5.3              # via -r dependencies/pip/requirements.in, celery, django-redis-sessions
 requests==2.24.0          # via django-oauth-toolkit, transifex-client
+s3transfer==0.3.3         # via boto3
 https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter  # via -r dependencies/pip/requirements.in
+sentry-sdk==0.16.5        # via -r dependencies/pip/requirements.in
 simplejson==3.17.2        # via -r dependencies/pip/requirements.in
 six==1.15.0               # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2
 smmap==3.0.4              # via gitdb
@@ -95,7 +99,7 @@ traceback2==1.4.0         # via unittest2
 transifex-client==0.13.11  # via -r dependencies/pip/requirements.in
 unicodecsv==0.14.1        # via djangorestframework-csv, pyxform
 unittest2==1.1.0          # via pyxform
-urllib3==1.25.9           # via requests, transifex-client
+urllib3==1.25.9           # via botocore, requests, sentry-sdk, transifex-client
 vine==1.3.0               # via amqp, celery
 xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform
 xlsxwriter==1.2.9         # via formpack

docker/restore_permissions.bash

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+source /etc/profile
+
+echo "Restoring permissions for \`backup\` and \`log\` folders..."
+chown -R "${UWSGI_USER}":"${UWSGI_USER}" "${BACKUPS_DIR}"
+chown -R "${UWSGI_USER}":"${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}"

docker/run_uwsgi.bash

@@ -44,11 +44,6 @@ else
         sv start celery_beat
     fi
 
-    if [[ -n "$RAVEN_DSN" ]]; then
-        echo "Sentry detected. Installing \`raven\` pip dependency..."
-        gosu "${UWSGI_USER}" pip install raven
-    fi
-
     echo "Running KoBoCAT container with \`runserver_plus\` debugging application server."
     UWSGI_COMMAND="gosu $UWSGI_USER python manage.py runserver_plus 0:8001"
 fi

kobocat-template/templates/base.html

@@ -33,7 +33,7 @@
         <link rel="apple-touch-icon" href="{% static "images/apple-touch-icon.png" %}">
 
         <link href="{% static "css/phaseout/bootstrap.min.css" %}" rel="stylesheet">
-        <link href="{% static "css/screen.css?v=2809679b01" %}" rel="stylesheet">
+        <link href="{% static "css/screen.css" %}?v=20200829121939" rel="stylesheet">
         <link href="{% static "css/ona-screen-overrides.css" %}" rel="stylesheet">
         <link href="{% static "css/font-awesome.min.css" %}" rel="stylesheet">
         <!--[if lte IE 8]>

kobocat-template/templates/show.html

@@ -31,10 +31,14 @@ <h2 class="dashboard__group-label">
             </a>
 
             <div class="dashboard__download__drop hidden">
-                <a href="{% url "export_list" content_user.username xform.id_string 'xls' %}" class="download__drop-button">XLS</a>
-                <a href="{% url "export_list" content_user.username xform.id_string 'csv' %}" class="download__drop-button">CSV</a>
-                <a href="{% url "export_list" content_user.username xform.id_string 'zip' %}" class="download__drop-button">ZIP</a>
-                <a href="{% url "export_list" content_user.username xform.id_string 'kml'%}"  class="download__drop-button">KML</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'xls' %}"
+                   class="download__drop-button">XLS</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'csv' %}"
+                   class="download__drop-button">CSV</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'zip' %}"
+                   class="download__drop-button">ZIP</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'kml' %}"
+                   class="download__drop-button">KML</a>
             </div>
         </div>
 

manage.py

@@ -4,14 +4,11 @@
 import os
 import sys
 
-south_logger = logging.getLogger('south')
-south_logger.setLevel(logging.INFO)
-
 if __name__ == "__main__":
     # altered for new settings layout
     if not any([arg.startswith('--settings=') for arg in sys.argv]):
         os.environ.setdefault("DJANGO_SETTINGS_MODULE",
-                              "onadata.settings.common")
+                              "onadata.settings.base")
         print('Your environment is:"{}"'.format(
             os.environ['DJANGO_SETTINGS_MODULE']))