[Python3 and Django2.2 upgrade] part 6 - Upgrade to Django 2.2

Dockerfile

@@ -1,69 +1,112 @@
-FROM kobotoolbox/kobocat_base:latest
+FROM nikolaik/python-nodejs:python3.8-nodejs10
 
-ENV KOBOCAT_SRC_DIR=/srv/src/kobocat \
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+ENV VIRTUAL_ENV=/opt/venv
+
+
+ENV KOBOCAT_LOGS_DIR=/srv/logs \
+    DJANGO_SETTINGS_MODULE=kobo.settings.prod \
+    # The mountpoint of a volume shared with the `nginx` container. Static files will
+    #   be copied there.
+    NGINX_STATIC_DIR=/srv/static \
+    KOBOCAT_SRC_DIR=/srv/src/kobocat \
     BACKUPS_DIR=/srv/backups \
-    KOBOCAT_LOGS_DIR=/srv/logs
-
-# Install post-base-image `apt` additions from `apt_requirements.txt`, if modified.
-COPY ./apt_requirements.txt "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt"
-RUN if ! diff "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt" "${KOBOCAT_TMP_DIR}/base_apt_requirements.txt"; then \
-        apt-get update && \
-        apt-get install -y $(cat "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt") && \
-        apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    ; fi
-
-# Version 8 of pip doesn't really seem to upgrade packages when switching from
-# PyPI to editable Git
-RUN pip install --upgrade 'pip>=10,<11'
-
-# Install post-base-image `pip` additions/upgrades from `requirements/base.pip`, if modified.
-COPY ./requirements/ "${KOBOCAT_TMP_DIR}/current_requirements/"
-# FIXME: Replace this with the much simpler command `pip-sync ${KOBOCAT_TMP_DIR}/current_requirements/base.pip`.
-RUN if ! diff "${KOBOCAT_TMP_DIR}/current_requirements/base.pip" "${KOBOCAT_TMP_DIR}/base_requirements/base.pip"; then \
-        pip install --src "${PIP_EDITABLE_PACKAGES_DIR}/" -r "${KOBOCAT_TMP_DIR}/current_requirements/base.pip" \
-    ; fi
-
-# Install post-base-image `pip` additions/upgrades from `requirements/s3.pip`, if modified.
-RUN if ! diff "${KOBOCAT_TMP_DIR}/current_requirements/s3.pip" "${KOBOCAT_TMP_DIR}/base_requirements/s3.pip"; then \
-        pip install --src "${PIP_EDITABLE_PACKAGES_DIR}/" -r "${KOBOCAT_TMP_DIR}/current_requirements/s3.pip" \
-    ; fi
-
-# Uninstall `pip` packages installed in the base image from `requirements/uninstall.pip`, if present.
-# FIXME: Replace this with the much simpler `pip-sync` command equivalent.
-RUN if [ -e "${KOBOCAT_TMP_DIR}/current_requirements/uninstall.pip" ]; then \
-        pip uninstall --yes -r "${KOBOCAT_TMP_DIR}/current_requirements/uninstall.pip" \
-    ; fi
-
-# Wipe out the base image's `kobocat` dir (**including migration files**) and copy over this directory in its current state.
-RUN rm -rf "${KOBOCAT_SRC_DIR}"
+    TMP_PATH=/srv/tmp \
+    INIT_PATH=/srv/init
+
+# Install Dockerize.
+ENV DOCKERIZE_VERSION v0.6.1
+RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -P /tmp \
+    && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+
+##########################################
+# Create build directories               #
+##########################################
+
+RUN mkdir -p "${NGINX_STATIC_DIR}" && \
+    mkdir -p "${KOBOCAT_SRC_DIR}" && \
+    mkdir -p "${TMP_PATH}" && \
+    mkdir -p "${BACKUPS_DIR}" && \
+    mkdir -p "${INIT_PATH}"
+
+##########################################
+# Install `apt` packages.                #
+##########################################
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+
+RUN apt -qq update && \
+    apt -qq -y install \
+        gdal-bin \
+        libproj-dev \
+        gettext \
+        postgresql-client \
+        locales \
+        runit-init \
+        rsync \
+        vim && \
+    apt clean && \
+        rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+###########################
+# Install locales         #
+###########################
+
+RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen
+RUN locale-gen && dpkg-reconfigure locales -f noninteractive
+
+###########################
+# Copy KoBoCAT directory  #
+###########################
+
 COPY . "${KOBOCAT_SRC_DIR}"
 
+###########################
+# Install `pip` packages. #
+###########################
+
+RUN virtualenv "$VIRTUAL_ENV"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN pip install  --quiet --upgrade pip && \
+    pip install  --quiet pip-tools
+COPY ./dependencies/pip/prod.txt /srv/tmp/pip_dependencies.txt
+RUN pip-sync /srv/tmp/pip_dependencies.txt 1>/dev/null && \
+    rm -rf ~/.cache/pip
+
+##########################################
+# Persist the log and email directories. #
+##########################################
+
+RUN mkdir -p "${KOBOCAT_LOGS_DIR}/" "${KOBOCAT_SRC_DIR}/emails" && \
+    chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \
+    chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}"
+
+#################################################
+# Handle runtime tasks and create main process. #
+#################################################
+
+# Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides.
+RUN echo "export PATH=${PATH}" >> /etc/profile
+RUN echo 'source /etc/profile' >> /root/.bashrc
+
 # Prepare for execution.
-# TODO: Remove the wrong port warning and related files, say, at the start of 2021 (see kobotoolbox/kobo-docker#301)
 RUN mkdir -p /etc/service/uwsgi_wrong_port_warning && \
     cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" /etc/service/uwsgi_wrong_port_warning/run && \
     mkdir -p /etc/service/uwsgi && \
+    # Remove getty* services
+    rm -rf /etc/runit/runsvdir/default/getty-tty* && \
     cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \
     mkdir -p /etc/service/celery && \
     ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \
     mkdir -p /etc/service/celery_beat && \
     ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \
-    cp "${KOBOCAT_SRC_DIR}/docker/init.bash" /etc/my_init.d/10_init_kobocat.bash && \
-    cp "${KOBOCAT_SRC_DIR}/docker/sync_static.sh" /etc/my_init.d/11_sync_static.bash && \
-    mkdir -p "${KOBOCAT_SRC_DIR}/emails/" && \
-    chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \
-    mkdir -p "${BACKUPS_DIR}" && \
-    mkdir -p "${KOBOCAT_LOGS_DIR}" && \
-    chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}"
-
-RUN echo "db:*:*:kobo:kobo" > /root/.pgpass && \
-    chmod 600 /root/.pgpass
-
-# Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides.
-RUN echo 'source /etc/profile' >> /root/.bashrc
-
 
 WORKDIR "${KOBOCAT_SRC_DIR}"
 
 # TODO: Remove port 8000, say, at the start of 2021 (see kobotoolbox/kobo-docker#301 and wrong port warning above)
 EXPOSE 8001 8000
+
+CMD ["/bin/bash", "-c", "exec ${KOBOCAT_SRC_DIR}/docker/init.bash"]

dependencies/pip/dev.in

@@ -0,0 +1,14 @@
+-r requirements.in
+
+ipdb
+ipython
+shell_command
+Werkzeug
+sqlparse
+pytest
+pytest-django
+pytest-env
+mongomock
+mock
+httmock
+django-nose

dependencies/pip/dev.txt

@@ -0,0 +1,136 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile dependencies/pip/dev.in
+#
+-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
+amqp==2.6.0               # via -r dependencies/pip/requirements.in, kombu
+argparse==1.4.0           # via unittest2
+attrs==19.3.0             # via jsonschema, pytest
+backcall==0.2.0           # via ipython
+backports.csv==1.0.7      # via formpack
+begins==0.9               # via formpack
+billiard==3.6.3.0         # via celery
+celery[redis]==4.4.6      # via -r dependencies/pip/requirements.in, django-celery-beat
+certifi==2020.6.20        # via requests
+chardet==3.0.4            # via requests
+cssselect==1.1.0          # via pyquery
+decorator==4.4.2          # via ipython, traitlets
+defusedxml==0.6.0         # via djangorestframework-xml
+dict2xml==1.7.0           # via -r dependencies/pip/requirements.in
+dj-database-url==0.5.0    # via -r dependencies/pip/requirements.in
+django-celery-beat==2.0.0  # via -r dependencies/pip/requirements.in
+django-cors-headers==3.4.0  # via -r dependencies/pip/requirements.in
+django-db-readonly==0.6.0  # via -r dependencies/pip/requirements.in
+django-extensions==3.0.3  # via -r dependencies/pip/requirements.in
+django-filter==2.3.0      # via -r dependencies/pip/requirements.in
+django-guardian==2.3.0    # via -r dependencies/pip/requirements.in, djangorestframework-guardian
+django-nose==1.4.6        # via -r dependencies/pip/requirements.in
+django-oauth-toolkit==1.3.2  # via -r dependencies/pip/requirements.in
+django-pure-pagination==0.3.0  # via -r dependencies/pip/requirements.in
+django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
+django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
+django-render-block==0.7  # via django-templated-email
+django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
+django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
+django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
+django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
+django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
+djangorestframework-csv==2.1.0  # via -r dependencies/pip/requirements.in
+djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
+djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
+djangorestframework-xml==2.0.0  # via -r dependencies/pip/requirements.in
+djangorestframework==3.11.0  # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian
+docutils==0.16            # via statistics
+dpath==2.0.1              # via -r dependencies/pip/requirements.in
+ecdsa==0.15               # via tlslite-ng
+elaphe3==0.2.0            # via -r dependencies/pip/requirements.in
+et-xmlfile==1.0.1         # via openpyxl
+formencode==1.3.1         # via pyxform
+git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack  # via -r dependencies/pip/requirements.in
+future==0.18.2            # via celery
+gdata-python3==3.0.1      # via -r dependencies/pip/requirements.in
+geojson-rewind==0.2.0     # via formpack
+gitdb==4.0.5              # via gitpython
+gitpython==3.1.7          # via transifex-client
+httmock==1.3.0            # via -r dependencies/pip/dev.in
+idna==2.10                # via requests
+importlib-metadata==1.7.0  # via jsonschema, kombu, markdown, path, pluggy, pytest
+ipdb==0.13.3              # via -r dependencies/pip/dev.in
+ipython-genutils==0.2.0   # via traitlets
+ipython==7.16.1           # via -r dependencies/pip/dev.in, ipdb
+jdcal==1.4.1              # via openpyxl
+jedi==0.17.2              # via ipython
+jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
+jsonschema==3.2.0         # via formpack
+kombu==4.6.11             # via celery
+linecache2==1.0.0         # via traceback2
+lxml==4.5.2               # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery
+markdown==3.2.2           # via -r dependencies/pip/requirements.in
+mock==4.0.2               # via -r dependencies/pip/dev.in
+modilabs-python-utils==0.1.5  # via -r dependencies/pip/requirements.in
+mongomock==3.19.0         # via -r dependencies/pip/dev.in
+more-itertools==8.4.0     # via pytest
+nose==1.3.7               # via django-nose
+numpy==1.19.0             # via pandas
+oauthlib==3.1.0           # via django-oauth-toolkit
+openpyxl==3.0.4           # via -r dependencies/pip/requirements.in
+packaging==20.4           # via pytest
+pandas==1.0.5             # via -r dependencies/pip/requirements.in
+parso==0.7.0              # via jedi
+path.py==12.4.0           # via -r dependencies/pip/requirements.in, formpack
+path==13.1.0              # via path.py
+pexpect==4.8.0            # via ipython
+pickleshare==0.7.5        # via ipython
+pillow==7.2.0             # via -r dependencies/pip/requirements.in, elaphe3
+pluggy==0.13.1            # via pytest
+prompt-toolkit==3.0.5     # via ipython
+psycopg2-binary==2.8.5    # via -r dependencies/pip/requirements.in
+ptyprocess==0.6.0         # via pexpect
+py==1.9.0                 # via pytest
+pygments==2.6.1           # via ipython
+pymongo==3.10.1           # via -r dependencies/pip/requirements.in
+pyparsing==2.4.7          # via packaging
+pyquery==1.4.1            # via formpack
+pyrsistent==0.16.0        # via jsonschema
+pytest-django==3.9.0      # via -r dependencies/pip/dev.in
+pytest-env==0.6.2         # via -r dependencies/pip/dev.in
+pytest==5.4.3             # via -r dependencies/pip/dev.in, pytest-django, pytest-env
+python-crontab==2.5.1     # via django-celery-beat
+python-dateutil==2.8.1    # via pandas, python-crontab
+python-slugify==4.0.1     # via transifex-client
+pytz==2020.1              # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas
+pyxform==0.15.1           # via -r dependencies/pip/requirements.in, formpack
+raven==6.10.0             # via -r dependencies/pip/requirements.in
+redis==3.5.3              # via -r dependencies/pip/requirements.in, celery, django-redis-sessions
+requests==2.24.0          # via django-oauth-toolkit, httmock, transifex-client
+https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter  # via -r dependencies/pip/requirements.in
+sentinels==1.0.0          # via mongomock
+shell-command==0.1        # via -r dependencies/pip/dev.in
+simplejson==3.17.2        # via -r dependencies/pip/requirements.in
+six==1.15.0               # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, mongomock, packaging, pyrsistent, python-dateutil, ssrf-protect, traitlets, transifex-client, unittest2
+smmap==3.0.4              # via gitdb
+sqlparse==0.3.1           # via -r dependencies/pip/dev.in, django
+statistics==1.0.3.5       # via formpack
+text-unidecode==1.3       # via python-slugify
+tlslite-ng==0.7.5         # via gdata-python3
+traceback2==1.4.0         # via unittest2
+traitlets==4.3.3          # via ipython
+transifex-client==0.13.11  # via -r dependencies/pip/requirements.in
+unicodecsv==0.14.1        # via djangorestframework-csv, pyxform
+unittest2==1.1.0          # via pyxform
+urllib3==1.25.9           # via requests, transifex-client
+vine==1.3.0               # via amqp, celery
+wcwidth==0.2.5            # via prompt-toolkit, pytest
+werkzeug==1.0.1           # via -r dependencies/pip/dev.in
+xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform
+xlsxwriter==1.2.9         # via formpack
+xlwt==1.3.0               # via -r dependencies/pip/requirements.in
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

dependencies/pip/prod.in

@@ -0,0 +1,3 @@
+-r requirements.in
+
+uWSGI