diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 98cd84d09..99dc4b731 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ['3.8'] + python-version: ['3.10'] services: postgres: image: postgis/postgis:9.5-2.5 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e2371e3be..781b5f4ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ build: test: stage: build_test - image: python:3.8 + image: python:3.10 variables: DJANGO_SECRET_KEY: abcdef TEST_DATABASE_URL: postgis://kobo:kobo@postgres:5432/kobocat_test diff --git a/Dockerfile b/Dockerfile index 04b43a534..1566339b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 as build-python +FROM python:3.10 as build-python ENV VIRTUAL_ENV=/opt/venv \ KOBOCAT_SRC_DIR=/srv/src/kobocat \ @@ -12,7 +12,7 @@ RUN pip install --quiet --upgrade pip && \ COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null -FROM python:3.8-slim +FROM python:3.10-slim # Declare environment variables ENV DEBIAN_FRONTEND=noninteractive @@ -57,6 +57,7 @@ RUN apt-get -qq update && \ git \ gosu \ less \ + libpq5 \ libproj-dev \ libsqlite3-mod-spatialite \ locales \ diff --git a/apt_requirements.txt b/apt_requirements.txt deleted file mode 100644 index 46b5ad2d6..000000000 --- a/apt_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ghostscript -gettext diff --git a/dependencies/pip/dev.in b/dependencies/pip/dev.in index 4259d3b95..51a2959c8 100644 --- a/dependencies/pip/dev.in +++ b/dependencies/pip/dev.in @@ -10,4 +10,3 @@ pytest-env mongomock mock httmock -django-nose diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index f202c0098..c4bdecc7d 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -1,58 +1,84 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile dependencies/pip/dev.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest +-e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in -e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect +-e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 +amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu -attrs==19.3.0 +asttokens==2.0.5 + # via stack-data +async-timeout==4.0.2 + # via redis +attrs==21.4.0 # via pytest backcall==0.2.0 # via ipython -billiard==3.6.3.0 +billiard==3.6.4.0 # via celery -boto3==1.14.44 +boto3==1.22.0 # via -r dependencies/pip/requirements.in -botocore==1.17.44 +botocore==1.25.0 # via # boto3 # s3transfer -celery[redis]==4.4.6 +celery[redis]==5.2.6 # via # -r dependencies/pip/requirements.in # django-celery-beat -certifi==2020.6.20 +certifi==2021.10.8 # via # requests # sentry-sdk -chardet==3.0.4 +cffi==1.15.0 + # via cryptography +charset-normalizer==2.0.12 # via requests -decorator==4.4.2 +click==8.1.2 # via + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery +cryptography==36.0.2 + # via jwcrypto +decorator==5.1.1 + # via + # ipdb # ipython - # traitlets defusedxml==0.7.1 # via # djangorestframework-xml # pyxform -dict2xml==1.7.0 +deprecated==1.2.13 + # via + # jwcrypto + # redis +dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.14 +django==2.2.28 # via # -r dependencies/pip/requirements.in # django-celery-beat # django-cors-headers + # django-db-readonly + # django-extensions # django-filter # django-guardian # django-oauth-toolkit @@ -64,48 +90,46 @@ django==2.2.14 # djangorestframework # djangorestframework-guardian # jsonfield -django-celery-beat==2.0.0 +django-celery-beat==2.2.1 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 +django-cors-headers==3.11.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 +django-db-readonly==0.7.0 # via -r dependencies/pip/requirements.in django-environ==0.8.1 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 +django-extensions==3.1.5 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 +django-filter==21.1 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 +django-guardian==2.4.0 # via # -r dependencies/pip/requirements.in # djangorestframework-guardian -django-nose==1.4.7 - # via -r dependencies/pip/dev.in -django-oauth-toolkit==1.3.2 +django-oauth-toolkit==2.0.0 # via -r dependencies/pip/requirements.in django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 +django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in -django-render-block==0.7 +django-render-block==0.9.1 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 +django-taggit==2.1.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 +django-timezone-field==4.2.3 # via # -r dependencies/pip/requirements.in # django-celery-beat -djangorestframework==3.11.0 +djangorestframework==3.13.1 # via # -r dependencies/pip/requirements.in # djangorestframework-csv # djangorestframework-guardian -djangorestframework-csv==2.1.0 +djangorestframework-csv==2.1.1 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in @@ -113,193 +137,191 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -docutils==0.15.2 - # via botocore -dpath==2.0.1 +dpath==2.0.6 # via -r dependencies/pip/requirements.in -ecdsa==0.15 +ecdsa==0.17.0 # via tlslite-ng elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 +et-xmlfile==1.1.0 # via openpyxl -future==0.18.2 - # via celery +executing==0.8.3 + # via stack-data gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -gitdb==4.0.5 - # via gitpython -gitpython==3.1.7 - # via transifex-client -httmock==1.3.0 +httmock==1.4.0 # via -r dependencies/pip/dev.in -idna==2.10 +idna==3.3 # via requests -ipdb==0.13.3 +iniconfig==1.1.1 + # via pytest +ipdb==0.13.9 # via -r dependencies/pip/dev.in -ipython==7.16.1 +ipython==8.2.0 # via # -r dependencies/pip/dev.in # ipdb -ipython-genutils==0.2.0 - # via traitlets -jedi==0.17.2 +jedi==0.18.1 # via ipython -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -kombu==4.6.11 +jwcrypto==1.0 + # via django-oauth-toolkit +kombu==5.2.4 # via celery -lxml==4.5.2 +lxml==4.8.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.2.2 +markdown==3.3.6 # via -r dependencies/pip/requirements.in -mock==4.0.2 +matplotlib-inline==0.1.3 + # via ipython +mock==4.0.3 # via -r dependencies/pip/dev.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -mongomock==3.19.0 +mongomock==4.0.0 # via -r dependencies/pip/dev.in -more-itertools==8.4.0 - # via pytest -nose==1.3.7 - # via django-nose -numpy==1.19.0 +numpy==1.22.3 # via pandas -oauthlib==3.1.0 +oauthlib==3.2.0 # via django-oauth-toolkit openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -packaging==20.4 - # via pytest -pandas==1.0.5 +packaging==21.3 + # via + # mongomock + # pytest + # redis +pandas==1.4.2 # via -r dependencies/pip/requirements.in -parso==0.7.0 +parso==0.8.3 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==7.2.0 +pillow==9.1.0 # via # -r dependencies/pip/requirements.in # elaphe3 -pluggy==0.13.1 +pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.5 - # via ipython -psycopg2-binary==2.8.5 +prompt-toolkit==3.0.29 + # via + # click-repl + # ipython +psycopg2==2.8.6 # via -r dependencies/pip/requirements.in -ptyprocess==0.6.0 +ptyprocess==0.7.0 # via pexpect -py==1.9.0 +pure-eval==0.2.2 + # via stack-data +py==1.11.0 # via pytest -pygments==2.6.1 +pycparser==2.21 + # via cffi +pygments==2.12.0 # via ipython -pymongo==3.10.1 +pymongo==3.12.3 # via -r dependencies/pip/requirements.in -pyparsing==2.4.7 +pyparsing==3.0.8 # via packaging -pytest==5.4.3 +pytest==7.1.2 # via # -r dependencies/pip/dev.in # pytest-django # pytest-env -pytest-django==3.9.0 +pytest-django==4.5.2 # via -r dependencies/pip/dev.in pytest-env==0.6.2 # via -r dependencies/pip/dev.in -python-crontab==2.5.1 +python-crontab==2.6.0 # via django-celery-beat -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # botocore # pandas # python-crontab -python-slugify==4.0.1 - # via transifex-client -pytz==2020.1 +pytz==2022.1 # via - # -r dependencies/pip/requirements.in # celery # django # django-timezone-field + # djangorestframework # pandas pyxform==1.9.0 # via -r dependencies/pip/requirements.in -redis==3.5.3 +redis==4.2.2 # via # -r dependencies/pip/requirements.in # celery # django-redis-sessions -requests==2.24.0 +requests==2.27.1 # via # django-oauth-toolkit # httmock - # transifex-client -s3transfer==0.3.3 +s3transfer==0.5.2 # via boto3 savreaderwriter @ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip # via -r dependencies/pip/requirements.in sentinels==1.0.0 # via mongomock -sentry-sdk==0.16.5 +sentry-sdk==1.5.10 # via -r dependencies/pip/requirements.in shell-command==0.1 # via -r dependencies/pip/dev.in -simplejson==3.17.2 +simplejson==3.17.6 # via -r dependencies/pip/requirements.in -six==1.15.0 +six==1.16.0 # via - # django-extensions - # django-templated-email + # asttokens + # click-repl # djangorestframework-csv # ecdsa # mongomock - # packaging # python-dateutil - # ssrf-protect - # traitlets - # transifex-client -smmap==3.0.4 - # via gitdb -sqlparse==0.3.1 +sqlparse==0.4.2 # via # -r dependencies/pip/dev.in # django -text-unidecode==1.3 - # via python-slugify -tlslite-ng==0.7.5 - # via gdata-python3 -traitlets==4.3.3 +stack-data==0.2.0 # via ipython -transifex-client==0.14.3 - # via -r dependencies/pip/requirements.in +tlslite-ng==0.7.6 + # via gdata-python3 +toml==0.10.2 + # via ipdb +tomli==2.0.1 + # via pytest +traitlets==5.1.1 + # via + # ipython + # matplotlib-inline unicodecsv==0.14.1 # via djangorestframework-csv -urllib3==1.25.9 +urllib3==1.26.9 # via # botocore # requests # sentry-sdk - # transifex-client -vine==1.3.0 +vine==5.0.0 # via # amqp # celery + # kombu wcwidth==0.2.5 - # via - # prompt-toolkit - # pytest -werkzeug==1.0.1 + # via prompt-toolkit +werkzeug==2.0.3 # via -r dependencies/pip/requirements.in +wrapt==1.14.0 + # via deprecated xlrd==2.0.1 # via pyxform xlwt==1.3.0 diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index f8cd1b466..98318ad2b 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -1,50 +1,74 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile dependencies/pip/prod.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest +-e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in -e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect +-e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 +amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu -billiard==3.6.3.0 +async-timeout==4.0.2 + # via redis +billiard==3.6.4.0 # via celery -boto3==1.14.44 +boto3==1.22.0 # via -r dependencies/pip/requirements.in -botocore==1.17.44 +botocore==1.25.0 # via # boto3 # s3transfer -celery[redis]==4.4.6 +celery[redis]==5.2.6 # via # -r dependencies/pip/requirements.in # django-celery-beat -certifi==2020.6.20 +certifi==2021.10.8 # via # requests # sentry-sdk -chardet==3.0.4 +cffi==1.15.0 + # via cryptography +charset-normalizer==2.0.12 # via requests +click==8.1.2 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery +cryptography==36.0.2 + # via jwcrypto defusedxml==0.7.1 # via # djangorestframework-xml # pyxform -dict2xml==1.7.0 +deprecated==1.2.13 + # via + # jwcrypto + # redis +dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.14 +django==2.2.28 # via # -r dependencies/pip/requirements.in # django-celery-beat # django-cors-headers + # django-db-readonly + # django-extensions # django-filter # django-guardian # django-oauth-toolkit @@ -56,46 +80,46 @@ django==2.2.14 # djangorestframework # djangorestframework-guardian # jsonfield -django-celery-beat==2.0.0 +django-celery-beat==2.2.1 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 +django-cors-headers==3.11.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 +django-db-readonly==0.7.0 # via -r dependencies/pip/requirements.in django-environ==0.8.1 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 +django-extensions==3.1.5 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 +django-filter==21.1 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 +django-guardian==2.4.0 # via # -r dependencies/pip/requirements.in # djangorestframework-guardian -django-oauth-toolkit==1.3.2 +django-oauth-toolkit==2.0.0 # via -r dependencies/pip/requirements.in django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 +django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in -django-render-block==0.7 +django-render-block==0.9.1 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 +django-taggit==2.1.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 +django-timezone-field==4.2.3 # via # -r dependencies/pip/requirements.in # django-celery-beat -djangorestframework==3.11.0 +djangorestframework==3.13.1 # via # -r dependencies/pip/requirements.in # djangorestframework-csv # djangorestframework-guardian -djangorestframework-csv==2.1.0 +djangorestframework-csv==2.1.1 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in @@ -103,130 +127,123 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -docutils==0.15.2 - # via botocore -dpath==2.0.1 +dpath==2.0.6 # via -r dependencies/pip/requirements.in -ecdsa==0.15 +ecdsa==0.17.0 # via tlslite-ng elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 +et-xmlfile==1.1.0 # via openpyxl -future==0.18.2 - # via celery gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -gitdb==4.0.5 - # via gitpython -gitpython==3.1.7 - # via transifex-client -idna==2.10 +idna==3.3 # via requests -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -kombu==4.6.11 +jwcrypto==1.0 + # via django-oauth-toolkit +kombu==5.2.4 # via celery -lxml==4.5.2 +lxml==4.8.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.2.2 +markdown==3.3.6 # via -r dependencies/pip/requirements.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -numpy==1.19.0 +numpy==1.22.3 # via pandas -oauthlib==3.1.0 +oauthlib==3.2.0 # via django-oauth-toolkit openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -pandas==1.0.5 +packaging==21.3 + # via redis +pandas==1.4.2 # via -r dependencies/pip/requirements.in -pillow==7.2.0 +pillow==9.1.0 # via # -r dependencies/pip/requirements.in # elaphe3 -psycopg2-binary==2.8.5 +prompt-toolkit==3.0.29 + # via click-repl +psycopg2==2.8.6 # via -r dependencies/pip/requirements.in -pymongo==3.10.1 +pycparser==2.21 + # via cffi +pymongo==3.12.3 # via -r dependencies/pip/requirements.in -python-crontab==2.5.1 +pyparsing==3.0.8 + # via packaging +python-crontab==2.6.0 # via django-celery-beat -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # botocore # pandas # python-crontab -python-slugify==4.0.1 - # via transifex-client -pytz==2020.1 +pytz==2022.1 # via - # -r dependencies/pip/requirements.in # celery # django # django-timezone-field + # djangorestframework # pandas pyxform==1.9.0 # via -r dependencies/pip/requirements.in -redis==3.5.3 +redis==4.2.2 # via # -r dependencies/pip/requirements.in # celery # django-redis-sessions -requests==2.24.0 - # via - # django-oauth-toolkit - # transifex-client -s3transfer==0.3.3 +requests==2.27.1 + # via django-oauth-toolkit +s3transfer==0.5.2 # via boto3 savreaderwriter @ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip # via -r dependencies/pip/requirements.in -sentry-sdk==0.16.5 +sentry-sdk==1.5.10 # via -r dependencies/pip/requirements.in -simplejson==3.17.2 +simplejson==3.17.6 # via -r dependencies/pip/requirements.in -six==1.15.0 +six==1.16.0 # via - # django-extensions - # django-templated-email + # click-repl # djangorestframework-csv # ecdsa # python-dateutil - # ssrf-protect - # transifex-client -smmap==3.0.4 - # via gitdb -sqlparse==0.3.1 +sqlparse==0.4.2 # via django -text-unidecode==1.3 - # via python-slugify -tlslite-ng==0.7.5 +tlslite-ng==0.7.6 # via gdata-python3 -transifex-client==0.14.3 - # via -r dependencies/pip/requirements.in unicodecsv==0.14.1 # via djangorestframework-csv -urllib3==1.25.9 +urllib3==1.26.9 # via # botocore # requests # sentry-sdk - # transifex-client -uwsgi==2.0.19.1 +uwsgi==2.0.20 # via -r dependencies/pip/prod.in -vine==1.3.0 +vine==5.0.0 # via # amqp # celery -werkzeug==2.0.1 + # kombu +wcwidth==0.2.5 + # via prompt-toolkit +werkzeug==2.0.3 # via -r dependencies/pip/requirements.in +wrapt==1.14.0 + # via deprecated xlrd==2.0.1 # via pyxform xlwt==1.3.0 diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index b0b583edf..03bb72e0b 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -4,13 +4,13 @@ # More up-to-date version of django-digest than PyPI seems to have. # Also, python-digest is an unlisted dependency thereof. -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest +-e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # spss https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # ssrf --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect +-e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # jnm's django storages # Necessary to use this fork until the resolution of @@ -18,22 +18,21 @@ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip# -e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # Regular PyPI packages -pytz -Django>=2,<3 +Django>=2.2,<2.3 django-environ django-templated-email dpath gdata-python3 modilabs-python-utils Pillow -psycopg2-binary -pymongo +psycopg2>=2.8,<2.9 +pymongo==3.12.3 lxml pyxform==1.9.0 django-reversion<3.0.2 xlwt openpyxl -celery>=4.0,<5.0 +celery celery[redis] amqp # new export code relies on @@ -63,8 +62,6 @@ django-taggit jsonfield django-db-readonly -transifex-client - # Shell Plus django-extensions @@ -89,4 +86,4 @@ boto3 sentry-sdk # mimetype detection -Werkzeug +Werkzeug<=2.0.3 diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 032011b83..4d61d7443 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -1,50 +1,74 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile dependencies/pip/requirements.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest +-e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in -e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect +-e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 +amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu -billiard==3.6.3.0 +async-timeout==4.0.2 + # via redis +billiard==3.6.4.0 # via celery -boto3==1.14.44 +boto3==1.22.0 # via -r dependencies/pip/requirements.in -botocore==1.17.44 +botocore==1.25.0 # via # boto3 # s3transfer -celery[redis]==4.4.6 +celery[redis]==5.2.6 # via # -r dependencies/pip/requirements.in # django-celery-beat -certifi==2020.6.20 +certifi==2021.10.8 # via # requests # sentry-sdk -chardet==3.0.4 +cffi==1.15.0 + # via cryptography +charset-normalizer==2.0.12 # via requests +click==8.1.2 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery +cryptography==36.0.2 + # via jwcrypto defusedxml==0.7.1 # via # djangorestframework-xml # pyxform -dict2xml==1.7.0 +deprecated==1.2.13 + # via + # jwcrypto + # redis +dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.14 +django==2.2.28 # via # -r dependencies/pip/requirements.in # django-celery-beat # django-cors-headers + # django-db-readonly + # django-extensions # django-filter # django-guardian # django-oauth-toolkit @@ -56,46 +80,46 @@ django==2.2.14 # djangorestframework # djangorestframework-guardian # jsonfield -django-celery-beat==2.0.0 +django-celery-beat==2.2.1 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 +django-cors-headers==3.11.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 +django-db-readonly==0.7.0 # via -r dependencies/pip/requirements.in django-environ==0.8.1 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 +django-extensions==3.1.5 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 +django-filter==21.1 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 +django-guardian==2.4.0 # via # -r dependencies/pip/requirements.in # djangorestframework-guardian -django-oauth-toolkit==1.3.2 +django-oauth-toolkit==2.0.0 # via -r dependencies/pip/requirements.in django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 +django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in -django-render-block==0.7 +django-render-block==0.9.1 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 +django-taggit==2.1.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 +django-timezone-field==4.2.3 # via # -r dependencies/pip/requirements.in # django-celery-beat -djangorestframework==3.11.0 +djangorestframework==3.13.1 # via # -r dependencies/pip/requirements.in # djangorestframework-csv # djangorestframework-guardian -djangorestframework-csv==2.1.0 +djangorestframework-csv==2.1.1 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in @@ -103,128 +127,121 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -docutils==0.15.2 - # via botocore -dpath==2.0.1 +dpath==2.0.6 # via -r dependencies/pip/requirements.in -ecdsa==0.15 +ecdsa==0.17.0 # via tlslite-ng elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 +et-xmlfile==1.1.0 # via openpyxl -future==0.18.2 - # via celery gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -gitdb==4.0.5 - # via gitpython -gitpython==3.1.7 - # via transifex-client -idna==2.10 +idna==3.3 # via requests -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -kombu==4.6.11 +jwcrypto==1.0 + # via django-oauth-toolkit +kombu==5.2.4 # via celery -lxml==4.5.2 +lxml==4.8.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.2.2 +markdown==3.3.6 # via -r dependencies/pip/requirements.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -numpy==1.19.0 +numpy==1.22.3 # via pandas -oauthlib==3.1.0 +oauthlib==3.2.0 # via django-oauth-toolkit openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -pandas==1.0.5 +packaging==21.3 + # via redis +pandas==1.4.2 # via -r dependencies/pip/requirements.in -pillow==7.2.0 +pillow==9.1.0 # via # -r dependencies/pip/requirements.in # elaphe3 -psycopg2-binary==2.8.5 +prompt-toolkit==3.0.29 + # via click-repl +psycopg2==2.8.6 # via -r dependencies/pip/requirements.in -pymongo==3.10.1 +pycparser==2.21 + # via cffi +pymongo==3.12.3 # via -r dependencies/pip/requirements.in -python-crontab==2.5.1 +pyparsing==3.0.8 + # via packaging +python-crontab==2.6.0 # via django-celery-beat -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # botocore # pandas # python-crontab -python-slugify==4.0.1 - # via transifex-client -pytz==2020.1 +pytz==2022.1 # via - # -r dependencies/pip/requirements.in # celery # django # django-timezone-field + # djangorestframework # pandas pyxform==1.9.0 # via -r dependencies/pip/requirements.in -redis==3.5.3 +redis==4.2.2 # via # -r dependencies/pip/requirements.in # celery # django-redis-sessions -requests==2.24.0 - # via - # django-oauth-toolkit - # transifex-client -s3transfer==0.3.3 +requests==2.27.1 + # via django-oauth-toolkit +s3transfer==0.5.2 # via boto3 savreaderwriter @ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip # via -r dependencies/pip/requirements.in -sentry-sdk==0.16.5 +sentry-sdk==1.5.10 # via -r dependencies/pip/requirements.in -simplejson==3.17.2 +simplejson==3.17.6 # via -r dependencies/pip/requirements.in -six==1.15.0 +six==1.16.0 # via - # django-extensions - # django-templated-email + # click-repl # djangorestframework-csv # ecdsa # python-dateutil - # ssrf-protect - # transifex-client -smmap==3.0.4 - # via gitdb -sqlparse==0.3.1 +sqlparse==0.4.2 # via django -text-unidecode==1.3 - # via python-slugify -tlslite-ng==0.7.5 +tlslite-ng==0.7.6 # via gdata-python3 -transifex-client==0.14.3 - # via -r dependencies/pip/requirements.in unicodecsv==0.14.1 # via djangorestframework-csv -urllib3==1.25.9 +urllib3==1.26.9 # via # botocore # requests # sentry-sdk - # transifex-client -vine==1.3.0 +vine==5.0.0 # via # amqp # celery -werkzeug==2.0.1 + # kombu +wcwidth==0.2.5 + # via prompt-toolkit +werkzeug==2.0.3 # via -r dependencies/pip/requirements.in +wrapt==1.14.0 + # via deprecated xlrd==2.0.1 # via pyxform xlwt==1.3.0 diff --git a/docker/run_celery.bash b/docker/run_celery.bash index f5af0deef..f9c57ea9f 100755 --- a/docker/run_celery.bash +++ b/docker/run_celery.bash @@ -4,7 +4,7 @@ source /etc/profile # Run the main Celery worker cd "${KOBOCAT_SRC_DIR}" -exec celery worker -A onadata -Ofair --loglevel=info \ +exec celery -A onadata worker -Ofair --loglevel=info \ --hostname=kobocat_main_worker@%h \ --logfile=${KOBOCAT_LOGS_DIR}/celery.log \ --pidfile=${CELERY_PID_DIR}/celery.pid \ diff --git a/docker/run_celery_beat.bash b/docker/run_celery_beat.bash index faf270444..606684b85 100755 --- a/docker/run_celery_beat.bash +++ b/docker/run_celery_beat.bash @@ -4,7 +4,7 @@ source /etc/profile # Run the main Celery beat worker cd "${KOBOCAT_SRC_DIR}" -exec celery beat -A onadata --loglevel=info \ +exec celery -A onadata beat --loglevel=info \ --logfile=${KOBOCAT_LOGS_DIR}/celery_beat.log \ --pidfile=${CELERY_PID_DIR}/celery_beat.pid \ --scheduler django_celery_beat.schedulers:DatabaseScheduler \ diff --git a/onadata/apps/api/viewsets/xform_list_api.py b/onadata/apps/api/viewsets/xform_list_api.py index 0bb296f35..82586d89a 100644 --- a/onadata/apps/api/viewsets/xform_list_api.py +++ b/onadata/apps/api/viewsets/xform_list_api.py @@ -1,8 +1,7 @@ # coding: utf-8 -import pytz from datetime import datetime +from zoneinfo import ZoneInfo -from django.db import transaction from django.conf import settings from django.http import Http404 from django.shortcuts import get_object_or_404 @@ -50,8 +49,7 @@ def __init__(self, *args, **kwargs): ] def get_openrosa_headers(self): - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') + dt = datetime.now(tz=ZoneInfo('UTC')).strftime('%a, %d %b %Y %H:%M:%S %Z') return { 'Date': dt, diff --git a/onadata/apps/logger/fields.py b/onadata/apps/logger/fields.py index 1d2a223c5..3b9db6acb 100644 --- a/onadata/apps/logger/fields.py +++ b/onadata/apps/logger/fields.py @@ -1,5 +1,5 @@ # coding: utf-8 -import collections +from collections.abc import Callable from django.core.exceptions import FieldError from django.db import models @@ -37,7 +37,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def _get_lazy_default(self): - if isinstance(self.lazy_default, collections.Callable): + if isinstance(self.lazy_default, Callable): return self.lazy_default() else: return self.lazy_default diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 3501538a4..198d71ae6 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -1,7 +1,7 @@ # coding: utf-8 -import pytz from datetime import date from hashlib import sha256 +from zoneinfo import ZoneInfo import reversion from django.contrib.auth.models import User diff --git a/onadata/apps/logger/tasks.py b/onadata/apps/logger/tasks.py index 44757ad6c..265a4a8e5 100644 --- a/onadata/apps/logger/tasks.py +++ b/onadata/apps/logger/tasks.py @@ -5,17 +5,18 @@ from collections import defaultdict from io import StringIO -from celery import task, shared_task +from celery import shared_task from dateutil import relativedelta from django.contrib.auth.models import User from django.core.files.storage import get_storage_class from django.core.management import call_command +from onadata.celery import app from .models.submission_counter import SubmissionCounter from .models import Instance, XForm -@task() +@app.task() def create_monthly_counters(): user_ids = User.objects.values_list('pk', flat=True) for user_id in user_ids: diff --git a/onadata/apps/logger/tests/test_form_submission.py b/onadata/apps/logger/tests/test_form_submission.py index cd9aaa755..30dce9c85 100644 --- a/onadata/apps/logger/tests/test_form_submission.py +++ b/onadata/apps/logger/tests/test_form_submission.py @@ -2,12 +2,12 @@ import os import re +import pytest from django.http import Http404 from django_digest.test import DigestAuth from django_digest.test import Client as DigestClient from guardian.shortcuts import assign_perm from mock import patch -from nose import SkipTest from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.main.models.user_profile import UserProfile @@ -145,6 +145,7 @@ def test_submission_to_require_auth_without_perm(self): self.assertEqual(self.response.status_code, 403) + @pytest.mark.skip(reason='Send authentication challenge when xform.require_auth is set') def test_submission_to_require_auth_with_perm(self): """ test submission to a private form by non-owner is forbidden. @@ -153,7 +154,6 @@ def test_submission_to_require_auth_with_perm(self): This is non-trivial because we do not know the xform until we have parsed the XML. """ - raise SkipTest view = XFormViewSet.as_view({ 'patch': 'partial_update' diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index f042b52ad..b73118478 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -1,10 +1,8 @@ # coding: utf-8 -import datetime import json import logging -from bson import json_util, ObjectId -from celery import task +from bson import json_util from dateutil import parser from django.conf import settings from django.db import models @@ -12,6 +10,7 @@ from django.utils.six import string_types from django.utils.translation import gettext as t +from onadata.celery import app from onadata.apps.api.mongo_helper import MongoHelper from onadata.apps.logger.models import Instance from onadata.apps.logger.models import Note @@ -51,7 +50,8 @@ def datetime_from_str(text): return None return dt -@task + +@app.task def update_mongo_instance(record): # since our dict always has an id, save will always result in an upsert op # - so we dont need to worry whether its an edit or not diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index 455f46bc9..68c7d8457 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -1,14 +1,15 @@ # coding: utf-8 import logging -import pytz import re import sys -from celery import task, shared_task from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +from celery import shared_task from django.conf import settings from django.core.mail import mail_admins -from requests import ConnectionError +from onadata.celery import app from onadata.apps.viewer.models.export import Export from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.export_tools import ( @@ -18,6 +19,7 @@ ) from onadata.libs.utils.logger_tools import mongo_sync_status, report_exception + def create_async_export(xform, export_type, query, force_xlsx, options=None): username = xform.user.username id_string = xform.id_string @@ -74,7 +76,7 @@ def _create_export(xform, export_type): return None -@task() +@app.task() def create_xls_export(username, id_string, export_id, query=None, force_xlsx=True, group_delimiter='/', split_select_multiples=True, @@ -114,7 +116,7 @@ def create_xls_export(username, id_string, export_id, query=None, return gen_export.id -@task() +@app.task() def create_csv_export(username, id_string, export_id, query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): @@ -149,7 +151,7 @@ def create_csv_export(username, id_string, export_id, query=None, return gen_export.id -@task() +@app.task() def create_kml_export(username, id_string, export_id, query=None): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state @@ -177,7 +179,7 @@ def create_kml_export(username, id_string, export_id, query=None): return gen_export.id -@task() +@app.task() def create_zip_export(username, id_string, export_id, query=None): export = Export.objects.get(id=export_id) try: @@ -204,7 +206,7 @@ def create_zip_export(username, id_string, export_id, query=None): return gen_export.id -@task() +@app.task() def delete_export(export_id): try: export = Export.objects.get(id=export_id) @@ -232,7 +234,7 @@ def delete_export(export_id): re.IGNORECASE) -@task() +@app.task() def email_mongo_sync_status(): """Check the status of records in the mysql db versus mongodb, and, if necessary, invoke the command to re-sync the two databases, sending an @@ -262,7 +264,7 @@ def log_stuck_exports_and_mark_failed(): max_export_run_time = getattr(settings, 'CELERY_TASK_TIME_LIMIT', 2100) # Allow a generous grace period max_allowed_export_age = timedelta(seconds=max_export_run_time * 4) - this_moment = datetime.now(tz=pytz.UTC) + this_moment = datetime.now(tz=ZoneInfo('UTC')) oldest_allowed_timestamp = this_moment - max_allowed_export_age stuck_exports = Export.objects.filter( internal_status=Export.PENDING, @@ -282,5 +284,3 @@ def log_stuck_exports_and_mark_failed(): # Export.save() is a busybody; bypass it with update() stuck_exports.filter(pk=stuck_export.pk).update( internal_status=Export.FAILED) - - \ No newline at end of file diff --git a/onadata/libs/mixins/openrosa_headers_mixin.py b/onadata/libs/mixins/openrosa_headers_mixin.py index 0f8976767..ecb93f8a0 100644 --- a/onadata/libs/mixins/openrosa_headers_mixin.py +++ b/onadata/libs/mixins/openrosa_headers_mixin.py @@ -1,7 +1,7 @@ # coding: utf-8 -import pytz - from datetime import datetime +from zoneinfo import ZoneInfo + from django.conf import settings # 10,000,000 bytes @@ -10,8 +10,7 @@ class OpenRosaHeadersMixin: def get_openrosa_headers(self, request, location=True): - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') + dt = datetime.now(tz=ZoneInfo('UTC')).strftime('%a, %d %b %Y %H:%M:%S %Z') data = { 'Date': dt, diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 4fa3bf9a7..56d7306b2 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -5,8 +5,8 @@ import traceback from datetime import date, datetime from xml.parsers.expat import ExpatError +from zoneinfo import ZoneInfo -import pytz from dict2xml import dict2xml from django.conf import settings from django.contrib.auth.models import User @@ -761,8 +761,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') + dt = datetime.now(tz=ZoneInfo('UTC')).strftime('%a, %d %b %Y %H:%M:%S %Z') self['Date'] = dt self['X-OpenRosa-Accept-Content-Length'] = DEFAULT_CONTENT_LENGTH self['Content-Type'] = DEFAULT_CONTENT_TYPE diff --git a/pip-compile.sh b/pip-compile.sh index 8d1fd9d31..78bc49a3a 100755 --- a/pip-compile.sh +++ b/pip-compile.sh @@ -1,4 +1,10 @@ #!/bin/bash +# +if [ -z "$(dpkg -l|grep 'libpq-dev')" ]; then + echo "Installing dependencies needed to pip-compile..." + apt-get -qq update && apt-get -qq -y install gcc libpq-dev +fi + for in_file in dependencies/pip/*.in do # pass any arguments to pip-compile