From c69efff1efb70180cb6fea119b0d4bffa7fc7d2f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Apr 2022 14:08:35 -0400 Subject: [PATCH 1/7] Miscalleanous fixes to make the app start --- .gitlab-ci.yml | 4 +- dependencies/pip/dev.txt | 4 +- dependencies/pip/prod.txt | 4 +- dependencies/pip/requirements.in | 4 +- dependencies/pip/requirements.txt | 4 +- .../tests/viewsets/test_attachment_viewset.py | 6 +-- onadata/apps/api/viewsets/briefcase_api.py | 5 +- onadata/apps/api/viewsets/data_viewset.py | 3 +- onadata/apps/api/viewsets/xform_viewset.py | 7 ++- .../0023_alter_parsedinstance_id.py | 20 +++++++ onadata/apps/logger/models/instance.py | 10 ++-- onadata/apps/logger/models/xform.py | 4 +- onadata/apps/logger/views.py | 3 +- onadata/apps/logger/xform_instance_parser.py | 3 +- onadata/apps/main/models/audit.py | 3 +- onadata/apps/main/models/meta_data.py | 3 +- .../0005_alter_parsedinstance_id.py | 18 +++++++ onadata/apps/viewer/models/data_dictionary.py | 11 ++-- onadata/apps/viewer/models/parsed_instance.py | 12 ++--- onadata/apps/viewer/pandas_mongo_bridge.py | 5 +- .../apps/viewer/tests/test_export_builder.py | 8 +-- onadata/apps/viewer/tests/test_remongo.py | 1 - onadata/apps/viewer/xls_writer.py | 3 +- onadata/libs/filters.py | 3 +- onadata/libs/models/signals.py | 3 +- onadata/libs/renderers/renderers.py | 4 +- onadata/libs/serializers/fields/json_field.py | 7 ++- onadata/libs/utils/decorators.py | 54 +++++++++---------- onadata/libs/utils/export_tools.py | 20 ++++--- onadata/libs/utils/string.py | 3 +- onadata/settings/base.py | 6 ++- onadata/settings/testing.py | 2 +- 32 files changed, 139 insertions(+), 108 deletions(-) create mode 100644 onadata/apps/logger/migrations/0023_alter_parsedinstance_id.py create mode 100644 onadata/apps/viewer/migrations/0005_alter_parsedinstance_id.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 781b5f4ed..ee4a18f9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,9 +27,9 @@ test: POSTGRES_PASSWORD: kobo POSTGRES_DB: kobocat_test services: - - name: postgis/postgis:9.5-2.5 + - name: postgis/postgis:14-3.2 alias: postgres - - name: redis:3.2 + - name: redis:6.2 alias: redis_cache script: - apt-get update && apt-get install -y ghostscript gdal-bin libproj-dev gettext openjdk-11-jre diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index c4bdecc7d..a9d2c91ce 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -16,6 +16,8 @@ amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu +asgiref==3.5.0 + # via django asttokens==2.0.5 # via stack-data async-timeout==4.0.2 @@ -72,7 +74,7 @@ deprecated==1.2.13 # redis dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.28 +django==3.2 # via # -r dependencies/pip/requirements.in # django-celery-beat diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index 98318ad2b..65172fc60 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -16,6 +16,8 @@ amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu +asgiref==3.5.0 + # via django async-timeout==4.0.2 # via redis billiard==3.6.4.0 @@ -62,7 +64,7 @@ deprecated==1.2.13 # redis dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.28 +django==3.2 # via # -r dependencies/pip/requirements.in # django-celery-beat diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 03bb72e0b..45f5d841c 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -18,7 +18,7 @@ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip# -e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # Regular PyPI packages -Django>=2.2,<2.3 +Django==3.2 django-environ django-templated-email dpath @@ -29,7 +29,7 @@ psycopg2>=2.8,<2.9 pymongo==3.12.3 lxml pyxform==1.9.0 -django-reversion<3.0.2 +django-reversion xlwt openpyxl celery diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 4d61d7443..6c21c0528 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -16,6 +16,8 @@ amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu +asgiref==3.5.0 + # via django async-timeout==4.0.2 # via redis billiard==3.6.4.0 @@ -62,7 +64,7 @@ deprecated==1.2.13 # redis dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==2.2.28 +django==3.2 # via # -r dependencies/pip/requirements.in # django-celery-beat diff --git a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py index 61d6e89d2..e093fc973 100644 --- a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py @@ -1,8 +1,6 @@ # coding: utf-8 from os import path -from django.utils.six import string_types - from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ TestAbstractViewSet from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet @@ -112,7 +110,7 @@ def test_direct_image_link(self): request = self.factory.get('/', data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, string_types)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, self.attachment.secure_url()) data['filename'] = 10000000 @@ -137,5 +135,5 @@ def test_direct_image_link_uppercase(self): request = self.factory.get('/', data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, string_types)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, self.attachment.secure_url()) diff --git a/onadata/apps/api/viewsets/briefcase_api.py b/onadata/apps/api/viewsets/briefcase_api.py index 78b3129a0..452f559dd 100644 --- a/onadata/apps/api/viewsets/briefcase_api.py +++ b/onadata/apps/api/viewsets/briefcase_api.py @@ -7,7 +7,6 @@ from django.contrib.auth.models import User from django.http import Http404 from django.utils.translation import gettext as t -from django.utils import six from rest_framework import exceptions from rest_framework import mixins from rest_framework import status @@ -36,7 +35,7 @@ def _extract_uuid(text): - if isinstance(text, six.string_types): + if isinstance(text, str): form_id_parts = text.split('/') if form_id_parts.__len__() < 2: @@ -52,7 +51,7 @@ def _extract_uuid(text): def _extract_id_string(formId): - if isinstance(formId, six.string_types): + if isinstance(formId, str): return formId[0:formId.find('[')] return formId diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 7484fffe6..463238a6f 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -6,7 +6,6 @@ from django.db.models.signals import pre_delete from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.utils import six from django.utils.translation import gettext as t from rest_framework import status @@ -500,7 +499,7 @@ def filter_queryset(self, queryset, view=None): pk = self.kwargs.get(self.lookup_field) tags = self.request.query_params.get('tags', None) - if tags and isinstance(tags, six.string_types): + if tags and isinstance(tags, str): tags = tags.split(',') qs = qs.filter(tags__name__in=tags).distinct() diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 3ade5c266..cd3e4d392 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -6,9 +6,8 @@ from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.core.files.storage import get_storage_class -from django.http import Http404, HttpResponseBadRequest, HttpResponseRedirect +from django.http import Http404, HttpResponseBadRequest from django.shortcuts import get_object_or_404 -from django.utils import six from django.utils.translation import gettext as t from rest_framework import exceptions from rest_framework import status @@ -74,7 +73,7 @@ def _set_start_end_params(request, query): # check for start and end params if 'start' in request.GET or 'end' in request.GET: query = json.loads(query) \ - if isinstance(query, six.string_types) else query + if isinstance(query, str) else query query[SUBMISSION_TIME] = {} try: @@ -130,7 +129,7 @@ def _get_user(username): def _get_owner(request): owner = request.data.get('owner') or request.user - if isinstance(owner, six.string_types): + if isinstance(owner, str): owner = _get_user(owner) if owner is None: diff --git a/onadata/apps/logger/migrations/0023_alter_parsedinstance_id.py b/onadata/apps/logger/migrations/0023_alter_parsedinstance_id.py new file mode 100644 index 000000000..86d4b91da --- /dev/null +++ b/onadata/apps/logger/migrations/0023_alter_parsedinstance_id.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2022-04-27 19:05 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0022_misc_model_options'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='id', + field=models.BigAutoField(primary_key=True, serialize=False), + ), + ] diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 198d71ae6..55a740a81 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -1,18 +1,17 @@ # coding: utf-8 from datetime import date from hashlib import sha256 -from zoneinfo import ZoneInfo import reversion from django.contrib.auth.models import User from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point -from django.db import transaction +from django.db import models as django_models, transaction from django.db.models import F from django.db.models.signals import post_delete from django.db.models.signals import post_save from django.utils import timezone -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from jsonfield import JSONField from taggit.managers import TaggableManager @@ -171,6 +170,7 @@ class Instance(models.Model): XML_HASH_LENGTH = 64 DEFAULT_XML_HASH = None + id = django_models.BigAutoField(primary_key=True) json = JSONField(default={}, null=False) xml = models.TextField() xml_hash = models.CharField(max_length=XML_HASH_LENGTH, db_index=True, null=True, @@ -399,11 +399,11 @@ def get_hash(input_string): """ Compute the SHA256 hash of the given string. A wrapper to standardize hash computation. - :param string_types input_string: The string to be hashed. + :param str input_string: The string to be hashed. :return: The resulting hash. :rtype: str """ - input_string = smart_text(input_string) + input_string = smart_str(input_string) return sha256(input_string.encode()).hexdigest() @property diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index db30154be..51960e250 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -11,7 +11,7 @@ from django.urls import reverse from django.db import models from django.db.models.signals import post_save, post_delete -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from django.utils.translation import gettext_lazy as t from guardian.shortcuts import ( @@ -135,7 +135,7 @@ def _set_id_string(self): self.id_string = matches[0] def _set_title(self): - self.xml = smart_text(self.xml) + self.xml = smart_str(self.xml) text = re.sub(r'\s+', ' ', self.xml) matches = title_pattern.findall(text) title_xml = matches[0][:XFORM_TITLE_LENGTH] diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 44a2161a0..a30a5c8d7 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -20,7 +20,6 @@ from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.template import loader -from django.utils.six import text_type from django.utils.translation import gettext as t from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt @@ -47,7 +46,7 @@ def _bad_request(e): - strerror = text_type(e) + strerror = str(e) return strerror and strerror in IO_ERROR_STRINGS diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index ec63241f9..4251b7734 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -7,7 +7,6 @@ import six from django.utils.encoding import smart_str from django.utils.translation import gettext as t -from django.utils.six import text_type from xml.dom import minidom, Node from onadata.libs.utils.common_tags import XFORM_ID_STRING @@ -205,7 +204,7 @@ def _flatten_dict(d, prefix): # hack: removing [1] index to be consistent across # surveys that have a single repitition of the # loop versus mutliple. - item_prefix[-1] += "[%s]" % text_type(i + 1) + item_prefix[-1] += "[%s]" % str(i + 1) if type(item) == dict: for pair in _flatten_dict(item, item_prefix): yield pair diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 3b23aba41..8f5c593c7 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta from django.conf import settings -from django.utils.six import string_types from onadata.apps.viewer.models.parsed_instance import DATETIME_FORMAT from onadata.apps.api.mongo_helper import MongoHelper @@ -37,7 +36,7 @@ def query_mongo(cls, username, query=None, fields=None, sort=None, start=0, val, DATETIME_FORMAT) except ValueError: pass - elif isinstance(query[cls.CREATED_ON], string_types): + elif isinstance(query[cls.CREATED_ON], str): val = query[cls.CREATED_ON] try: created_on = datetime.strptime(val, DATETIME_FORMAT) diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index f394c0e6d..ac7791338 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -3,7 +3,7 @@ import os import requests from contextlib import closing - +from urllib.parse import urlparse from django.core.exceptions import ValidationError from django.core.files.temp import NamedTemporaryFile @@ -12,7 +12,6 @@ from django.db import models from django.conf import settings from django.utils import timezone -from six.moves.urllib.parse import urlparse from requests.exceptions import RequestException from onadata.apps.logger.models import XForm diff --git a/onadata/apps/viewer/migrations/0005_alter_parsedinstance_id.py b/onadata/apps/viewer/migrations/0005_alter_parsedinstance_id.py new file mode 100644 index 000000000..8168660c4 --- /dev/null +++ b/onadata/apps/viewer/migrations/0005_alter_parsedinstance_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2022-04-27 19:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0004_update_meta_data_export_types'), + ] + + operations = [ + migrations.AlterField( + model_name='parsedinstance', + name='id', + field=models.BigAutoField(primary_key=True, serialize=False), + ), + ] diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 9a02b5de5..9aa2fec97 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -5,8 +5,7 @@ from django.db import models from django.db.models.signals import post_save -from django.utils.encoding import smart_text -from django.utils.six import text_type +from django.utils.encoding import smart_str from guardian.shortcuts import assign_perm, get_perms_for_model from pyxform import SurveyElementBuilder from pyxform.builder import create_survey_from_xls @@ -132,13 +131,13 @@ def set_uuid_in_xml(self, file_name=None, id_string=None): calculate_node.setAttribute("calculate", "'%s'" % self.uuid) model_node.appendChild(calculate_node) - self.xml = smart_text(doc.toprettyxml(indent=" ", encoding='utf-8')) + self.xml = smart_str(doc.toprettyxml(indent=" ", encoding='utf-8')) # hack # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ text_re = re.compile(r'>\n\s+([^<>\s].*?)\n\s+)\n( )*') - pretty_xml = text_re.sub(r'>\g<1>\g<1>', pretty_xml) inline_output = re.compile(r'').sub( '', inline_output) @@ -213,7 +212,7 @@ def get_mongo_field_names_dict(self): """ names = {} for elem in self.get_survey_elements(): - names[MongoHelper.encode(text_type(elem.get_abbreviated_xpath()))] = \ + names[MongoHelper.encode(str(elem.get_abbreviated_xpath()))] = \ elem.get_abbreviated_xpath() return names @@ -248,7 +247,7 @@ def xpaths(self, prefix='', survey_element=None, result=None, return [] if result is None: result = [] - path = '/'.join([prefix, text_type(survey_element.name)]) + path = '/'.join([prefix, str(survey_element.name)]) if survey_element.children is not None: # add xpaths to result for each child indices = [''] if type(survey_element) != RepeatingSection else \ diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index b73118478..040103302 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -7,7 +7,6 @@ from django.conf import settings from django.db import models from django.db.models.signals import pre_delete -from django.utils.six import string_types from django.utils.translation import gettext as t from onadata.celery import app @@ -71,6 +70,7 @@ class ParsedInstance(models.Model): DEFAULT_LIMIT = 30000 DEFAULT_BATCHSIZE = 1000 + id = models.BigAutoField(primary_key=True) instance = models.OneToOneField(Instance, related_name="parsed_instance", on_delete=models.CASCADE) start_time = models.DateTimeField(null=True) end_time = models.DateTimeField(null=True) @@ -106,7 +106,7 @@ def query_mongo(cls, username, id_string, query, fields, sort, start=0, cursor = cls._get_mongo_cursor(query, fields) - if isinstance(sort, string_types): + if isinstance(sort, str): sort = json.loads(sort, object_hook=json_util.object_hook) sort = sort if sort else {} @@ -123,7 +123,7 @@ def mongo_aggregate(cls, query, pipeline): pipeline - list of dicts or dict of mongodb pipeline operators, http://docs.mongodb.org/manual/reference/operator/aggregation-pipeline """ - if isinstance(query, string_types): + if isinstance(query, str): query = json.loads( query, object_hook=json_util.object_hook) if query else {} if not (isinstance(pipeline, dict) or isinstance(pipeline, list)): @@ -158,7 +158,7 @@ def query_mongo_minimal( cursor = cls._get_mongo_cursor(query, fields) - if isinstance(sort, string_types): + if isinstance(sort, str): sort = json.loads(sort, object_hook=json_util.object_hook) sort = sort if sort else {} @@ -199,7 +199,7 @@ def _get_mongo_cursor(cls, query, fields): fields_to_select = {cls.USERFORM_ID: 0} # fields must be a string array i.e. '["name", "age"]' - if isinstance(fields, string_types): + if isinstance(fields, str): fields = json.loads(fields, object_hook=json_util.object_hook) fields = fields if fields else [] @@ -227,7 +227,7 @@ def _get_mongo_cursor_query(cls, query, username=None, id_string=None): """ # TODO: give more detailed error messages to 3rd parties # using the API when json.loads fails - if isinstance(query, string_types): + if isinstance(query, str): query = json.loads(query, object_hook=json_util.object_hook) query = query if query else {} query = MongoHelper.to_safe_dict(query, reading=True) diff --git a/onadata/apps/viewer/pandas_mongo_bridge.py b/onadata/apps/viewer/pandas_mongo_bridge.py index 64a5aa5ca..b5408f957 100644 --- a/onadata/apps/viewer/pandas_mongo_bridge.py +++ b/onadata/apps/viewer/pandas_mongo_bridge.py @@ -4,7 +4,6 @@ from itertools import chain from django.conf import settings -from django.utils.six import string_types from pandas.core.frame import DataFrame # an immediate fix to an error with the installation of pandas v0.15 @@ -193,7 +192,7 @@ def _tag_edit_string(cls, record): def _split_gps_fields(cls, record, gps_fields): updated_gps_fields = {} for key, value in record.items(): - if key in gps_fields and isinstance(value, string_types): + if key in gps_fields and isinstance(value, str): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key) gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) # hack, check if its a list and grab the object within that @@ -487,7 +486,7 @@ def _add_column_to_section(self, sheet_name, column): xpath = None if isinstance(column, SurveyElement): xpath = column.get_abbreviated_xpath() - elif isinstance(column, string_types): + elif isinstance(column, str): xpath = column assert(xpath) # make sure column is not already in list diff --git a/onadata/apps/viewer/tests/test_export_builder.py b/onadata/apps/viewer/tests/test_export_builder.py index 0f82253c2..f47572388 100644 --- a/onadata/apps/viewer/tests/test_export_builder.py +++ b/onadata/apps/viewer/tests/test_export_builder.py @@ -2,17 +2,11 @@ import csv import datetime import os -import shutil -import tempfile -import unittest -import zipfile from django.conf import settings from django.core.files.temp import NamedTemporaryFile -from django.utils.six import string_types from openpyxl import load_workbook from pyxform.builder import create_survey_from_xls -from savReaderWriter import SavReader from onadata.apps.api.mongo_helper import MongoHelper from onadata.apps.main.tests.test_base import TestBase @@ -758,7 +752,7 @@ def test_type_conversion(self): data = dict_to_joined_export(submission_2, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) - self.assertIsInstance(new_row['amount'], string_types) + self.assertIsInstance(new_row['amount'], str) self.assertEqual(new_row['amount'], '') def test_xls_convert_dates_before_1900(self): diff --git a/onadata/apps/viewer/tests/test_remongo.py b/onadata/apps/viewer/tests/test_remongo.py index dfc47f49d..cc8811c90 100644 --- a/onadata/apps/viewer/tests/test_remongo.py +++ b/onadata/apps/viewer/tests/test_remongo.py @@ -4,7 +4,6 @@ from django.conf import settings from django.core.management import call_command from django_digest.test import DigestAuth -from django.utils.six import string_types from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.parsed_instance import ParsedInstance diff --git a/onadata/apps/viewer/xls_writer.py b/onadata/apps/viewer/xls_writer.py index a8ef535ac..81c1f287b 100644 --- a/onadata/apps/viewer/xls_writer.py +++ b/onadata/apps/viewer/xls_writer.py @@ -1,7 +1,6 @@ # coding: utf-8 from collections import defaultdict -from django.utils.six import text_type from pyxform import Section, Question from xlwt import Workbook @@ -86,7 +85,7 @@ def write_tables_to_workbook(self, tables): self.add_sheet(table_name) for i, row in enumerate(table): for j, value in enumerate(row): - self._sheets[table_name].write(i, j, text_type(value)) + self._sheets[table_name].write(i, j, str(value)) return self._workbook def save_workbook_to_file(self): diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index a4151f1aa..5adeb7a50 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,6 +1,5 @@ # coding: utf-8 from django.shortcuts import get_object_or_404 -from django.utils import six from rest_framework import filters from rest_framework_guardian import filters as guardian_filters from rest_framework.exceptions import ParseError @@ -70,7 +69,7 @@ def filter_queryset(self, request, queryset, view): # filter by tags if available. tags = request.query_params.get('tags', None) - if tags and isinstance(tags, six.string_types): + if tags and isinstance(tags, str): tags = tags.split(',') return queryset.filter(tags__name__in=tags) diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 14d0b4227..aafa7feff 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,6 +1,5 @@ # coding: utf-8 import django.dispatch -from django.utils.six import string_types from onadata.apps.logger.models import XForm @@ -26,7 +25,7 @@ def add_tags_to_xform_instances(sender, **kwargs): def delete_tag_from_xform_instances(sender, **kwargs): xform = kwargs.get('xform', None) tag = kwargs.get('tag', None) - if isinstance(xform, XForm) and isinstance(tag, string_types): + if isinstance(xform, XForm) and isinstance(tag, str): # update existing instances with the new tag for instance in xform.instances.all(): if tag in instance.tags.names(): diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 60b435d4b..53db17668 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -2,7 +2,7 @@ import io from django.utils.xmlutils import SimplerXMLGenerator -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from rest_framework.negotiation import DefaultContentNegotiation from rest_framework.renderers import BaseRenderer from rest_framework.renderers import TemplateHTMLRenderer @@ -117,7 +117,7 @@ def _to_xml(self, xml, data): pass else: - xml.characters(smart_text(data)) + xml.characters(smart_str(data)) class XFormManifestRenderer(XFormListRenderer): diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index 604e1c5c8..6ff02429c 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -1,26 +1,25 @@ # coding: utf-8 import json -from django.utils.six import string_types from rest_framework import serializers class JsonField(serializers.Field): def to_representation(self, value): - if isinstance(value, string_types): + if isinstance(value, str): return json.loads(value) return value def to_internal_value(self, value): - if isinstance(value, string_types): + if isinstance(value, str): return json.loads(value) return value @classmethod def to_json(cls, data): - if isinstance(data, string_types): + if isinstance(data, str): return json.loads(data) return data diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 1a02ddbe5..9e4a628d1 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,11 +1,11 @@ # coding: utf-8 from functools import wraps -from urllib.parse import urlparse - -from django.contrib.auth import REDIRECT_FIELD_NAME -from django.utils.decorators import available_attrs -from django.conf import settings -from django.http import HttpResponseRedirect +# from urllib.parse import urlparse +# +# from django.contrib.auth import REDIRECT_FIELD_NAME +# from django.utils.decorators import available_attrs +# from django.conf import settings +# from django.http import HttpResponseRedirect from onadata.apps.logger.models import XForm @@ -19,27 +19,27 @@ def with_check_obj(*args, **kwargs): return with_check_obj -def is_owner(view_func): - @wraps(view_func, assigned=available_attrs(view_func)) - def _wrapped_view(request, *args, **kwargs): - # assume username is first arg - if request.user.is_authenticated: - if request.user.username == kwargs['username']: - return view_func(request, *args, **kwargs) - protocol = "https" if request.is_secure() else "http" - return HttpResponseRedirect(f'{protocol}://{request.get_host()}') - path = request.build_absolute_uri() - login_url = request.build_absolute_uri(settings.LOGIN_URL) - # If the login url is the same scheme and net location then just - # use the path as the "next" url. - login_scheme, login_netloc = urlparse(login_url)[:2] - current_scheme, current_netloc = urlparse(path)[:2] - if ((not login_scheme or login_scheme == current_scheme) and - (not login_netloc or login_netloc == current_netloc)): - path = request.get_full_path() - from django.contrib.auth.views import redirect_to_login - return redirect_to_login(path, None, REDIRECT_FIELD_NAME) - return _wrapped_view +# def is_owner(view_func): +# @wraps(view_func, assigned=available_attrs(view_func)) +# def _wrapped_view(request, *args, **kwargs): +# # assume username is first arg +# if request.user.is_authenticated: +# if request.user.username == kwargs['username']: +# return view_func(request, *args, **kwargs) +# protocol = "https" if request.is_secure() else "http" +# return HttpResponseRedirect(f'{protocol}://{request.get_host()}') +# path = request.build_absolute_uri() +# login_url = request.build_absolute_uri(settings.LOGIN_URL) +# # If the login url is the same scheme and net location then just +# # use the path as the "next" url. +# login_scheme, login_netloc = urlparse(login_url)[:2] +# current_scheme, current_netloc = urlparse(path)[:2] +# if ((not login_scheme or login_scheme == current_scheme) and +# (not login_netloc or login_netloc == current_netloc)): +# path = request.get_full_path() +# from django.contrib.auth.views import redirect_to_login +# return redirect_to_login(path, None, REDIRECT_FIELD_NAME) +# return _wrapped_view def apply_form_field_names(func): diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a308132dc..1e54ced26 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -10,7 +10,7 @@ from django.core.files.temp import NamedTemporaryFile from django.core.files.storage import get_storage_class from django.contrib.auth.models import User -from django.shortcuts import render_to_response +from django.shortcuts import render from django.utils.text import slugify from openpyxl.utils.datetime import to_excel, time_to_days, timedelta_to_days from openpyxl.workbook import Workbook @@ -699,12 +699,20 @@ def generate_attachments_zip_export( def generate_kml_export( - export_type, extension, username, id_string, export_id=None, - filter_query=None): + export_type, + extension, + username, + id_string, + export_id=None, + filter_query=None, # Not used, ToDo removed it? +): user = User.objects.get(username=username) xform = XForm.objects.get(user__username=username, id_string=id_string) - response = render_to_response( - 'survey.kml', {'data': kml_export_data(id_string, user)}) + response = render( + request=None, + template_name='survey.kml', + context={'data': kml_export_data(id_string, user)}, + ) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) @@ -728,7 +736,7 @@ def generate_kml_export( dir_name, basename = os.path.split(export_filename) # get or create export object - if(export_id): + if export_id: export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) diff --git a/onadata/libs/utils/string.py b/onadata/libs/utils/string.py index 906506ccf..60813cfc0 100644 --- a/onadata/libs/utils/string.py +++ b/onadata/libs/utils/string.py @@ -1,6 +1,5 @@ # coding: utf-8 import base64 -from django.utils.six import string_types def base64_encodestring(obj): @@ -16,4 +15,4 @@ def base64_decodestring(obj): def str2bool(v): return v.lower() in ( - 'yes', 'true', 't', '1') if isinstance(v, string_types) else v + 'yes', 'true', 't', '1') if isinstance(v, str) else v diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 71cda360c..bd8c7b0dc 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -67,7 +67,7 @@ def skip_suspicious_operations(record): # http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = 'en-us' -SITE_ID = env.str('DJANGO_SITE_ID', '1') +SITE_ID = env.int('DJANGO_SITE_ID', 1) # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. @@ -430,6 +430,8 @@ def skip_suspicious_operations(record): 'VIEW_DESCRIPTION_FUNCTION': 'onadata.apps.api.tools.get_view_description', } +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + ################################ # KoBoCAT settings # ################################ @@ -726,7 +728,7 @@ def skip_suspicious_operations(record): # PyMongo 3 does acknowledged writes by default # https://emptysqua.re/blog/pymongos-new-default-safe-writes/ MONGO_CONNECTION = MongoClient( - MONGO_CONNECTION_URL, j=True, tz_aware=True) + MONGO_CONNECTION_URL, journal=True, tz_aware=True) MONGO_DB = MONGO_CONNECTION[MONGO_DATABASE['NAME']] diff --git a/onadata/settings/testing.py b/onadata/settings/testing.py index 70f8c057a..1487c9cfd 100644 --- a/onadata/settings/testing.py +++ b/onadata/settings/testing.py @@ -69,5 +69,5 @@ MONGO_CONNECTION_URL = 'mongodb://fakehost/formhub_test' MONGO_CONNECTION = MockMongoClient( - MONGO_CONNECTION_URL, j=True, tz_aware=True) + MONGO_CONNECTION_URL, journal=True, tz_aware=True) MONGO_DB = MONGO_CONNECTION['formhub_test'] From 8540147d90ead5d0066e04c4d57e2d15f8afe1f8 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Apr 2022 16:01:29 -0400 Subject: [PATCH 2/7] Update pip dependencies --- dependencies/pip/dev.txt | 8 ++++---- dependencies/pip/prod.txt | 8 ++++---- dependencies/pip/requirements.txt | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index a9d2c91ce..969d21c36 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -28,9 +28,9 @@ backcall==0.2.0 # via ipython billiard==3.6.4.0 # via celery -boto3==1.22.0 +boto3==1.22.2 # via -r dependencies/pip/requirements.in -botocore==1.25.0 +botocore==1.25.2 # via # boto3 # s3transfer @@ -58,7 +58,7 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery -cryptography==36.0.2 +cryptography==37.0.0 # via jwcrypto decorator==5.1.1 # via @@ -116,7 +116,7 @@ django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.1 +django-reversion==5.0.0 # via -r dependencies/pip/requirements.in django-taggit==2.1.0 # via -r dependencies/pip/requirements.in diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index 65172fc60..d45f83f29 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -22,9 +22,9 @@ async-timeout==4.0.2 # via redis billiard==3.6.4.0 # via celery -boto3==1.22.0 +boto3==1.22.2 # via -r dependencies/pip/requirements.in -botocore==1.25.0 +botocore==1.25.2 # via # boto3 # s3transfer @@ -52,7 +52,7 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery -cryptography==36.0.2 +cryptography==37.0.0 # via jwcrypto defusedxml==0.7.1 # via @@ -106,7 +106,7 @@ django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.1 +django-reversion==5.0.0 # via -r dependencies/pip/requirements.in django-taggit==2.1.0 # via -r dependencies/pip/requirements.in diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 6c21c0528..d84ee7d75 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -22,9 +22,9 @@ async-timeout==4.0.2 # via redis billiard==3.6.4.0 # via celery -boto3==1.22.0 +boto3==1.22.2 # via -r dependencies/pip/requirements.in -botocore==1.25.0 +botocore==1.25.2 # via # boto3 # s3transfer @@ -52,7 +52,7 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery -cryptography==36.0.2 +cryptography==37.0.0 # via jwcrypto defusedxml==0.7.1 # via @@ -106,7 +106,7 @@ django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.1 +django-reversion==5.0.0 # via -r dependencies/pip/requirements.in django-taggit==2.1.0 # via -r dependencies/pip/requirements.in From a0967d6799116feb395d29c439aafdaa28b92ac9 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Apr 2022 16:19:35 -0400 Subject: [PATCH 3/7] Remove obsolete code (is_owner decorator) --- onadata/libs/utils/decorators.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 9e4a628d1..b4a54e2fa 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,11 +1,5 @@ # coding: utf-8 from functools import wraps -# from urllib.parse import urlparse -# -# from django.contrib.auth import REDIRECT_FIELD_NAME -# from django.utils.decorators import available_attrs -# from django.conf import settings -# from django.http import HttpResponseRedirect from onadata.apps.logger.models import XForm @@ -19,29 +13,6 @@ def with_check_obj(*args, **kwargs): return with_check_obj -# def is_owner(view_func): -# @wraps(view_func, assigned=available_attrs(view_func)) -# def _wrapped_view(request, *args, **kwargs): -# # assume username is first arg -# if request.user.is_authenticated: -# if request.user.username == kwargs['username']: -# return view_func(request, *args, **kwargs) -# protocol = "https" if request.is_secure() else "http" -# return HttpResponseRedirect(f'{protocol}://{request.get_host()}') -# path = request.build_absolute_uri() -# login_url = request.build_absolute_uri(settings.LOGIN_URL) -# # If the login url is the same scheme and net location then just -# # use the path as the "next" url. -# login_scheme, login_netloc = urlparse(login_url)[:2] -# current_scheme, current_netloc = urlparse(path)[:2] -# if ((not login_scheme or login_scheme == current_scheme) and -# (not login_netloc or login_netloc == current_netloc)): -# path = request.get_full_path() -# from django.contrib.auth.views import redirect_to_login -# return redirect_to_login(path, None, REDIRECT_FIELD_NAME) -# return _wrapped_view - - def apply_form_field_names(func): @wraps(func) def wrapper(*args, **kwargs): From 0a3195d7ba6796b8c9104e794c7d21bf66300ead Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Apr 2022 16:24:56 -0400 Subject: [PATCH 4/7] Update PostgreSQL driver to latest version --- dependencies/pip/dev.txt | 2 +- dependencies/pip/prod.txt | 2 +- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index 969d21c36..28c81279e 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -220,7 +220,7 @@ prompt-toolkit==3.0.29 # via # click-repl # ipython -psycopg2==2.8.6 +psycopg2==2.9.3 # via -r dependencies/pip/requirements.in ptyprocess==0.7.0 # via pexpect diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index d45f83f29..f625196f6 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -177,7 +177,7 @@ pillow==9.1.0 # elaphe3 prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 +psycopg2==2.9.3 # via -r dependencies/pip/requirements.in pycparser==2.21 # via cffi diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 45f5d841c..676495e09 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -25,7 +25,7 @@ dpath gdata-python3 modilabs-python-utils Pillow -psycopg2>=2.8,<2.9 +psycopg2 pymongo==3.12.3 lxml pyxform==1.9.0 diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index d84ee7d75..228055595 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -177,7 +177,7 @@ pillow==9.1.0 # elaphe3 prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 +psycopg2==2.9.3 # via -r dependencies/pip/requirements.in pycparser==2.21 # via cffi From 21d7b607f9801f85d1c145f0cc73142275d5ad2b Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Apr 2022 16:25:24 -0400 Subject: [PATCH 5/7] Make GitHub Actions tests run with PostgreSQL 14 --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 99dc4b731..833645424 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: python-version: ['3.10'] services: postgres: - image: postgis/postgis:9.5-2.5 + image: postgis/postgis:14-3.2 env: POSTGRES_USER: kobo POSTGRES_PASSWORD: kobo @@ -29,7 +29,7 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis_cache: - image: redis:3.2 + image: redis:6.2 ports: - 6380:6379 steps: From 60f045385d15c668d15a7d102aeab0729673f3a5 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 3 May 2022 13:59:02 -0400 Subject: [PATCH 6/7] Make pip dependencies consistent with KPI --- .github/workflows/pytest.yml | 2 +- .gitlab-ci.yml | 2 +- Dockerfile | 4 +- .../pip/{dev.in => dev_requirements.in} | 0 .../pip/{dev.txt => dev_requirements.txt} | 24 +- dependencies/pip/prod.in | 3 - dependencies/pip/prod.txt | 252 ------------------ dependencies/pip/requirements.in | 3 + dependencies/pip/requirements.txt | 2 + docker/init.bash | 12 +- 10 files changed, 28 insertions(+), 276 deletions(-) rename dependencies/pip/{dev.in => dev_requirements.in} (100%) rename dependencies/pip/{dev.txt => dev_requirements.txt} (92%) delete mode 100644 dependencies/pip/prod.in delete mode 100644 dependencies/pip/prod.txt diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 833645424..9521c8b8b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -45,7 +45,7 @@ jobs: - name: Install apt dependencies run: sudo apt-get install ghostscript libxml2-dev libxslt-dev python3-dev gdal-bin libproj-dev gettext postgresql-client openjdk-11-jre - name: Install Python dependencies - run: pip install -r dependencies/pip/dev.txt + run: pip install -r dependencies/pip/dev_requirements.txt - name: Run pytest run: pytest -vv -rf env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee4a18f9a..e554e1b72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ test: alias: redis_cache script: - apt-get update && apt-get install -y ghostscript gdal-bin libproj-dev gettext openjdk-11-jre - - pip install -r dependencies/pip/dev.txt + - pip install -r dependencies/pip/dev_requirements.txt - pytest -vv -rf deploy-beta: diff --git a/Dockerfile b/Dockerfile index 1566339b4..6f63c48e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m venv "$VIRTUAL_ENV" ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install --quiet --upgrade pip && \ pip install --quiet pip-tools -COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +COPY ./dependencies/pip/requirements.txt "${TMP_DIR}/pip_dependencies.txt" RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null FROM python:3.10-slim @@ -79,7 +79,7 @@ RUN adduser --disabled-password --gecos '' "$UWSGI_USER" # Copy virtualenv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +COPY ./dependencies/pip/requirements.txt "${TMP_DIR}/pip_dependencies.txt" COPY --from=build-python "$VIRTUAL_ENV" "$VIRTUAL_ENV" COPY . "${KOBOCAT_SRC_DIR}" diff --git a/dependencies/pip/dev.in b/dependencies/pip/dev_requirements.in similarity index 100% rename from dependencies/pip/dev.in rename to dependencies/pip/dev_requirements.in diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev_requirements.txt similarity index 92% rename from dependencies/pip/dev.txt rename to dependencies/pip/dev_requirements.txt index 28c81279e..e3673619c 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev_requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile dependencies/pip/dev.in +# pip-compile dependencies/pip/dev_requirements.in # -e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in @@ -152,16 +152,16 @@ executing==0.8.3 gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in httmock==1.4.0 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in idna==3.3 # via requests iniconfig==1.1.1 # via pytest ipdb==0.13.9 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in ipython==8.2.0 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # ipdb jedi==0.18.1 # via ipython @@ -184,11 +184,11 @@ markdown==3.3.6 matplotlib-inline==0.1.3 # via ipython mock==4.0.3 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in mongomock==4.0.0 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in numpy==1.22.3 # via pandas oauthlib==3.2.0 @@ -238,13 +238,13 @@ pyparsing==3.0.8 # via packaging pytest==7.1.2 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # pytest-django # pytest-env pytest-django==4.5.2 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in pytest-env==0.6.2 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in python-crontab==2.6.0 # via django-celery-beat python-dateutil==2.8.2 @@ -279,7 +279,7 @@ sentinels==1.0.0 sentry-sdk==1.5.10 # via -r dependencies/pip/requirements.in shell-command==0.1 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in simplejson==3.17.6 # via -r dependencies/pip/requirements.in six==1.16.0 @@ -292,7 +292,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.2 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # django stack-data==0.2.0 # via ipython @@ -313,6 +313,8 @@ urllib3==1.26.9 # botocore # requests # sentry-sdk +uwsgi==2.0.20 + # via -r dependencies/pip/requirements.in vine==5.0.0 # via # amqp diff --git a/dependencies/pip/prod.in b/dependencies/pip/prod.in deleted file mode 100644 index bd6f83b94..000000000 --- a/dependencies/pip/prod.in +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.in - -uWSGI diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt deleted file mode 100644 index f625196f6..000000000 --- a/dependencies/pip/prod.txt +++ /dev/null @@ -1,252 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# pip-compile dependencies/pip/prod.in -# --e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages - # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect - # via -r dependencies/pip/requirements.in -amqp==5.1.1 - # via - # -r dependencies/pip/requirements.in - # kombu -asgiref==3.5.0 - # via django -async-timeout==4.0.2 - # via redis -billiard==3.6.4.0 - # via celery -boto3==1.22.2 - # via -r dependencies/pip/requirements.in -botocore==1.25.2 - # via - # boto3 - # s3transfer -celery[redis]==5.2.6 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat -certifi==2021.10.8 - # via - # requests - # sentry-sdk -cffi==1.15.0 - # via cryptography -charset-normalizer==2.0.12 - # via requests -click==8.1.2 - # via - # celery - # click-didyoumean - # click-plugins - # click-repl -click-didyoumean==0.3.0 - # via celery -click-plugins==1.1.1 - # via celery -click-repl==0.2.0 - # via celery -cryptography==37.0.0 - # via jwcrypto -defusedxml==0.7.1 - # via - # djangorestframework-xml - # pyxform -deprecated==1.2.13 - # via - # jwcrypto - # redis -dict2xml==1.7.1 - # via -r dependencies/pip/requirements.in -django==3.2 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat - # django-cors-headers - # django-db-readonly - # django-extensions - # django-filter - # django-guardian - # django-oauth-toolkit - # django-render-block - # django-reversion - # django-storages - # django-taggit - # django-timezone-field - # djangorestframework - # djangorestframework-guardian - # jsonfield -django-celery-beat==2.2.1 - # via -r dependencies/pip/requirements.in -django-cors-headers==3.11.0 - # via -r dependencies/pip/requirements.in -django-db-readonly==0.7.0 - # via -r dependencies/pip/requirements.in -django-environ==0.8.1 - # via -r dependencies/pip/requirements.in -django-extensions==3.1.5 - # via -r dependencies/pip/requirements.in -django-filter==21.1 - # via -r dependencies/pip/requirements.in -django-guardian==2.4.0 - # via - # -r dependencies/pip/requirements.in - # djangorestframework-guardian -django-oauth-toolkit==2.0.0 - # via -r dependencies/pip/requirements.in -django-pure-pagination==0.3.0 - # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.2 - # via -r dependencies/pip/requirements.in -django-render-block==0.9.1 - # via django-templated-email -django-reversion==5.0.0 - # via -r dependencies/pip/requirements.in -django-taggit==2.1.0 - # via -r dependencies/pip/requirements.in -django-templated-email==3.0.0 - # via -r dependencies/pip/requirements.in -django-timezone-field==4.2.3 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat -djangorestframework==3.13.1 - # via - # -r dependencies/pip/requirements.in - # djangorestframework-csv - # djangorestframework-guardian -djangorestframework-csv==2.1.1 - # via -r dependencies/pip/requirements.in -djangorestframework-guardian==0.3.0 - # via -r dependencies/pip/requirements.in -djangorestframework-jsonp==1.0.2 - # via -r dependencies/pip/requirements.in -djangorestframework-xml==2.0.0 - # via -r dependencies/pip/requirements.in -dpath==2.0.6 - # via -r dependencies/pip/requirements.in -ecdsa==0.17.0 - # via tlslite-ng -elaphe3==0.2.0 - # via -r dependencies/pip/requirements.in -et-xmlfile==1.1.0 - # via openpyxl -gdata-python3==3.0.1 - # via -r dependencies/pip/requirements.in -idna==3.3 - # via requests -jmespath==1.0.0 - # via - # boto3 - # botocore -jsonfield==3.1.0 - # via -r dependencies/pip/requirements.in -jwcrypto==1.0 - # via django-oauth-toolkit -kombu==5.2.4 - # via celery -lxml==4.8.0 - # via - # -r dependencies/pip/requirements.in - # gdata-python3 -markdown==3.3.6 - # via -r dependencies/pip/requirements.in -modilabs-python-utils==0.1.5 - # via -r dependencies/pip/requirements.in -numpy==1.22.3 - # via pandas -oauthlib==3.2.0 - # via django-oauth-toolkit -openpyxl==3.0.9 - # via - # -r dependencies/pip/requirements.in - # pyxform -packaging==21.3 - # via redis -pandas==1.4.2 - # via -r dependencies/pip/requirements.in -pillow==9.1.0 - # via - # -r dependencies/pip/requirements.in - # elaphe3 -prompt-toolkit==3.0.29 - # via click-repl -psycopg2==2.9.3 - # via -r dependencies/pip/requirements.in -pycparser==2.21 - # via cffi -pymongo==3.12.3 - # via -r dependencies/pip/requirements.in -pyparsing==3.0.8 - # via packaging -python-crontab==2.6.0 - # via django-celery-beat -python-dateutil==2.8.2 - # via - # botocore - # pandas - # python-crontab -pytz==2022.1 - # via - # celery - # django - # django-timezone-field - # djangorestframework - # pandas -pyxform==1.9.0 - # via -r dependencies/pip/requirements.in -redis==4.2.2 - # via - # -r dependencies/pip/requirements.in - # celery - # django-redis-sessions -requests==2.27.1 - # via django-oauth-toolkit -s3transfer==0.5.2 - # via boto3 -savreaderwriter @ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip - # via -r dependencies/pip/requirements.in -sentry-sdk==1.5.10 - # via -r dependencies/pip/requirements.in -simplejson==3.17.6 - # via -r dependencies/pip/requirements.in -six==1.16.0 - # via - # click-repl - # djangorestframework-csv - # ecdsa - # python-dateutil -sqlparse==0.4.2 - # via django -tlslite-ng==0.7.6 - # via gdata-python3 -unicodecsv==0.14.1 - # via djangorestframework-csv -urllib3==1.26.9 - # via - # botocore - # requests - # sentry-sdk -uwsgi==2.0.20 - # via -r dependencies/pip/prod.in -vine==5.0.0 - # via - # amqp - # celery - # kombu -wcwidth==0.2.5 - # via prompt-toolkit -werkzeug==2.0.3 - # via -r dependencies/pip/requirements.in -wrapt==1.14.0 - # via deprecated -xlrd==2.0.1 - # via pyxform -xlwt==1.3.0 - # via -r dependencies/pip/requirements.in diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 676495e09..f1fa76880 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -87,3 +87,6 @@ sentry-sdk # mimetype detection Werkzeug<=2.0.3 + +# WSGI server +uWSGI diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 228055595..f5f0c33e5 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -233,6 +233,8 @@ urllib3==1.26.9 # botocore # requests # sentry-sdk +uwsgi==2.0.20 + # via -r dependencies/pip/requirements.in vine==5.0.0 # via # amqp diff --git a/docker/init.bash b/docker/init.bash index f9381b1a5..5e4ef8989 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -17,18 +17,18 @@ fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then # `diff` returns exit code 1 if it finds a difference between the files - if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/requirements.txt" "${TMP_DIR}/pip_dependencies.txt" then echo "Syncing production pip dependencies..." - pip-sync dependencies/pip/prod.txt 1>/dev/null - cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + pip-sync dependencies/pip/requirements.txt 1>/dev/null + cp "dependencies/pip/requirements.txt" "${TMP_DIR}/pip_dependencies.txt" fi else - if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/dev_requirements.txt" "${TMP_DIR}/pip_dependencies.txt" then echo "Syncing development pip dependencies..." - pip-sync dependencies/pip/dev.txt 1>/dev/null - cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + pip-sync dependencies/pip/dev_requirements.txt 1>/dev/null + cp "dependencies/pip/dev_requirements.txt" "${TMP_DIR}/pip_dependencies.txt" fi fi From aadaae86f84faa242906e798623c5db51766e9cb Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 16 Aug 2022 09:17:59 -0400 Subject: [PATCH 7/7] Upgrade Django pip requirement --- dependencies/pip/dev_requirements.txt | 2 +- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 551f8934d..4d6216b22 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -81,7 +81,7 @@ deprecated==1.2.13 # redis dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==3.2 +django==3.2.15 # via # -r dependencies/pip/requirements.in # django-celery-beat diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 0bb63e038..b5f11d392 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -13,7 +13,7 @@ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip# -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # Regular PyPI packages -Django==3.2 +Django>=3.2,<3.3 django-csp django-environ django-storages[azure,boto3] diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 0a65e9be2..69f2ea94e 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -71,7 +71,7 @@ deprecated==1.2.13 # redis dict2xml==1.7.1 # via -r dependencies/pip/requirements.in -django==3.2 +django==3.2.15 # via # -r dependencies/pip/requirements.in # django-celery-beat