[COST-4514] Upgrade to django 4.0 (#4839)

* update django to 4.0 and update relevant method names

* update pipfile.lock and remove gettext alias

* mock get_response

* more mocks for get_response required by middlewaremixin

* use method_decorator to apply neevr_cache on class method

* method set_context deprecated in DRF 3.12 in favor of requires_context

* user django RedisCache and remove django_redis pkg

Pipfile

@@ -20,15 +20,14 @@ celery = ">=5.2.2"
 ciso8601 = ">=2.1"
 confluent-kafka = ">=2.1.0"
 croniter = "*"
-Django = "<4.0"
+Django = "<4.1"
 django-cors-headers = ">=3.1"
 django-environ = ">=0.4"
 django-extensions = ">=2.2"
 django-filter = ">=2.2"
 django-prometheus = ">=1.1"
-django-redis = ">=4.10"
 django-tenants = ">=3.4"
-djangorestframework = ">=3.11,!=3.14.0"
+djangorestframework = ">=3.14.0"
 djangorestframework-csv = ">=2.1"
 google-api-python-client = ">=1.12.4"
 google-auth = ">=1.22.1"
@@ -50,6 +49,7 @@ psycopg2 = ">=2.9"
 pyarrow = ">=0.17.1"
 python-dateutil = ">=2.8"
 querystring-parser = ">=1.2"
+redis = ">=5.0.1"
 requests = ">=2.20"
 sentry-sdk = ">=0.13"
 statsmodels = ">=0.12"

koku/api/common/__init__.py

@@ -1,7 +1,7 @@
 # noqa
 from uuid import UUID
 
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 
 RH_IDENTITY_HEADER = "HTTP_X_RH_IDENTITY"
 
@@ -11,7 +11,7 @@
 
 def error_obj(key, message):
     """Create an error object."""
-    return {key: [_(message)]}
+    return {key: [gettext(message)]}
 
 
 def log_json(tracing_id="", *, msg, context=None, **kwargs):

koku/api/dataexport/syncer/__init__.py

@@ -11,7 +11,7 @@
 from dateutil.rrule import MONTHLY
 from dateutil.rrule import rrule
 from django.conf import settings
-from django.utils.translation import gettext as _
+from django.utils.translation import gettext
 
 from api.provider.models import Provider
 
@@ -85,15 +85,15 @@ def _copy_object(self, s3_destination_bucket, source_object):
             if source_object.storage_class == "GLACIER" and e.response["Error"]["Code"] == "InvalidObjectState":
                 request = {"Days": 2, "GlacierJobParameters": {"Tier": "Standard"}}
                 source_object.restore_object(RestoreRequest=request)
-                LOG.info(_("Glacier Storage restore for %s is in progress."), source_object.key)
+                LOG.info(gettext("Glacier Storage restore for %s is in progress."), source_object.key)
                 raise SyncedFileInColdStorageError(
                     f"Requested file {source_object.key} is currently in AWS Glacier Storage, "
                     f"an request has been made to restore the file."
                 )
             # if object cannot be copied because restore is already in progress raise
             # SyncedFileInColdStorageError and wait a while longer
             elif e.response["Error"]["Code"] == "RestoreAlreadyInProgress":
-                LOG.info(_("Glacier Storage restore for %s is in progress."), source_object.key)
+                LOG.info(gettext("Glacier Storage restore for %s is in progress."), source_object.key)
                 raise SyncedFileInColdStorageError(
                     f"Requested file {source_object.key} has not yet been restored from AWS Glacier Storage."
                 )

koku/api/dataexport/validators.py

@@ -8,14 +8,7 @@
 class DataExportRequestValidator:
     """Validator that ensures date fields are appropriately defined for a new request."""
 
-    def set_context(self, serializer):
-        """
-        Set extra data from the serializer so we can do extra lookup validation.
-
-        This hook is called by the serializer instance prior to the validation call being made.
-        """
-        self.queryset = serializer.context["view"].get_queryset()
-        self.instance = getattr(serializer, "instance", None)
+    requires_context = True
 
     def pending_instance_exists(self, start_date, end_date):
         """Check for a pending or processing instance that matches the requested dates."""
@@ -29,8 +22,13 @@ def pending_instance_exists(self, start_date, end_date):
         )
         return queryset.exists()
 
-    def __call__(self, attrs):
+    def __call__(self, attrs, serializer_field):
         """Enforce validation of all relevant fields."""
+
+        # Set extra data from the serializer to do extra lookup validation.
+        self.queryset = serializer_field.context["view"].get_queryset()
+        self.instance = getattr(serializer_field, "instance", None)
+
         start_date = attrs["start_date"]
         end_date = attrs["end_date"]
         if end_date < start_date:

koku/api/deprecated_settings/view.py

@@ -0,0 +1,53 @@
+#
+# Copyright 2021 Red Hat Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+"""View for Settings."""
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext
+from django.views.decorators.cache import never_cache
+from rest_framework.response import Response
+from rest_framework.serializers import ValidationError
+from rest_framework.views import APIView
+
+from api.common.permissions.settings_access import SettingsAccessPermission
+from api.deprecated_settings.settings import Settings
+from api.utils import DateHelper
+
+SETTINGS_GENERATORS = {"settings": Settings}
+
+
+class SettingsView(APIView):
+    """
+    View to interact with settings for a customer.
+    """
+
+    permission_classes = [SettingsAccessPermission]
+    deprecation_datetime = DateHelper().create_end_of_life_date(2023, 9, 29)
+    sunset_datetime = DateHelper().create_end_of_life_date(2024, 1, 31)
+    link = "https://github.com/project-koku/koku/pull/4670"
+
+    @method_decorator(never_cache)
+    def get(self, request):
+        """
+        Return a list of all settings.
+        """
+        settings = self._build_settings(request)
+        return Response(settings)
+
+    def post(self, request):
+        """Handle all changed settings."""
+        if not isinstance(request.data, dict):
+            msg = "Invalid input format."
+            raise ValidationError({"details": gettext(msg)})
+        for settings_clazz in SETTINGS_GENERATORS.values():
+            instance = settings_clazz(request)
+            instance.handle_settings(request.data)
+        return Response()
+
+    def _build_settings(self, request):
+        settings = []
+        for settings_clazz in SETTINGS_GENERATORS.values():
+            instance = settings_clazz(request)
+            settings += instance.build_settings()
+        return settings

koku/api/iam/test/iam_test_case.py

@@ -255,12 +255,14 @@ def wrapper(*args, **kwargs):
                 "entitlements": {"cost_management": {"is_entitled": "True"}},
             }
 
+            get_response = Mock()
+
             with override_settings(DEVELOPMENT=True):
                 with override_settings(DEVELOPMENT_IDENTITY=identity):
                     with override_settings(FORCE_HEADER_OVERRIDE=True):
                         with override_settings(MIDDLEWARE=middleware):
                             request_context = IamTestCase._create_request_context(self.customer, user)
-                            middleware = DevelopmentIdentityHeaderMiddleware()
+                            middleware = DevelopmentIdentityHeaderMiddleware(get_response)
                             middleware.process_request(request_context["request"])
                             result = function(*args, **kwargs)
             return result

koku/api/metrics/views.py

@@ -5,7 +5,7 @@
 """Views for CostModelMetricsMap."""
 import copy
 
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from django.views.decorators.vary import vary_on_headers
 from rest_framework import permissions
 from rest_framework import status
@@ -55,4 +55,4 @@ class CostModelMetricMapJSONException(APIException):
     def __init__(self, message):
         """Initialize with status code 500."""
         self.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
-        self.detail = {"detail": force_text(message)}
+        self.detail = {"detail": force_str(message)}

koku/api/query_params.py

@@ -13,7 +13,7 @@
 from django.conf import settings
 from django.core.exceptions import PermissionDenied
 from django.db.models import Q
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 from django_tenants.utils import tenant_context
 from querystring_parser import parser
 from rest_framework.serializers import ValidationError
@@ -188,7 +188,7 @@ def _get_providers(self, provider):
             for p in provider_list:
                 if self.provider_resource_list.get(p) is None:
                     msg = f'Invalid provider "{p}".'
-                    raise ValidationError({"details": _(msg)})
+                    raise ValidationError({"details": gettext(msg)})
                 access.extend(self.provider_resource_list[p])
         return access
 
@@ -619,4 +619,4 @@ def get_tenant(user):
             pass
     if tenant:
         return tenant
-    raise ValidationError({"details": _("Invalid user definition")})
+    raise ValidationError({"details": gettext("Invalid user definition")})

koku/api/report/aws/serializers.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 """AWS Report Serializers."""
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 from rest_framework import serializers
 
 from api.report.constants import AWS_COST_TYPE_CHOICES
@@ -122,7 +122,7 @@ def validate(self, data):
             data["cost_type"] = get_cost_type(self.context.get("request"))
         error = {}
         if "delta" in data.get("order_by", {}) and "delta" not in data:
-            error["order_by"] = _("Cannot order by delta without a delta param")
+            error["order_by"] = gettext("Cannot order by delta without a delta param")
             raise serializers.ValidationError(error)
         return data
 
@@ -162,7 +162,7 @@ def validate_group_by(self, value):
                 # group_by[org_unit_id]=x&group_by[or:org_unit_id]=OU_001 is invalid
                 # If we ever want to change this we need to decide what would be appropriate to see
                 # here.
-                error = {"or_unit_id": _("Multiple org_unit_id must be represented with the or: prefix.")}
+                error = {"or_unit_id": gettext("Multiple org_unit_id must be represented with the or: prefix.")}
                 raise serializers.ValidationError(error)
             key_used = key_used[0]
             request = self.context.get("request")
@@ -172,13 +172,15 @@ def validate_group_by(self, value):
                 # or if we are grouping by org_unit_id with the * since that is essentially grouping by
                 # accounts. If we ever want to change this we need to decide what would be appropriate to see
                 # here. Such as all org units or top level org units
-                error = {"org_unit_id": _("Unsupported parameter or invalid value")}
+                error = {"org_unit_id": gettext("Unsupported parameter or invalid value")}
                 raise serializers.ValidationError(error)
             if "or:" not in key_used:
                 if isinstance(group_by_params.get(key_used), list):
                     if len(group_by_params.get(key_used)) > 1:
                         # group_by[org_unit_id]=x&group_by[org_unit_id]=OU_001 is invalid
                         # because no child nodes would ever intersect due to the tree structure.
-                        error = {"or_unit_id": _("Multiple org_unit_id must be represented with the or: prefix.")}
+                        error = {
+                            "or_unit_id": gettext("Multiple org_unit_id must be represented with the or: prefix.")
+                        }
                         raise serializers.ValidationError(error)
         return value

koku/api/report/oci/serializers.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 """OCI Report Serializers."""
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 from rest_framework import serializers
 
 from api.report.serializers import ExcludeSerializer as BaseExcludeSerializer
@@ -95,6 +95,6 @@ def validate(self, data):
             data["cost_type"] = get_cost_type(self.context.get("request"))
         error = {}
         if "delta" in data.get("order_by", {}) and "delta" not in data:
-            error["order_by"] = _("Cannot order by delta without a delta param")
+            error["order_by"] = gettext("Cannot order by delta without a delta param")
             raise serializers.ValidationError(error)
         return data

koku/api/report/ocp/serializers.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 """OCP Report Serializers."""
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 from rest_framework import serializers
 
 from api.models import Provider
@@ -20,7 +20,7 @@
 def order_by_field_requires_group_by(data, order_name, group_by_key):
     error = {}
     if order_name in data.get("order_by", {}) and group_by_key not in data.get("group_by", {}):
-        error["order_by"] = _(f"Cannot order by field {order_name} without grouping by {group_by_key}.")
+        error["order_by"] = gettext(f"Cannot order by field {order_name} without grouping by {group_by_key}.")
         raise serializers.ValidationError(error)
 
 
@@ -161,15 +161,15 @@ def validate(self, data):
         super().validate(data)
         error = {}
         if "delta" in data.get("order_by", {}) and "delta" not in data:
-            error["order_by"] = _("Cannot order by delta without a delta param")
+            error["order_by"] = gettext("Cannot order by delta without a delta param")
             raise serializers.ValidationError(error)
         order_by_field_requires_group_by(data, DISTRIBUTED_COST_INTERNAL["distributed_cost"], "project")
         order_by_field_requires_group_by(data, "storage_class", "persistentvolumeclaim")
         order_by_field_requires_group_by(data, "persistentvolumeclaim", "persistentvolumeclaim")
         if data.get("delta") == DISTRIBUTED_COST_INTERNAL["distributed_cost"] and "project" not in data.get(
             "group_by", {}
         ):
-            error["delta"] = _("Cannot use distributed_cost delta without grouping by project.")
+            error["delta"] = gettext("Cannot use distributed_cost delta without grouping by project.")
             raise serializers.ValidationError(error)
         return data
 
@@ -202,15 +202,15 @@ def validate_delta(self, value):
         if "__" in value:
             values = value.split("__")
             if len(values) != 2:
-                error[value] = _("Only two fields may be compared")
+                error[value] = gettext("Only two fields may be compared")
                 raise serializers.ValidationError(error)
             for val in values:
                 if val not in self.delta_fields:
-                    error[value] = _("Unsupported parameter")
+                    error[value] = gettext("Unsupported parameter")
                     raise serializers.ValidationError(error)
         else:
             if value not in self.delta_choices:
-                error[value] = _("Unsupported parameter")
+                error[value] = gettext("Unsupported parameter")
                 raise serializers.ValidationError(error)
         return value
 

koku/api/report/serializers.py

@@ -6,7 +6,7 @@
 import copy
 from collections.abc import Mapping
 
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 from rest_framework import serializers
 from rest_framework.fields import DateField
 
@@ -43,7 +43,7 @@ def handle_invalid_fields(this, data):
     if unknown_keys:
         error = {}
         for unknown_key in unknown_keys:
-            error[unknown_key] = _("Unsupported parameter or invalid value")
+            error[unknown_key] = gettext("Unsupported parameter or invalid value")
         raise serializers.ValidationError(error)
     return data
 
@@ -82,7 +82,7 @@ def validate_field(this, field, serializer_cls, value, **kwargs):
     subclasses = serializer_cls.__subclasses__()
     if subclasses and not serializer.is_valid():
         message = "Unsupported parameter or invalid value"
-        error = serializers.ValidationError({field: _(message)})
+        error = serializers.ValidationError({field: gettext(message)})
         for subcls in subclasses:
             for parent in subcls.__bases__:
                 # when using multiple inheritance, the data is valid as long as one
@@ -502,7 +502,7 @@ def validate_order_by(self, value):  # noqa: C901
                 continue  # fields that do not require a group-by
 
             if "or:" in key:
-                error[key] = _(f'The order_by key "{key}" can not contain the or parameter.')
+                error[key] = gettext(f'The order_by key "{key}" can not contain the or parameter.')
                 raise serializers.ValidationError(error)
 
             if "group_by" in self.initial_data:
@@ -537,12 +537,12 @@ def validate_order_by(self, value):  # noqa: C901
                         and value.get("date") <= dh.today.date()
                     ):
                         continue
-                    error[key] = _(
+                    error[key] = gettext(
                         f"Order-by date must be from {materialized_view_month_start(dh).date()} to {dh.today.date()}"
                     )
                     raise serializers.ValidationError(error)
 
-            error[key] = _(f'Order-by "{key}" requires matching Group-by.')
+            error[key] = gettext(f'Order-by "{key}" requires matching Group-by.')
             raise serializers.ValidationError(error)
         validate_field(self, "order_by", self.ORDER_BY_SERIALIZER, value)
         return value

koku/api/settings/views.py

@@ -8,7 +8,7 @@
 from dataclasses import field
 
 from django.utils.decorators import method_decorator
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from django.views.decorators.cache import never_cache
 from rest_framework import permissions
 from rest_framework import status
@@ -39,7 +39,7 @@ class SettingsInvalidFilterException(APIException):
     def __init__(self, message):
         """Initialize with status code 404."""
         self.status_code = status.HTTP_404_NOT_FOUND
-        self.detail = {"detail": force_text(message)}
+        self.detail = {"detail": force_str(message)}
 
 
 @dataclass