[COST-3794] Add new API for managing custom cost groups (#4401)

* Schedule a summary when the cost groups change

    On PUT or DELETE, schedule a summary task to recalculate OCP data for the current month


* Prevent default projects from being removed

    Cache list of default projects to prevent multiple database queries.


* Update the bakery to populate the OCP projects table for unittests
* Add validation for group and project_name fields
* Add some unittests for our new serializer validation.

* Support ordering by multiple values

    This mathches our custom query param syntax with the behavior of Django and DRF.

Co-authored-by: Cody Myers <[email protected]>

koku/api/query_filter.py

@@ -68,6 +68,11 @@ def compose_key(self):
 
     def composed_Q(self):
         """Return a Q object formatted for Django's ORM."""
+        if isinstance(self.parameter, Q):
+            # This will allow us to add Qs directly to
+            # the filter collection
+            return self.parameter
+
         query_dict = {self.composed_query_string(): self.parameter}
         return Q(**query_dict)
 

koku/api/report/test/util/common.py

@@ -8,6 +8,7 @@
 from api.report.test.util.constants import OCP_PLATFORM_NAMESPACE
 from reporting.provider.ocp.models import OCPCluster
 from reporting.provider.ocp.models import OCPNode
+from reporting.provider.ocp.models import OCPProject
 from reporting.provider.ocp.models import OCPUsageLineItemDailySummary
 from reporting.provider.ocp.models import OpenshiftCostCategory
 
@@ -26,6 +27,14 @@ def populate_ocp_topology(schema, provider, cluster_id):
             if node[0]:
                 n = OCPNode(node=node[0], resource_id=node[1], cluster=cluster)
                 n.save()
+        projects = (
+            OCPUsageLineItemDailySummary.objects.filter(cluster_id=cluster_id)
+            .values_list("namespace", flat=True)
+            .distinct()
+        )
+        for project in projects:
+            p = OCPProject(project=project, cluster=cluster)
+            p.save()
 
 
 def update_cost_category(schema):

koku/api/settings/cost_groups/query_handler.py

@@ -0,0 +1,208 @@
+import logging
+from types import MappingProxyType
+
+from django.contrib.postgres.aggregates import ArrayAgg
+from django.db import IntegrityError
+from django.db.models import Case
+from django.db.models import CharField
+from django.db.models import F
+from django.db.models import Q
+from django.db.models import Value
+from django.db.models import When
+
+from api.models import Provider
+from api.query_filter import QueryFilter
+from api.query_filter import QueryFilterCollection
+from api.query_params import QueryParameters
+from api.utils import DateHelper
+from reporting.provider.ocp.models import OCPProject
+from reporting.provider.ocp.models import OpenshiftCostCategory
+from reporting.provider.ocp.models import OpenshiftCostCategoryNamespace
+
+LOG = logging.getLogger(__name__)
+
+
+def _remove_default_projects(projects: list[dict[str, str]]) -> list[dict[str, str]]:
+    try:
+        _remove_default_projects.system_default_namespaces  # type: ignore[attr-defined]
+    except AttributeError:
+        # Cache the system default namespcases
+        _remove_default_projects.system_default_namespaces = OpenshiftCostCategoryNamespace.objects.filter(  # type: ignore[attr-defined]  # noqa: E501
+            system_default=True
+        ).values_list(
+            "namespace", flat=True
+        )
+
+    exact_matches = {
+        project for project in _remove_default_projects.system_default_namespaces if not project.endswith("%")  # type: ignore[attr-defined]  # noqa: E501
+    }
+    prefix_matches = set(_remove_default_projects.system_default_namespaces).difference(exact_matches)  # type: ignore[attr-defined]  # noqa: E501
+
+    scrubbed_projects = []
+    for request in projects:
+        if request["project_name"] in exact_matches:
+            continue
+
+        if any(request["project_name"].startswith(prefix.replace("%", "")) for prefix in prefix_matches):
+            continue
+
+        scrubbed_projects.append(request)
+
+    return scrubbed_projects
+
+
+def put_openshift_namespaces(projects: list[dict[str, str]]) -> list[dict[str, str]]:
+    projects = _remove_default_projects(projects)
+
+    # Build mapping of cost groups to cost category IDs in order to easiy get
+    # the ID of the cost group to update
+    cost_groups = {item["name"]: item["id"] for item in OpenshiftCostCategory.objects.values("name", "id")}
+
+    namespaces_to_create = [
+        OpenshiftCostCategoryNamespace(
+            namespace=new_project["project_name"],
+            system_default=False,
+            cost_category_id=cost_groups[new_project["group"]],
+        )
+        for new_project in projects
+    ]
+    try:
+        # Perform bulk create
+        OpenshiftCostCategoryNamespace.objects.bulk_create(namespaces_to_create)
+    except IntegrityError as e:
+        # Handle IntegrityError (e.g., if a unique constraint is violated)
+        LOG.warning(f"IntegrityError: {e}")
+
+    return projects
+
+
+def delete_openshift_namespaces(projects: list[dict[str, str]]) -> list[dict[str, str]]:
+    projects = _remove_default_projects(projects)
+    projects_to_delete = [item["project_name"] for item in projects]
+    deleted_count, _ = (
+        OpenshiftCostCategoryNamespace.objects.filter(namespace__in=projects_to_delete)
+        .exclude(system_default=True)
+        .delete()
+    )
+    LOG.info(f"Deleted {deleted_count} namespace records from openshift cost groups.")
+
+    return projects
+
+
+class CostGroupsQueryHandler:
+    """Query Handler for the cost groups"""
+
+    provider = Provider.PROVIDER_OCP
+    _filter_map = MappingProxyType(
+        {
+            "group": MappingProxyType({"field": "group", "operation": "icontains"}),
+            "default": MappingProxyType({"field": "default", "operation": "exact"}),
+            "project_name": MappingProxyType({"field": "project_name", "operation": "icontains"}),
+        }
+    )
+
+    def __init__(self, parameters: QueryParameters) -> None:
+        """
+        Args:
+            parameters    (QueryParameters): parameter object for query
+        """
+        self.parameters = parameters
+        self.dh = DateHelper()
+        self.filters = QueryFilterCollection()
+        self.exclusion = QueryFilterCollection()
+        self._default_order_by = ["project_name"]
+
+        self._set_filters_or_exclusion()
+
+    @property
+    def order_by(self) -> list[str]:
+        order_by_params = self.parameters._parameters.get("order_by")
+        if not order_by_params:
+            return self._default_order_by
+
+        result: list[str] = []
+        for key, order in order_by_params.items():
+            if order == "desc":
+                result.insert(0, f"-{key}")
+            else:
+                result.insert(0, f"{key}")
+
+        return result
+
+    def _check_parameters_for_filter_param(self, q_param) -> None:
+        """Populate the query filter collections."""
+        filter_values = self.parameters.get_filter(q_param, list())
+        if filter_values:
+            for item in filter_values if isinstance(filter_values, list) else [filter_values]:
+                q_filter = QueryFilter(parameter=item, **self._filter_map[q_param])
+                self.filters.add(q_filter)
+
+    def _check_parameters_for_exclude_param(self, q_param):
+        """Populate the exclude collections."""
+        exclude_values = self.parameters.get_exclude(q_param, list())
+        if exclude_values:
+            for item in exclude_values if isinstance(exclude_values, list) else [exclude_values]:
+                q_filter = QueryFilter(parameter=item, **self._filter_map[q_param])
+                if q_param in ["group", "default"]:
+                    # .exclude() will remove nulls, so use Q objects directly to include them
+                    q_kwargs = {q_param: item, f"{q_param}__isnull": False}
+                    self.exclusion.add(QueryFilter(parameter=Q(**q_kwargs)))
+                else:
+                    self.exclusion.add(q_filter)
+
+    def _set_filters_or_exclusion(self) -> None:
+        """Populate the query filter and exclusion collections for search filters."""
+        for q_param in self._filter_map:
+            self._check_parameters_for_filter_param(q_param)
+            self._check_parameters_for_exclude_param(q_param)
+        self.exclusion = self.exclusion.compose(logical_operator="or")
+        self.filters = self.filters.compose()
+
+    def _add_worker_unallocated(self, field_name):
+        """Specail handling for the worker unallocated project.
+
+        Worker unallocated is a default project, but it does not currently
+        belong to a cost group.
+        """
+        default_value = Value(None, output_field=CharField())
+        if field_name == "default":
+            default_value = Value(True)
+        return When(project_name="Worker unallocated", then=default_value)
+
+    def build_when_conditions(self, cost_group_projects, field_name):
+        """Builds when conditions given a field name in the cost_group_projects."""
+        # __like is a custom django lookup we added to perform a postgresql LIKE
+        when_conditions = []
+        for project in cost_group_projects:
+            when_conditions.append(When(project_name__like=project["project_name"], then=Value(project[field_name])))
+        when_conditions.append(self._add_worker_unallocated(field_name))
+        return when_conditions
+
+    def execute_query(self):
+        """Executes a query to grab the information we need for the api return."""
+        # This query builds the information we need for our when conditions
+        cost_group_projects = OpenshiftCostCategoryNamespace.objects.annotate(
+            project_name=F("namespace"),
+            default=F("system_default"),
+            group=F("cost_category__name"),
+        ).values("project_name", "default", "group")
+
+        ocp_summary_query = (
+            OCPProject.objects.values(project_name=F("project"))
+            .annotate(
+                group=Case(*self.build_when_conditions(cost_group_projects, "group")),
+                default=Case(*self.build_when_conditions(cost_group_projects, "default")),
+                clusters=ArrayAgg(F("cluster__cluster_alias"), distinct=True),
+            )
+            .values("project_name", "group", "clusters", "default")
+            .distinct()
+        )
+
+        if self.exclusion:
+            ocp_summary_query = ocp_summary_query.exclude(self.exclusion)
+        if self.filters:
+            ocp_summary_query = ocp_summary_query.filter(self.filters)
+
+        ocp_summary_query = ocp_summary_query.order_by(*self.order_by)
+
+        return ocp_summary_query

koku/api/settings/cost_groups/serializers.py

@@ -0,0 +1,81 @@
+#
+# Copyright 2023 Red Hat Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Serializers for settings cost groups."""
+from rest_framework import serializers
+
+from api.report.serializers import ExcludeSerializer
+from api.report.serializers import FilterSerializer
+from api.report.serializers import OrderSerializer
+from api.report.serializers import ReportQueryParamSerializer
+from reporting.provider.ocp.models import OCPProject
+from reporting.provider.ocp.models import OpenshiftCostCategory
+
+
+class CostGroupFilterSerializer(FilterSerializer):
+    """Serializer for Cost Group Settings."""
+
+    project_name = serializers.CharField(required=False)
+    group = serializers.CharField(required=False)
+    default = serializers.BooleanField(required=False)
+
+
+class CostGroupExcludeSerializer(ExcludeSerializer):
+    """Serializer for Cost Group Settings."""
+
+    project_name = serializers.CharField(required=False)
+    group = serializers.CharField(required=False)
+    default = serializers.BooleanField(required=False)
+
+
+class CostGroupOrderSerializer(OrderSerializer):
+    """Serializer for Cost Group Settings."""
+
+    ORDER_CHOICES = (("asc", "asc"), ("desc", "desc"))
+
+    project_name = serializers.ChoiceField(choices=ORDER_CHOICES, required=False)
+    group = serializers.ChoiceField(choices=ORDER_CHOICES, required=False)
+    default = serializers.ChoiceField(choices=ORDER_CHOICES, required=False)
+
+
+class CostGroupQueryParamSerializer(ReportQueryParamSerializer):
+    """Serializer for handling query parameters."""
+
+    FILTER_SERIALIZER = CostGroupFilterSerializer
+    EXCLUDE_SERIALIZER = CostGroupExcludeSerializer
+    ORDER_BY_SERIALIZER = CostGroupOrderSerializer
+
+    order_by_allowlist = frozenset(("project_name", "group", "default"))
+
+
+class CostGroupProjectSerializer(serializers.Serializer):
+    project_name = serializers.CharField()
+    group = serializers.CharField()
+
+    def _is_valid_field_value(self, model, data: str, field_name: str) -> None:
+        """Check that the provided data matches a value in the model field.
+
+        Raises a ValidationError if the data is not found in the model field.
+        """
+
+        valid_values = sorted(model.objects.values_list(field_name, flat=True).distinct())
+        if data not in valid_values:
+            msg = "Select a valid choice"
+            if 0 < len(valid_values) < 7:
+                verb = "Choice is" if len(valid_values) == 1 else "Choices are"
+                msg = f"{msg}. {verb} {', '.join(valid_values)}."
+            else:
+                msg = f"{msg}. '{data}' is not a valid choice."
+
+            raise serializers.ValidationError(msg)
+
+    def validate_project_name(self, data):
+        self._is_valid_field_value(OCPProject, data, "project")
+
+        return data
+
+    def validate_group(self, data):
+        self._is_valid_field_value(OpenshiftCostCategory, data, "name")
+
+        return data