Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Hackathon 2024 12 alerting oncall simplified #5334

Draft
wants to merge 14 commits into
base: dev
Choose a base branch
from
7 changes: 6 additions & 1 deletion engine/apps/api/tests/test_alert_receive_channel.py
Original file line number Diff line number Diff line change
Expand Up @@ -2719,7 +2719,12 @@ def test_alert_receive_channel_integration_options_search(
response = client.get(search_url, format="json", **make_user_auth_headers(user, token))
assert response.status_code == status.HTTP_200_OK
returned_choices = [i["display_name"] for i in response.json()]
assert returned_choices == ["Grafana Alerting", "Grafana Legacy Alerting", "(Deprecated) Grafana Alerting"]
assert returned_choices == [
"Grafana Alerting",
"Grafana Legacy Alerting",
"(Deprecated) Grafana Alerting",
"Adaptive Grafana Alerting",
]

search_url = f"{url}?search=notfound"
response = client.get(search_url, format="json", **make_user_auth_headers(user, token))
Expand Down
3 changes: 3 additions & 0 deletions engine/apps/grafana_plugin/helpers/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,9 @@ def validate_grafana_token_format(grafana_token: str) -> bool:
return False
return True

def forward_alert(self, alert_payload):
return self.api_post("api/alertmanager/grafana/api/v2/alerts", alert_payload)


class GcomAPIClient(APIClient):
ACTIVE_INSTANCE_QUERY = "instances?status=active"
Expand Down
48 changes: 48 additions & 0 deletions engine/apps/integrations/mixins/alert_forwarding_mixin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import json
import logging

from django.http import JsonResponse

from apps.alerts.models import AlertReceiveChannel
from apps.grafana_plugin.helpers import GrafanaAPIClient

logger = logging.getLogger(__name__)


class AlertForwardingMixin:
def dispatch(self, *args, **kwargs):
if kwargs.get("integration_type") == "elastalert":
token = str(kwargs["alert_channel_key"])
# TODO: replace with proper caching logic later
alert_receive_channel = AlertReceiveChannel.objects.get(token=token)
organization = alert_receive_channel.organization
if not alert_receive_channel:
return JsonResponse({"error": "Invalid alert receive channel"}, status=400)

forwarded_payload = [
{
"labels": {"alertname": "HighLatency", "service": "my-service", "severity": "critical"},
"annotations": {
"summary": "The service is experiencing unusually high latency.",
"description": "Latency has exceeded 300ms for the last 10 minutes.",
},
"startsAt": "2024-12-09T10:00:00Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://my-service.example.com/metrics",
}
]
try:
data = json.loads(self.body)
# Transform Here

except json.JSONDecodeError:
return JsonResponse({"error": "Invalid JSON"}, status=400)

# Forward
client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
_, status = client.forward_alert(forwarded_payload)
if status["status_code"] != 200:
return JsonResponse({"status": status}, status=status["status_code"])
return JsonResponse({"data": data}, status=200)

return None
70 changes: 62 additions & 8 deletions engine/apps/integrations/tests/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,16 @@ def setup_failing_redis_cache(settings):
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -104,7 +113,16 @@ def test_integration_universal_endpoint(
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -246,7 +264,16 @@ def test_integration_old_grafana_endpoint(
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -280,7 +307,16 @@ def test_integration_universal_endpoint_not_allow_files(
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -383,7 +419,16 @@ def test_integration_grafana_endpoint_without_db_has_alerts(
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -483,7 +528,16 @@ def test_integration_grafana_endpoint_without_cache_has_alerts(
[
arc_type
for arc_type in INTEGRATION_TYPES
if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
if arc_type
not in [
"amazon_sns",
"grafana",
"alertmanager",
"grafana_alerting",
"maintenance",
"adaptive_grafana_alerting",
"elastalert",
]
],
)
@pytest.mark.django_db
Expand Down Expand Up @@ -540,7 +594,7 @@ def test_integration_outdated_cached_model(
)
@pytest.mark.parametrize(
"integration_type",
[arc_type for arc_type in INTEGRATION_TYPES],
[arc_type for arc_type in INTEGRATION_TYPES if arc_type not in ["adaptive_grafana_alerting", "elastalert"]],
)
@pytest.mark.django_db
def test_non_existent_integration_does_not_repeat_access_db(
Expand Down Expand Up @@ -576,7 +630,7 @@ def test_non_existent_integration_does_not_repeat_access_db(
)
@pytest.mark.parametrize(
"integration_type",
[arc_type for arc_type in INTEGRATION_TYPES],
[arc_type for arc_type in INTEGRATION_TYPES if arc_type not in ["adaptive_grafana_alerting", "elastalert"]],
)
@pytest.mark.django_db
def test_deleted_integration_does_not_repeat_access_db(
Expand Down
6 changes: 6 additions & 0 deletions engine/apps/integrations/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from common.api_helpers.optional_slash_router import optional_slash_path

from .views import (
AdaptiveGrafanaAlertingAPIView,
AlertManagerAPIView,
AmazonSNS,
GrafanaAlertingAPIView,
Expand All @@ -32,6 +33,11 @@
path("grafana_alerting/<str:alert_channel_key>/", GrafanaAlertingAPIView.as_view(), name="grafana_alerting"),
path("alertmanager/<str:alert_channel_key>/", AlertManagerAPIView.as_view(), name="alertmanager"),
path("amazon_sns/<str:alert_channel_key>/", AmazonSNS.as_view(), name="amazon_sns"),
path(
"adaptive_grafana_alerting/<str:alert_channel_key>/",
AdaptiveGrafanaAlertingAPIView.as_view(),
name="adaptive_grafana_alerting",
),
path("<str:integration_type>/<str:alert_channel_key>/", UniversalAPIView.as_view(), name="universal"),
# integration backsync
path("backsync/", IntegrationBacksyncAPIView.as_view(), name="integration_backsync"),
Expand Down
Loading
Loading