From 9b40a0bfd114e5a0b05aeffb014a23068f4d705a Mon Sep 17 00:00:00 2001 From: Matthew Macdonald-Wallace Date: Fri, 26 Apr 2024 09:04:45 +0100 Subject: [PATCH 1/5] Add observability via OpenTelemetry Fixes: #256 --- memberportal/membermatters/asgi.py | 42 +++++++++ memberportal/requirements.txt | 140 +++++++++++++++++++++++------ 2 files changed, 154 insertions(+), 28 deletions(-) diff --git a/memberportal/membermatters/asgi.py b/memberportal/membermatters/asgi.py index 1d54714e..416383c0 100644 --- a/memberportal/membermatters/asgi.py +++ b/memberportal/membermatters/asgi.py @@ -1,4 +1,20 @@ import os + +## Tracing +from opentelemetry import trace, context, propagate +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware + +## Logging +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + + from django.conf.urls import url from django.core.asgi import get_asgi_application @@ -9,9 +25,35 @@ from channels.routing import ProtocolTypeRouter, URLRouter from membermatters.websocket_urls import urlpatterns +### OTEL EXPORTER SETUP ### + +resource = Resource( + attributes={ + "service.name": "member_portal_backend", + "deployment.environment": "production", + "service.namespace": "membermatters", + } +) + +#### TRACES +trace.set_tracer_provider(TracerProvider(resource=resource)) +tracer = trace.get_tracer(__name__) + +otlp_trace_exporter = OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces") + +span_processor = BatchSpanProcessor(otlp_trace_exporter) + +trace.get_tracer_provider().add_span_processor(span_processor) + +#### LOGS +logger_provider = LoggerProvider(resource=resource) +otlp_logs_exporter = OTLPLogExporter(endpoint="http://localhost:4318/v1/logs") +logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_logs_exporter)) + application = ProtocolTypeRouter( { "http": django_asgi_app, "websocket": AuthMiddlewareStack(URLRouter(urlpatterns)), } ) +application = OpenTelemetryMiddleware(application) diff --git a/memberportal/requirements.txt b/memberportal/requirements.txt index 3655849d..29445250 100644 --- a/memberportal/requirements.txt +++ b/memberportal/requirements.txt @@ -1,31 +1,115 @@ -django~=3.2.13 -requests~=2.27.1 -stripe==2.74.0 -humanize~=4.1.0 -django-constance~=2.9.0 -django-picklefield~=3.0.1 -django-cors-headers~=3.12.0 +aioredis==1.3.1 +amqp==5.2.0 +arrow==1.3.0 +asgiref==3.8.1 +async-timeout==4.0.3 +attrs==23.2.0 +autobahn==23.6.2 +Automat==22.10.0 +billiard==3.6.4.0 black==24.1.0 +celery==5.2.7 +certifi==2024.2.2 +cffi==1.16.0 +cfgv==3.4.0 +channels==3.0.5 +channels-redis==3.4.1 +charset-normalizer==2.0.12 +click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +constantly==23.10.4 +cryptography==42.0.5 +daphne==3.0.2 +Deprecated==1.2.14 +distlib==0.3.8 +Django==3.2.25 +django-celery-beat==2.2.1 +django-celery-results==2.3.1 +django-constance==2.9.1 +django-cors-headers==3.12.0 +django-oidc-provider==0.8.2 +django-picklefield==3.0.1 +django-prometheus==2.3.1 +django-timezone-field==4.2.3 +django_csp==3.8 +djangorestframework==3.13.1 +djangorestframework-api-key==2.3.0 +djangorestframework-simplejwt==5.3.1 +filelock==3.13.4 +future==1.0.0 +googleapis-common-protos==1.63.0 +grpcio==1.62.2 +hiredis==2.3.2 +humanize==4.1.0 +hyperlink==21.0.0 +ics==0.7.2 +identify==2.5.36 +idna==3.7 +ifaddr==0.2.0 +importlib-metadata==7.0.0 +incremental==22.10.0 +kombu==5.3.7 +mailchimp-marketing @ git+https://github.com/mailchimp/mailchimp-marketing-python.git@3305fa45b3f436767a539c5fba9cb2b0a083d761 +msgpack==1.0.8 +mypy-extensions==1.0.0 +mysqlclient==2.2.4 +nodeenv==1.8.0 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-django==0.45b0 +opentelemetry-instrumentation-wsgi==0.45b0 +opentelemetry-proto==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 +opentelemetry-util-http==0.45b0 +packaging==24.0 +pathspec==0.12.1 +platformdirs==4.2.1 +postmarker==1.0 pre-commit==2.19.0 -djangorestframework~=3.13.1 -djangorestframework-simplejwt>=4.7.2 -django-csp~=3.7 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +protobuf==4.25.3 +psycopg2-binary==2.9.9 +pwned-passwords-django==1.6.1 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pycparser==2.22 +pycryptodomex==3.20.0 +pyjwkest==1.4.2 +PyJWT==2.8.0 +pyOpenSSL==24.1.0 +python-crontab==3.0.0 +python-dateutil==2.9.0.post0 +python-http-client==3.3.7 +pytz==2024.1 +PyYAML==6.0.1 +redis==4.3.6 +requests==2.27.1 sentry-sdk==1.14.0 -mysqlclient -git+https://github.com/mailchimp/mailchimp-marketing-python.git -channels~=3.0.4 -zeroconf~=0.38.6 -postmarker~=1.0 -channels_redis~=3.4.0 -ics~=0.7 -python_http_client~=3.3.7 -pwned-passwords-django~=1.6.0 -celery~=5.2.7 -django-celery-results~=2.3.1 -django-celery-beat~=2.2.1 -redis~=4.3.1 -twilio~=7.9.2 -django-prometheus==2.3.1 -psycopg2-binary~=2.9.6 -django-oidc-provider~=0.8.0 -djangorestframework-api-key==2.* \ No newline at end of file +service-identity==24.1.0 +six==1.16.0 +sqlparse==0.5.0 +stripe==2.74.0 +TatSu==5.12.0 +toml==0.10.2 +twilio==7.9.3 +Twisted==24.3.0 +txaio==23.1.1 +types-python-dateutil==2.9.0.20240316 +typing_extensions==4.11.0 +urllib3==1.26.18 +vine==5.1.0 +virtualenv==20.26.0 +wcwidth==0.2.13 +wrapt==1.16.0 +zeroconf==0.38.7 +zipp==3.18.1 +zope.interface==6.3 From ffde0d1a6f118c7fcbcfc253aac61d51e88193a6 Mon Sep 17 00:00:00 2001 From: Matthew Macdonald-Wallace Date: Fri, 26 Apr 2024 10:57:20 +0100 Subject: [PATCH 2/5] Move locale and timezone to env vars but keep current values as default --- memberportal/membermatters/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/memberportal/membermatters/settings.py b/memberportal/membermatters/settings.py index 01b8d13a..d58cbea6 100644 --- a/memberportal/membermatters/settings.py +++ b/memberportal/membermatters/settings.py @@ -264,9 +264,9 @@ # Internationalization # https://docs.djangoproject.com/en/2.0/topics/i18n/ -LANGUAGE_CODE = "en-au" +LANGUAGE_CODE = os.environ.get("MM_LOCALE", "en-au") -TIME_ZONE = "Australia/Brisbane" +TIME_ZONE = os.environ.get("MM_TZ", "Australia/Brisbane") USE_I18N = True USE_L10N = True USE_TZ = True From dc5d7eb000b0807df9f9f933f48aa4e5d92c5423 Mon Sep 17 00:00:00 2001 From: Matthew Macdonald-Wallace Date: Sat, 27 Apr 2024 06:22:54 +0100 Subject: [PATCH 3/5] A basic trace against the spacedir api view --- memberportal/api_spacedirectory/views.py | 206 ++++++++++++----------- 1 file changed, 106 insertions(+), 100 deletions(-) diff --git a/memberportal/api_spacedirectory/views.py b/memberportal/api_spacedirectory/views.py index 97d4e14a..68df214e 100644 --- a/memberportal/api_spacedirectory/views.py +++ b/memberportal/api_spacedirectory/views.py @@ -6,6 +6,7 @@ from .models import SpaceAPI, SpaceAPISensor, SpaceAPISensorProperties from profile.models import Profile from api_general.models import SiteSession +from otel import tracer import json @@ -15,108 +16,113 @@ class SpaceDirectoryStatus(APIView): permission_classes = (permissions.AllowAny,) def get(self, request): - if not config.ENABLE_SPACE_DIRECTORY: - return Response( - "Space Directory is not enabled on this server.", - status=status.HTTP_404_NOT_FOUND, - ) - - else: - spaceapi = { - "space": config.SITE_OWNER, - "logo": config.SITE_LOGO, - "url": config.MAIN_SITE_URL, - "contact": { - "email": config.SPACE_DIRECTORY_CONTACT_EMAIL, - "twitter": config.SPACE_DIRECTORY_CONTACT_TWITTER, - "phone": config.SPACE_DIRECTORY_CONTACT_PHONE, - "facebook": config.SPACE_DIRECTORY_CONTACT_FACEBOOK, - }, - "spacefed": { - "spacenet": config.SPACE_DIRECTORY_FED_SPACENET, - "spacesaml": config.SPACE_DIRECTORY_FED_SPACESAML, - "spacephone": config.SPACE_DIRECTORY_FED_SPACEPHONE, - }, - "projects": json.loads(config.SPACE_DIRECTORY_PROJECTS), - "issue_report_channels": ["email"], - } - - # Get the default data - spaceapi_data = SpaceAPI.objects.get() - - # Get a list of all the sensors - spaceapi_sensors = SpaceAPISensor.objects.all() - - # Create an empty dict to add the sensor data to - sensor_data = {} - - # Iterate over the sensors and update the dict as appropriate - for sensor in spaceapi_sensors: - sensor_details = {} - # Do we already have a sensor of this type? If not, create it now - if sensor.sensor_type not in sensor_data: - sensor_data[sensor.sensor_type] = [] - - ## Setup the basic details - sensor_details = { - "name": sensor.name, - "description": sensor.description or "", - "location": sensor.location, + with tracer.start_as_current_span("get_spaceapi_details"): + if not config.ENABLE_SPACE_DIRECTORY: + return Response( + "Space Directory is not enabled on this server.", + status=status.HTTP_404_NOT_FOUND, + ) + + else: + spaceapi = { + "space": config.SITE_OWNER, + "logo": config.SITE_LOGO, + "url": config.MAIN_SITE_URL, + "contact": { + "email": config.SPACE_DIRECTORY_CONTACT_EMAIL, + "twitter": config.SPACE_DIRECTORY_CONTACT_TWITTER, + "phone": config.SPACE_DIRECTORY_CONTACT_PHONE, + "facebook": config.SPACE_DIRECTORY_CONTACT_FACEBOOK, + }, + "spacefed": { + "spacenet": config.SPACE_DIRECTORY_FED_SPACENET, + "spacesaml": config.SPACE_DIRECTORY_FED_SPACESAML, + "spacephone": config.SPACE_DIRECTORY_FED_SPACEPHONE, + }, + "projects": json.loads(config.SPACE_DIRECTORY_PROJECTS), + "issue_report_channels": ["email"], } - ### Do we have properties? If so, let's add them - if len(sensor.properties.all()) > 0: - sensor_details["properties"] = {} - for prop in sensor.properties.all(): - properties = { - prop.name: {"value": prop.value, "unit": prop.unit} - } - sensor_details["properties"].update(properties) - else: - sensor_details.update({"value": sensor.value, "unit": sensor.unit}) - - sensor_data[sensor.sensor_type].append(sensor_details) - - ## Add the user count and members on site count to the sensors - spaceapi_user_count = Profile.objects.all().filter(state="active").count() - spaceapi_members_on_site = SiteSession.objects.filter( - signout_date=None - ).order_by("-signin_date") - - sensor_data["total_member_count"] = [{"value": spaceapi_user_count}] - - sensor_data["people_now_present"] = [ - {"value": spaceapi_members_on_site.count()} - ] - - # Is the camera array empty? If not, add them - if not config.SPACE_DIRECTORY_CAMS: - spaceapi["cameras"] = config.SPACE_DIRECTORY_CAMS - - # Set the STATE part of the schema, the icons, and the schema version - spaceapi["state"] = { - "open": spaceapi_data.space_is_open, - "message": spaceapi_data.space_message, - "lastchange": spaceapi_data.status_last_change.timestamp(), - } - spaceapi["icon"] = { - "open": config.SPACE_DIRECTORY_ICON_OPEN, - "closed": config.SPACE_DIRECTORY_ICON_CLOSED, - } - spaceapi["api_compatibility"] = ["14"] - - ## Add the sensor data to the main body of the schema - spaceapi["sensors"] = sensor_data - - ## Add the location data based on the values in Constance - spaceapi["location"] = { - "address": config.SPACE_DIRECTORY_LOCATION_ADDRESS, - "lat": config.SPACE_DIRECTORY_LOCATION_LAT, - "lon": config.SPACE_DIRECTORY_LOCATION_LON, - } - - # Return the JSON document - return Response(spaceapi) + # Get the default data + spaceapi_data = SpaceAPI.objects.get() + + # Get a list of all the sensors + spaceapi_sensors = SpaceAPISensor.objects.all() + + # Create an empty dict to add the sensor data to + sensor_data = {} + + # Iterate over the sensors and update the dict as appropriate + for sensor in spaceapi_sensors: + sensor_details = {} + # Do we already have a sensor of this type? If not, create it now + if sensor.sensor_type not in sensor_data: + sensor_data[sensor.sensor_type] = [] + + ## Setup the basic details + sensor_details = { + "name": sensor.name, + "description": sensor.description or "", + "location": sensor.location, + } + + ### Do we have properties? If so, let's add them + if len(sensor.properties.all()) > 0: + sensor_details["properties"] = {} + for prop in sensor.properties.all(): + properties = { + prop.name: {"value": prop.value, "unit": prop.unit} + } + sensor_details["properties"].update(properties) + else: + sensor_details.update( + {"value": sensor.value, "unit": sensor.unit} + ) + + sensor_data[sensor.sensor_type].append(sensor_details) + + ## Add the user count and members on site count to the sensors + spaceapi_user_count = ( + Profile.objects.all().filter(state="active").count() + ) + spaceapi_members_on_site = SiteSession.objects.filter( + signout_date=None + ).order_by("-signin_date") + + sensor_data["total_member_count"] = [{"value": spaceapi_user_count}] + + sensor_data["people_now_present"] = [ + {"value": spaceapi_members_on_site.count()} + ] + + # Is the camera array empty? If not, add them + if not config.SPACE_DIRECTORY_CAMS: + spaceapi["cameras"] = config.SPACE_DIRECTORY_CAMS + + # Set the STATE part of the schema, the icons, and the schema version + spaceapi["state"] = { + "open": spaceapi_data.space_is_open, + "message": spaceapi_data.space_message, + "lastchange": spaceapi_data.status_last_change.timestamp(), + } + spaceapi["icon"] = { + "open": config.SPACE_DIRECTORY_ICON_OPEN, + "closed": config.SPACE_DIRECTORY_ICON_CLOSED, + } + spaceapi["api_compatibility"] = ["14"] + + ## Add the sensor data to the main body of the schema + spaceapi["sensors"] = sensor_data + + ## Add the location data based on the values in Constance + spaceapi["location"] = { + "address": config.SPACE_DIRECTORY_LOCATION_ADDRESS, + "lat": config.SPACE_DIRECTORY_LOCATION_LAT, + "lon": config.SPACE_DIRECTORY_LOCATION_LON, + } + + # Return the JSON document + return Response(spaceapi) class SpaceDirectoryUpdate(APIView): From 1b1549219267cb5bd809af4cb2aafef519bd03a6 Mon Sep 17 00:00:00 2001 From: Matthew Macdonald-Wallace Date: Sat, 27 Apr 2024 06:23:25 +0100 Subject: [PATCH 4/5] Django basic instrumentation --- memberportal/membermatters/asgi.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/memberportal/membermatters/asgi.py b/memberportal/membermatters/asgi.py index 416383c0..76c297df 100644 --- a/memberportal/membermatters/asgi.py +++ b/memberportal/membermatters/asgi.py @@ -14,10 +14,17 @@ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +## Databases +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +## DJANGO +from opentelemetry.instrumentation.django import DjangoInstrumentor + +# INTERNAL URLS from django.conf.urls import url from django.core.asgi import get_asgi_application + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "membermatters.settings") django_asgi_app = get_asgi_application() @@ -50,6 +57,12 @@ otlp_logs_exporter = OTLPLogExporter(endpoint="http://localhost:4318/v1/logs") logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_logs_exporter)) +#### Database +Psycopg2Instrumentor().instrument(enable_commenter=True, commenter_options={}) + +#### Django +DjangoInstrumentor().instrument(is_sql_commentor_enabled=True) + application = ProtocolTypeRouter( { "http": django_asgi_app, From d1d29cca263260af2ca745038d0fcd84b7fa7365 Mon Sep 17 00:00:00 2001 From: Matthew Macdonald-Wallace Date: Sat, 27 Apr 2024 06:39:00 +0100 Subject: [PATCH 5/5] Move tracing init etc. into dedicated file to follow DRY patterns --- memberportal/membermatters/asgi.py | 50 +----------------------------- memberportal/otel.py | 40 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 49 deletions(-) create mode 100644 memberportal/otel.py diff --git a/memberportal/membermatters/asgi.py b/memberportal/membermatters/asgi.py index 76c297df..a747b6fe 100644 --- a/memberportal/membermatters/asgi.py +++ b/memberportal/membermatters/asgi.py @@ -1,24 +1,6 @@ import os -## Tracing -from opentelemetry import trace, context, propagate -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware - -## Logging -from opentelemetry._logs import set_logger_provider -from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter -from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - -## Databases -from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor - -## DJANGO -from opentelemetry.instrumentation.django import DjangoInstrumentor +from otel import trace, OpenTelemetryMiddleware # INTERNAL URLS from django.conf.urls import url @@ -32,36 +14,6 @@ from channels.routing import ProtocolTypeRouter, URLRouter from membermatters.websocket_urls import urlpatterns -### OTEL EXPORTER SETUP ### - -resource = Resource( - attributes={ - "service.name": "member_portal_backend", - "deployment.environment": "production", - "service.namespace": "membermatters", - } -) - -#### TRACES -trace.set_tracer_provider(TracerProvider(resource=resource)) -tracer = trace.get_tracer(__name__) - -otlp_trace_exporter = OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces") - -span_processor = BatchSpanProcessor(otlp_trace_exporter) - -trace.get_tracer_provider().add_span_processor(span_processor) - -#### LOGS -logger_provider = LoggerProvider(resource=resource) -otlp_logs_exporter = OTLPLogExporter(endpoint="http://localhost:4318/v1/logs") -logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_logs_exporter)) - -#### Database -Psycopg2Instrumentor().instrument(enable_commenter=True, commenter_options={}) - -#### Django -DjangoInstrumentor().instrument(is_sql_commentor_enabled=True) application = ProtocolTypeRouter( { diff --git a/memberportal/otel.py b/memberportal/otel.py new file mode 100644 index 00000000..2a9e9ffa --- /dev/null +++ b/memberportal/otel.py @@ -0,0 +1,40 @@ +import os + +## Tracing +from opentelemetry import trace, context, propagate +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware + +## Logging +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + +### OTEL EXPORTER SETUP ### + +resource = Resource( + attributes={ + "service.name": os.environ.get("MM_OTEL_SVC_NAME", "member_portal_backend"), + "deployment.environment": os.environ.get("MM_ENV", "development"), + "service.namespace": "membermatters", + } +) + +#### TRACES +trace.set_tracer_provider(TracerProvider(resource=resource)) +tracer = trace.get_tracer(__name__) + +otlp_trace_exporter = OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces") + +span_processor = BatchSpanProcessor(otlp_trace_exporter) + +trace.get_tracer_provider().add_span_processor(span_processor) + +#### LOGS +logger_provider = LoggerProvider(resource=resource) +otlp_logs_exporter = OTLPLogExporter(endpoint="http://localhost:4318/v1/logs") +logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_logs_exporter))