From 6a6f489514170cf6b4bfdef3aa552f599642fe0b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 26 Mar 2024 16:23:47 +1100 Subject: [PATCH 01/13] Set public network access to deny and allow azure network access by default --- infra/core/security/keyvault.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/infra/core/security/keyvault.bicep b/infra/core/security/keyvault.bicep index 8795cc3598..09b8ab27ea 100644 --- a/infra/core/security/keyvault.bicep +++ b/infra/core/security/keyvault.bicep @@ -1,6 +1,19 @@ metadata description = 'Creates an Azure Key Vault.' param name string param location string = resourceGroup().location + +// Allow public network access to Key Vault +param allowPublicNetworkAccess bool = false + +// Allow all Azure services to bypass Key Vault network rules +param allowAzureServicesAccess bool = true + +param networkAcls object = { + bypass: allowAzureServicesAccess ? 'AzureServices' : 'None' + defaultAction: allowPublicNetworkAccess ? 'Allow' : 'Deny' + ipRules: [] + virtualNetworkRules: [] +} param tags object = {} param principalId string = '' @@ -14,6 +27,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { sku: { family: 'A', name: 'standard' } enablePurgeProtection: true enableSoftDelete: true + networkAcls: networkAcls accessPolicies: !empty(principalId) ? [ { objectId: principalId From 39ee118fc6c1f5dae641142976f7d8f747729ad6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 27 Mar 2024 19:00:31 +1100 Subject: [PATCH 02/13] OLTP sandbox --- app/backend/app.py | 22 +++++++++----- app/backend/arize_pheonix.py | 14 +++++++++ app/backend/oltp_tracing.py | 29 ++++++++++++++++++ app/backend/requirements.in | 6 ++++ app/backend/requirements.txt | 57 +++++++++++++++++++++++++++--------- 5 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 app/backend/arize_pheonix.py create mode 100644 app/backend/oltp_tracing.py diff --git a/app/backend/app.py b/app/backend/app.py index 971e03f102..f082e8c85a 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -415,14 +415,20 @@ def create_app(): if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): configure_azure_monitor() - # This tracks HTTP requests made by aiohttp: - AioHttpClientInstrumentor().instrument() - # This tracks HTTP requests made by httpx: - HTTPXClientInstrumentor().instrument() - # This tracks OpenAI SDK requests: - OpenAIInstrumentor().instrument() - # This middleware tracks app route requests: - app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[method-assign] + else: + from oltp_tracing import configure_oltp_tracing + configure_oltp_tracing() + + # This tracks HTTP requests made by aiohttp: + AioHttpClientInstrumentor().instrument() + # This tracks HTTP requests made by httpx: + HTTPXClientInstrumentor().instrument() + # This tracks OpenAI SDK requests: + OpenAIInstrumentor().instrument() + # This middleware tracks app route requests: + app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[method-assign] + + # Level should be one of https://docs.python.org/3/library/logging.html#logging-levels default_level = "INFO" # In development, log more verbosely diff --git a/app/backend/arize_pheonix.py b/app/backend/arize_pheonix.py new file mode 100644 index 0000000000..4493cba7c6 --- /dev/null +++ b/app/backend/arize_pheonix.py @@ -0,0 +1,14 @@ +from openinference.instrumentation.openai import OpenAIInstrumentor +from opentelemetry import trace as trace_api +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + +def instrument(host: str = "localhost"): + tracer_provider = trace_sdk.TracerProvider() + span_exporter = OTLPSpanExporter(f"http://{host}:6006/v1/traces") + span_processor = SimpleSpanProcessor(span_exporter) + tracer_provider.add_span_processor(span_processor) + trace_api.set_tracer_provider(tracer_provider) + OpenAIInstrumentor().instrument() diff --git a/app/backend/oltp_tracing.py b/app/backend/oltp_tracing.py new file mode 100644 index 0000000000..b977ad3335 --- /dev/null +++ b/app/backend/oltp_tracing.py @@ -0,0 +1,29 @@ +from opentelemetry.sdk.resources import SERVICE_NAME, Resource + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from opentelemetry import metrics +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + + +def configure_oltp_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4318"): + # Service name is required for most backends + resource = Resource(attributes={ + SERVICE_NAME: service_name + }) + + traceProvider = TracerProvider(resource=resource) + processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + traceProvider.add_span_processor(processor) + trace.set_tracer_provider(traceProvider) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics") + ) + meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(meterProvider) diff --git a/app/backend/requirements.in b/app/backend/requirements.in index 036781703e..d81bf46c13 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -14,9 +14,15 @@ opentelemetry-instrumentation-httpx opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiohttp-client opentelemetry-instrumentation-openai + msal azure-keyvault-secrets cryptography python-jose[cryptography] Pillow types-Pillow + +# Arize Pheonix +opentelemetry-semantic-conventions +opentelemetry-exporter-otlp-proto-http +openinference-instrumentation-openai \ No newline at end of file diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 9837fc6e64..0498f2661d 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -16,7 +16,7 @@ anyio==4.3.0 # via # httpx # openai -asgiref==3.7.2 +asgiref==3.8.1 # via opentelemetry-instrumentation-asgi attrs==23.2.0 # via aiohttp @@ -74,7 +74,9 @@ cryptography==42.0.5 # pyjwt # python-jose deprecated==1.2.14 - # via opentelemetry-api + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-http distro==1.9.0 # via openai ecdsa==0.18.0 @@ -87,6 +89,8 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal +googleapis-common-protos==1.63.0 + # via opentelemetry-exporter-otlp-proto-http h11==0.14.0 # via # httpcore @@ -132,7 +136,7 @@ markupsafe==2.1.5 # jinja2 # quart # werkzeug -msal==1.27.0 +msal==1.28.0 # via # -r requirements.in # azure-identity @@ -152,12 +156,18 @@ numpy==1.26.4 # pandas-stubs oauthlib==3.2.2 # via requests-oauthlib -openai[datalib]==1.13.3 +openai[datalib]==1.14.3 # via -r requirements.in +openinference-instrumentation-openai==0.1.4 + # via -r requirements.in +openinference-semantic-conventions==0.1.5 + # via openinference-instrumentation-openai opentelemetry-api==1.23.0 # via # azure-core-tracing-opentelemetry # azure-monitor-opentelemetry-exporter + # openinference-instrumentation-openai + # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi @@ -173,8 +183,13 @@ opentelemetry-api==1.23.0 # opentelemetry-instrumentation-urllib3 # opentelemetry-instrumentation-wsgi # opentelemetry-sdk +opentelemetry-exporter-otlp-proto-common==1.23.0 + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-http==1.23.0 + # via -r requirements.in opentelemetry-instrumentation==0.44b0 # via + # openinference-instrumentation-openai # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-dbapi @@ -204,7 +219,7 @@ opentelemetry-instrumentation-flask==0.44b0 # via azure-monitor-opentelemetry opentelemetry-instrumentation-httpx==0.44b0 # via -r requirements.in -opentelemetry-instrumentation-openai==0.13.1 +opentelemetry-instrumentation-openai==0.15.0 # via -r requirements.in opentelemetry-instrumentation-psycopg2==0.44b0 # via azure-monitor-opentelemetry @@ -220,14 +235,21 @@ opentelemetry-instrumentation-wsgi==0.44b0 # via # opentelemetry-instrumentation-django # opentelemetry-instrumentation-flask +opentelemetry-proto==1.23.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-http opentelemetry-resource-detector-azure==0.1.3 # via azure-monitor-opentelemetry opentelemetry-sdk==1.23.0 # via # azure-monitor-opentelemetry-exporter + # opentelemetry-exporter-otlp-proto-http # opentelemetry-resource-detector-azure opentelemetry-semantic-conventions==0.44b0 # via + # -r requirements.in + # openinference-instrumentation-openai # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-dbapi @@ -240,7 +262,7 @@ opentelemetry-semantic-conventions==0.44b0 # opentelemetry-instrumentation-urllib3 # opentelemetry-instrumentation-wsgi # opentelemetry-sdk -opentelemetry-semantic-conventions-ai==0.0.20 +opentelemetry-semantic-conventions-ai==0.1.1 # via opentelemetry-instrumentation-openai opentelemetry-util-http==0.44b0 # via @@ -254,13 +276,13 @@ opentelemetry-util-http==0.44b0 # opentelemetry-instrumentation-urllib # opentelemetry-instrumentation-urllib3 # opentelemetry-instrumentation-wsgi -packaging==23.2 +packaging==24.0 # via # msal-extensions # opentelemetry-instrumentation-flask pandas==2.2.1 # via openai -pandas-stubs==2.2.0.240218 +pandas-stubs==2.2.1.240316 # via openai pillow==10.2.0 # via -r requirements.in @@ -268,13 +290,17 @@ portalocker==2.8.2 # via msal-extensions priority==2.0.0 # via hypercorn -pyasn1==0.5.1 +protobuf==4.25.3 + # via + # googleapis-common-protos + # opentelemetry-proto +pyasn1==0.6.0 # via # python-jose # rsa pycparser==2.21 # via cffi -pydantic==2.6.3 +pydantic==2.6.4 # via openai pydantic-core==2.16.3 # via pydantic @@ -299,9 +325,10 @@ requests==2.31.0 # azure-core # msal # msrest + # opentelemetry-exporter-otlp-proto-http # requests-oauthlib # tiktoken -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via msrest rsa==4.9 # via python-jose @@ -322,7 +349,7 @@ tiktoken==0.6.0 # via -r requirements.in tqdm==4.66.2 # via openai -types-pillow==10.2.0.20240213 +types-pillow==10.2.0.20240324 # via -r requirements.in types-pytz==2024.1.0.20240203 # via pandas-stubs @@ -332,6 +359,7 @@ typing-extensions==4.10.0 # azure-keyvault-secrets # azure-storage-blob # openai + # openinference-instrumentation-openai # opentelemetry-sdk # pydantic # pydantic-core @@ -339,7 +367,7 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -uvicorn==0.27.1 +uvicorn==0.29.0 # via -r requirements.in werkzeug==3.0.1 # via @@ -348,6 +376,7 @@ werkzeug==3.0.1 wrapt==1.16.0 # via # deprecated + # openinference-instrumentation-openai # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-dbapi @@ -356,7 +385,7 @@ wsproto==1.2.0 # via hypercorn yarl==1.9.4 # via aiohttp -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 2097e98d4f5c7577e21b4254566660a70f7d1f5e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 4 Apr 2024 15:03:10 +1100 Subject: [PATCH 03/13] OTLP with GRPC and custom metrics --- app/backend/app.py | 4 +- .../approaches/chatreadretrieveread.py | 9 +++- app/backend/oltp_tracing.py | 44 ++++++++++++++++--- app/backend/requirements.in | 4 +- app/backend/requirements.txt | 3 +- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index f082e8c85a..ede7b11d93 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -416,8 +416,8 @@ def create_app(): if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): configure_azure_monitor() else: - from oltp_tracing import configure_oltp_tracing - configure_oltp_tracing() + from oltp_tracing import configure_oltp_grpc_tracing + configure_oltp_grpc_tracing() # This tracks HTTP requests made by aiohttp: AioHttpClientInstrumentor().instrument() diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index a707157fcf..7959411886 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -14,6 +14,8 @@ from core.authentication import AuthenticationHelper from core.modelhelper import get_token_limit +from opentelemetry import metrics + class ChatReadRetrieveReadApproach(ChatApproach): """ @@ -49,6 +51,8 @@ def __init__( self.query_language = query_language self.query_speller = query_speller self.chatgpt_token_limit = get_token_limit(chatgpt_model) + self.meter_ratelimit_remaining_tokens = metrics.get_meter(__name__).create_gauge("openai.ratelimit-remaining-tokens", description="OpenAI tokens remaining in limit") + self.meter_ratelimit_remaining_requests = metrics.get_meter(__name__).create_gauge("openai.ratelimit-remaining-requests", description="OpenAI remaining requests") @property def system_message_chat_conversation(self): @@ -128,7 +132,7 @@ async def run_until_final_call( few_shots=self.query_prompt_few_shots, ) - chat_completion: ChatCompletion = await self.openai_client.chat.completions.create( + chat_completion_response = await self.openai_client.chat.completions.with_raw_response.create( messages=query_messages, # type: ignore # Azure OpenAI takes the deployment name as the model name model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model, @@ -138,6 +142,9 @@ async def run_until_final_call( tools=tools, tool_choice="auto", ) + self.meter_ratelimit_remaining_tokens.set(int(chat_completion_response.headers.get("x-ratelimit-remaining-tokens", 0))) + self.meter_ratelimit_remaining_requests.set(int(chat_completion_response.headers.get("x-ratelimit-remaining-requests", 0))) + chat_completion = chat_completion_response.parse() query_text = self.get_search_query(chat_completion, original_user_query) diff --git a/app/backend/oltp_tracing.py b/app/backend/oltp_tracing.py index b977ad3335..fe51a54e2c 100644 --- a/app/backend/oltp_tracing.py +++ b/app/backend/oltp_tracing.py @@ -1,29 +1,63 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as OTLPHTTPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as OTLPGRPCSpanExporter from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry import metrics -from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHTTPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as OTLPGRPCMetricExporter from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +import httpx -def configure_oltp_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4318"): + +class RequestsHttpxAdapter(httpx.Client): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def post(self, verify=False, *args, **kwargs): + # We don't use verify here. + result = super().post(*args, **kwargs) + result.ok = result.status_code < 400 + return result + + +def configure_oltp_http_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4318"): + # Service name is required for most backends + resource = Resource(attributes={ + SERVICE_NAME: service_name + }) + + traceProvider = TracerProvider(resource=resource) + # Trick OLTP into thinking the httpx client is a requests session so that HTTP2 works + processor = BatchSpanProcessor(OTLPHTTPSpanExporter(endpoint=f"{endpoint}/v1/traces",)) + traceProvider.add_span_processor(processor) + trace.set_tracer_provider(traceProvider) + + reader = PeriodicExportingMetricReader( + OTLPHTTPMetricExporter(endpoint=f"{endpoint}/v1/metrics") + ) + meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(meterProvider) + +def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317"): # Service name is required for most backends resource = Resource(attributes={ SERVICE_NAME: service_name }) traceProvider = TracerProvider(resource=resource) - processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + # Trick OLTP into thinking the httpx client is a requests session so that HTTP2 works + processor = BatchSpanProcessor(OTLPGRPCSpanExporter(endpoint=endpoint, insecure=True)) traceProvider.add_span_processor(processor) trace.set_tracer_provider(traceProvider) reader = PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics") + OTLPGRPCMetricExporter(endpoint=endpoint, insecure=True) ) meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) metrics.set_meter_provider(meterProvider) diff --git a/app/backend/requirements.in b/app/backend/requirements.in index d81bf46c13..65bda781b9 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -25,4 +25,6 @@ types-Pillow # Arize Pheonix opentelemetry-semantic-conventions opentelemetry-exporter-otlp-proto-http -openinference-instrumentation-openai \ No newline at end of file +opentelemetry-exporter-otlp-proto-grpc +# openinference-instrumentation-openai +httpx[http2] \ No newline at end of file diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 0498f2661d..7682487bbd 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -103,7 +103,7 @@ hpack==4.0.0 # via h2 httpcore==1.0.4 # via httpx -httpx==0.27.0 +httpx[http2]==0.27.0 # via openai hypercorn==0.16.0 # via quart @@ -186,6 +186,7 @@ opentelemetry-api==1.23.0 opentelemetry-exporter-otlp-proto-common==1.23.0 # via opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-http==1.23.0 +opentelemetry-exporter-otlp-proto-grpc==1.23.0 # via -r requirements.in opentelemetry-instrumentation==0.44b0 # via From f6a22bbaf7da532fea7aa1b39c3736e6327b8fbf Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 4 Apr 2024 15:47:47 +1100 Subject: [PATCH 04/13] Tidy up experiments --- app/backend/app.py | 32 ++++--- app/backend/arize_pheonix.py | 14 --- .../{oltp_tracing.py => otlp_tracing.py} | 14 +-- app/backend/requirements.in | 9 +- app/backend/requirements.txt | 92 ++++++++++--------- 5 files changed, 70 insertions(+), 91 deletions(-) delete mode 100644 app/backend/arize_pheonix.py rename app/backend/{oltp_tracing.py => otlp_tracing.py} (87%) diff --git a/app/backend/app.py b/app/backend/app.py index ede7b11d93..11e87f31e8 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -413,21 +413,27 @@ def create_app(): app = Quart(__name__) app.register_blueprint(bp) + def instrument_app(): + # This tracks HTTP requests made by aiohttp: + AioHttpClientInstrumentor().instrument() + # This tracks HTTP requests made by httpx: + HTTPXClientInstrumentor().instrument() + # This tracks OpenAI SDK requests: + OpenAIInstrumentor().instrument() + # This middleware tracks app route requests: + app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[method-assign] + if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): configure_azure_monitor() - else: - from oltp_tracing import configure_oltp_grpc_tracing - configure_oltp_grpc_tracing() - - # This tracks HTTP requests made by aiohttp: - AioHttpClientInstrumentor().instrument() - # This tracks HTTP requests made by httpx: - HTTPXClientInstrumentor().instrument() - # This tracks OpenAI SDK requests: - OpenAIInstrumentor().instrument() - # This middleware tracks app route requests: - app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[method-assign] - + instrument_app() + elif os.getenv("OTLP_GRPC_ENDPOINT"): + from app.backend.otlp_tracing import configure_oltp_grpc_tracing + configure_oltp_grpc_tracing(endpoint=os.getenv("OTLP_GRPC_ENDPOINT")) + instrument_app() + elif os.getenv("OTLP_HTTP_ENDPOINT"): + from app.backend.otlp_tracing import configure_oltp_http_tracing + configure_oltp_http_tracing(endpoint=os.getenv("OTLP_HTTP_ENDPOINT")) + instrument_app() # Level should be one of https://docs.python.org/3/library/logging.html#logging-levels diff --git a/app/backend/arize_pheonix.py b/app/backend/arize_pheonix.py deleted file mode 100644 index 4493cba7c6..0000000000 --- a/app/backend/arize_pheonix.py +++ /dev/null @@ -1,14 +0,0 @@ -from openinference.instrumentation.openai import OpenAIInstrumentor -from opentelemetry import trace as trace_api -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk import trace as trace_sdk -from opentelemetry.sdk.trace.export import SimpleSpanProcessor - - -def instrument(host: str = "localhost"): - tracer_provider = trace_sdk.TracerProvider() - span_exporter = OTLPSpanExporter(f"http://{host}:6006/v1/traces") - span_processor = SimpleSpanProcessor(span_exporter) - tracer_provider.add_span_processor(span_processor) - trace_api.set_tracer_provider(tracer_provider) - OpenAIInstrumentor().instrument() diff --git a/app/backend/oltp_tracing.py b/app/backend/otlp_tracing.py similarity index 87% rename from app/backend/oltp_tracing.py rename to app/backend/otlp_tracing.py index fe51a54e2c..c61d1cdc44 100644 --- a/app/backend/oltp_tracing.py +++ b/app/backend/otlp_tracing.py @@ -12,19 +12,6 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader -import httpx - - -class RequestsHttpxAdapter(httpx.Client): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def post(self, verify=False, *args, **kwargs): - # We don't use verify here. - result = super().post(*args, **kwargs) - result.ok = result.status_code < 400 - return result - def configure_oltp_http_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4318"): # Service name is required for most backends @@ -44,6 +31,7 @@ def configure_oltp_http_tracing(service_name: str = "azure-search-openai-demo", meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) metrics.set_meter_provider(meterProvider) + def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317"): # Service name is required for most backends resource = Resource(attributes={ diff --git a/app/backend/requirements.in b/app/backend/requirements.in index 65bda781b9..07fdbfad6e 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -14,6 +14,8 @@ opentelemetry-instrumentation-httpx opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiohttp-client opentelemetry-instrumentation-openai +opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-http msal azure-keyvault-secrets @@ -21,10 +23,3 @@ cryptography python-jose[cryptography] Pillow types-Pillow - -# Arize Pheonix -opentelemetry-semantic-conventions -opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc -# openinference-instrumentation-openai -httpx[http2] \ No newline at end of file diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 7682487bbd..8013d2cfbc 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -76,6 +76,7 @@ cryptography==42.0.5 deprecated==1.2.14 # via # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http distro==1.9.0 # via openai @@ -90,7 +91,11 @@ frozenlist==1.4.1 # aiohttp # aiosignal googleapis-common-protos==1.63.0 - # via opentelemetry-exporter-otlp-proto-http + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.62.1 + # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore @@ -101,9 +106,9 @@ h2==4.1.0 # via hypercorn hpack==4.0.0 # via h2 -httpcore==1.0.4 +httpcore==1.0.5 # via httpx -httpx[http2]==0.27.0 +httpx==0.27.0 # via openai hypercorn==0.16.0 # via quart @@ -115,8 +120,10 @@ idna==3.6 # httpx # requests # yarl -importlib-metadata==6.11.0 - # via opentelemetry-api +importlib-metadata==7.0.0 + # via + # opentelemetry-api + # opentelemetry-instrumentation-flask isodate==0.6.1 # via # azure-keyvault-secrets @@ -156,17 +163,13 @@ numpy==1.26.4 # pandas-stubs oauthlib==3.2.2 # via requests-oauthlib -openai[datalib]==1.14.3 - # via -r requirements.in -openinference-instrumentation-openai==0.1.4 +openai[datalib]==1.16.1 # via -r requirements.in -openinference-semantic-conventions==0.1.5 - # via openinference-instrumentation-openai -opentelemetry-api==1.23.0 +opentelemetry-api==1.24.0 # via # azure-core-tracing-opentelemetry # azure-monitor-opentelemetry-exporter - # openinference-instrumentation-openai + # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client @@ -183,14 +186,16 @@ opentelemetry-api==1.23.0 # opentelemetry-instrumentation-urllib3 # opentelemetry-instrumentation-wsgi # opentelemetry-sdk -opentelemetry-exporter-otlp-proto-common==1.23.0 - # via opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-http==1.23.0 -opentelemetry-exporter-otlp-proto-grpc==1.23.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.24.0 + # via -r requirements.in +opentelemetry-exporter-otlp-proto-http==1.24.0 # via -r requirements.in -opentelemetry-instrumentation==0.44b0 +opentelemetry-instrumentation==0.45b0 # via - # openinference-instrumentation-openai # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-dbapi @@ -204,53 +209,53 @@ opentelemetry-instrumentation==0.44b0 # opentelemetry-instrumentation-urllib # opentelemetry-instrumentation-urllib3 # opentelemetry-instrumentation-wsgi -opentelemetry-instrumentation-aiohttp-client==0.44b0 +opentelemetry-instrumentation-aiohttp-client==0.45b0 # via -r requirements.in -opentelemetry-instrumentation-asgi==0.44b0 +opentelemetry-instrumentation-asgi==0.45b0 # via # -r requirements.in # opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-dbapi==0.44b0 +opentelemetry-instrumentation-dbapi==0.45b0 # via opentelemetry-instrumentation-psycopg2 -opentelemetry-instrumentation-django==0.44b0 +opentelemetry-instrumentation-django==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-fastapi==0.44b0 +opentelemetry-instrumentation-fastapi==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-flask==0.44b0 +opentelemetry-instrumentation-flask==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-httpx==0.44b0 +opentelemetry-instrumentation-httpx==0.45b0 # via -r requirements.in -opentelemetry-instrumentation-openai==0.15.0 +opentelemetry-instrumentation-openai==0.15.9 # via -r requirements.in -opentelemetry-instrumentation-psycopg2==0.44b0 +opentelemetry-instrumentation-psycopg2==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-requests==0.44b0 +opentelemetry-instrumentation-requests==0.45b0 # via # -r requirements.in # azure-monitor-opentelemetry -opentelemetry-instrumentation-urllib==0.44b0 +opentelemetry-instrumentation-urllib==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-urllib3==0.44b0 +opentelemetry-instrumentation-urllib3==0.45b0 # via azure-monitor-opentelemetry -opentelemetry-instrumentation-wsgi==0.44b0 +opentelemetry-instrumentation-wsgi==0.45b0 # via # opentelemetry-instrumentation-django # opentelemetry-instrumentation-flask -opentelemetry-proto==1.23.0 +opentelemetry-proto==1.24.0 # via # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http opentelemetry-resource-detector-azure==0.1.3 # via azure-monitor-opentelemetry -opentelemetry-sdk==1.23.0 +opentelemetry-sdk==1.24.0 # via # azure-monitor-opentelemetry-exporter + # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-resource-detector-azure -opentelemetry-semantic-conventions==0.44b0 +opentelemetry-semantic-conventions==0.45b0 # via - # -r requirements.in - # openinference-instrumentation-openai # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-dbapi @@ -258,6 +263,7 @@ opentelemetry-semantic-conventions==0.44b0 # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-flask # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-openai # opentelemetry-instrumentation-requests # opentelemetry-instrumentation-urllib # opentelemetry-instrumentation-urllib3 @@ -265,7 +271,7 @@ opentelemetry-semantic-conventions==0.44b0 # opentelemetry-sdk opentelemetry-semantic-conventions-ai==0.1.1 # via opentelemetry-instrumentation-openai -opentelemetry-util-http==0.44b0 +opentelemetry-util-http==0.45b0 # via # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-asgi @@ -285,7 +291,7 @@ pandas==2.2.1 # via openai pandas-stubs==2.2.1.240316 # via openai -pillow==10.2.0 +pillow==10.3.0 # via -r requirements.in portalocker==2.8.2 # via msal-extensions @@ -299,7 +305,7 @@ pyasn1==0.6.0 # via # python-jose # rsa -pycparser==2.21 +pycparser==2.22 # via cffi pydantic==2.6.4 # via openai @@ -313,7 +319,7 @@ python-jose[cryptography]==3.3.0 # via -r requirements.in pytz==2024.1 # via pandas -quart==0.19.4 +quart==0.19.5 # via # -r requirements.in # quart-cors @@ -350,7 +356,7 @@ tiktoken==0.6.0 # via -r requirements.in tqdm==4.66.2 # via openai -types-pillow==10.2.0.20240324 +types-pillow==10.2.0.20240331 # via -r requirements.in types-pytz==2024.1.0.20240203 # via pandas-stubs @@ -360,7 +366,6 @@ typing-extensions==4.10.0 # azure-keyvault-secrets # azure-storage-blob # openai - # openinference-instrumentation-openai # opentelemetry-sdk # pydantic # pydantic-core @@ -370,14 +375,13 @@ urllib3==2.2.1 # via requests uvicorn==0.29.0 # via -r requirements.in -werkzeug==3.0.1 +werkzeug==3.0.2 # via # flask # quart wrapt==1.16.0 # via # deprecated - # openinference-instrumentation-openai # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-dbapi From 950279a02008527b19e04cb85a839de666fdfaac Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 30 Apr 2024 09:55:28 +1000 Subject: [PATCH 05/13] Add logging support --- app/backend/app.py | 4 +-- app/backend/otlp_tracing.py | 59 ++++++++++++++++++++++--------------- docs/opentelemetry.md | 21 +++++++++++++ 3 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 docs/opentelemetry.md diff --git a/app/backend/app.py b/app/backend/app.py index 11e87f31e8..87a09122a7 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -427,11 +427,11 @@ def instrument_app(): configure_azure_monitor() instrument_app() elif os.getenv("OTLP_GRPC_ENDPOINT"): - from app.backend.otlp_tracing import configure_oltp_grpc_tracing + from otlp_tracing import configure_oltp_grpc_tracing configure_oltp_grpc_tracing(endpoint=os.getenv("OTLP_GRPC_ENDPOINT")) instrument_app() elif os.getenv("OTLP_HTTP_ENDPOINT"): - from app.backend.otlp_tracing import configure_oltp_http_tracing + from otlp_tracing import configure_oltp_http_tracing configure_oltp_http_tracing(endpoint=os.getenv("OTLP_HTTP_ENDPOINT")) instrument_app() diff --git a/app/backend/otlp_tracing.py b/app/backend/otlp_tracing.py index c61d1cdc44..0dd8dfc12e 100644 --- a/app/backend/otlp_tracing.py +++ b/app/backend/otlp_tracing.py @@ -1,51 +1,62 @@ +import logging + from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as OTLPHTTPSpanExporter -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as OTLPGRPCSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry import metrics -from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHTTPMetricExporter -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as OTLPGRPCMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +# Logging (Experimental) +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.resources import Resource + -def configure_oltp_http_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4318"): +def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317", insecure=True, api_key=None): # Service name is required for most backends resource = Resource(attributes={ SERVICE_NAME: service_name }) + if api_key: + headers = { + "x-otlp-api-key": api_key + } + else: + headers = None + + # Configure Tracing traceProvider = TracerProvider(resource=resource) - # Trick OLTP into thinking the httpx client is a requests session so that HTTP2 works - processor = BatchSpanProcessor(OTLPHTTPSpanExporter(endpoint=f"{endpoint}/v1/traces",)) + processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=insecure, headers=headers)) traceProvider.add_span_processor(processor) trace.set_tracer_provider(traceProvider) + # Configure Metrics reader = PeriodicExportingMetricReader( - OTLPHTTPMetricExporter(endpoint=f"{endpoint}/v1/metrics") + OTLPMetricExporter(endpoint=endpoint, insecure=insecure, headers=headers) ) meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) metrics.set_meter_provider(meterProvider) + # Configure Logging + logger_provider = LoggerProvider( + resource=resource + ) + set_logger_provider(logger_provider) -def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317"): - # Service name is required for most backends - resource = Resource(attributes={ - SERVICE_NAME: service_name - }) - - traceProvider = TracerProvider(resource=resource) - # Trick OLTP into thinking the httpx client is a requests session so that HTTP2 works - processor = BatchSpanProcessor(OTLPGRPCSpanExporter(endpoint=endpoint, insecure=True)) - traceProvider.add_span_processor(processor) - trace.set_tracer_provider(traceProvider) + exporter = OTLPLogExporter(endpoint=endpoint, insecure=insecure, headers=headers) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) - reader = PeriodicExportingMetricReader( - OTLPGRPCMetricExporter(endpoint=endpoint, insecure=True) - ) - meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) - metrics.set_meter_provider(meterProvider) + # Attach OTLP handler to root logger + logging.getLogger().addHandler(handler) diff --git a/docs/opentelemetry.md b/docs/opentelemetry.md new file mode 100644 index 0000000000..96a4ee3a8f --- /dev/null +++ b/docs/opentelemetry.md @@ -0,0 +1,21 @@ + +## Starting the Aspire Dashboard + +The .NET Aspire dashboard is an OpenTelemetry dashboard service that can be run locally using Docker. The dashboard is a web application that can be accessed at `http://localhost:18888`. + +You can set a temporary key for OTLP API as an environment variable when starting the container. The key is used to verify that incoming data is from a trusted source. The key is used to authenticate the data source and is not used to authenticate users accessing the dashboard. + +```console +set OTLP_KEY=f142d227-486e-4e80-b7bd-3446e6aa8ea1 # Your own unique key +docker run --rm -it -p 18888:18888 -p 4317:18889 --name aspire-dashboard \ + -e DASHBOARD__OTLP__AUTHMODE='ApiKey' \ + -e DASHBOARD__OTLP__PRIMARYAPIKEY='${OTLP_KEY}' \ + mcr.microsoft.com/dotnet/nightly/aspire-dashboard:8.0-preview +``` + +Once you have started the container, look at the output for a link to the dashboard, the service generates a unique sign-in key that you can use to access the dashboard and prints this to stdout. + +## Starting the service with OpenTelemetry + +To send data to the Aspire dashboard, you need to configure your application to send data to the OpenTelemetry collector. The collector is a service that receives telemetry data from your application and forwards it to the dashboard. + From 5d2a34b5bca3ed58abef4af9ee4654195ffc5820 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 30 Apr 2024 10:00:41 +1000 Subject: [PATCH 06/13] Clean up and formatting --- app/backend/app.py | 14 +++++++------- app/backend/otlp_tracing.py | 20 ++++++++------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 3b37dd81b5..26f5f7bbf3 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -574,15 +574,15 @@ def instrument_app(): if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): configure_azure_monitor() instrument_app() - elif os.getenv("OTLP_GRPC_ENDPOINT"): + elif os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"): from otlp_tracing import configure_oltp_grpc_tracing - configure_oltp_grpc_tracing(endpoint=os.getenv("OTLP_GRPC_ENDPOINT")) - instrument_app() - elif os.getenv("OTLP_HTTP_ENDPOINT"): - from otlp_tracing import configure_oltp_http_tracing - configure_oltp_http_tracing(endpoint=os.getenv("OTLP_HTTP_ENDPOINT")) - instrument_app() + configure_oltp_grpc_tracing( + service_name=os.getenv("OTEL_SERVICE_NAME", "azure-search-openai-demo"), + endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), + api_key=os.getenv("OTEL_EXPORTER_OTLP_TRACES_API_KEY"), + ) + instrument_app() # Level should be one of https://docs.python.org/3/library/logging.html#logging-levels default_level = "INFO" # In development, log more verbosely diff --git a/app/backend/otlp_tracing.py b/app/backend/otlp_tracing.py index 0dd8dfc12e..6af45047cb 100644 --- a/app/backend/otlp_tracing.py +++ b/app/backend/otlp_tracing.py @@ -1,25 +1,21 @@ import logging -from opentelemetry.sdk.resources import SERVICE_NAME, Resource - -from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor - -from opentelemetry import metrics -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry import metrics, trace # Logging (Experimental) from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( OTLPLogExporter, ) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317", insecure=True, api_key=None): From 0804dd9fb1335880067b6d6ba4ec40a165080ad1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 30 Apr 2024 11:12:29 +1000 Subject: [PATCH 07/13] Add docs on OTEL, sort imports --- app/backend/app.py | 1 + .../approaches/chatreadretrieveread.py | 3 +-- docs/opentelemetry.md | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 26f5f7bbf3..0f8e4843f8 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -580,6 +580,7 @@ def instrument_app(): configure_oltp_grpc_tracing( service_name=os.getenv("OTEL_SERVICE_NAME", "azure-search-openai-demo"), endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), + insecure=os.getenv("OTEL_EXPORTER_OTLP_TRACES_INSECURE", "true").lower() == "true", api_key=os.getenv("OTEL_EXPORTER_OTLP_TRACES_API_KEY"), ) instrument_app() diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 8071a27b29..372d5f4797 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -8,14 +8,13 @@ ChatCompletionChunk, ChatCompletionToolParam, ) +from opentelemetry import metrics from approaches.approach import ThoughtStep from approaches.chatapproach import ChatApproach from core.authentication import AuthenticationHelper from core.modelhelper import get_token_limit -from opentelemetry import metrics - class ChatReadRetrieveReadApproach(ChatApproach): """ diff --git a/docs/opentelemetry.md b/docs/opentelemetry.md index 96a4ee3a8f..d80ce5e289 100644 --- a/docs/opentelemetry.md +++ b/docs/opentelemetry.md @@ -1,12 +1,18 @@ +# OpenTelemetry support -## Starting the Aspire Dashboard +This project has instrumentation for OpenTelemetry. The OpenTelemetry project provides a single set of APIs, libraries, agents, and instrumentation resources to capture distributed traces and metrics from your application. There are two supported options for viewing traces emitted by this application: -The .NET Aspire dashboard is an OpenTelemetry dashboard service that can be run locally using Docker. The dashboard is a web application that can be accessed at `http://localhost:18888`. +1. Locally using [the .NET Aspire Dashboard](#starting-the-aspire-dashboard) +2. Remotely using [Azure Monitor and Application Insights](../README.md#monitoring-with-application-insights) + +## Starting the .NET Aspire Dashboard + +The .NET Aspire dashboard is an OpenTelemetry dashboard service that can be run locally using Docker. The dashboard is a web application that can be accessed at `http://localhost:18888`. The .NET Aspire Dashboard is designed for local development and testing. Once the container is stopped, any traces, logs and metrics will be destroyed. For persistent logging, see the [Azure Monitor and Application Insights](../README.md#monitoring-with-application-insights) integration. You can set a temporary key for OTLP API as an environment variable when starting the container. The key is used to verify that incoming data is from a trusted source. The key is used to authenticate the data source and is not used to authenticate users accessing the dashboard. ```console -set OTLP_KEY=f142d227-486e-4e80-b7bd-3446e6aa8ea1 # Your own unique key +export OTLP_KEY=f142d227-486e-4e80-b7bd-3446e6aa8ea1 # Your own unique key docker run --rm -it -p 18888:18888 -p 4317:18889 --name aspire-dashboard \ -e DASHBOARD__OTLP__AUTHMODE='ApiKey' \ -e DASHBOARD__OTLP__PRIMARYAPIKEY='${OTLP_KEY}' \ @@ -19,3 +25,10 @@ Once you have started the container, look at the output for a link to the dashbo To send data to the Aspire dashboard, you need to configure your application to send data to the OpenTelemetry collector. The collector is a service that receives telemetry data from your application and forwards it to the dashboard. +From the `app/` directory, you can start the service with the following command and additional environment variables: + +```console +$ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 OTEL_EXPORTER_OTLP_TRACES_API_KEY=${OTLP_KEY} ./start.sh +``` + +This will launch the web services and connect all tracing to the OTLP endpoint running in the dashboard container. From 1d61425d0ee2c72c89120f9dff416df53b680d20 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 30 Apr 2024 11:25:22 +1000 Subject: [PATCH 08/13] Update default args for OTLP configuration --- app/backend/otlp_tracing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/otlp_tracing.py b/app/backend/otlp_tracing.py index 6af45047cb..40c007baa7 100644 --- a/app/backend/otlp_tracing.py +++ b/app/backend/otlp_tracing.py @@ -18,7 +18,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor -def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint: str = "http://localhost:4317", insecure=True, api_key=None): +def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint=None, insecure=True, api_key=None): # Service name is required for most backends resource = Resource(attributes={ SERVICE_NAME: service_name From 70fa2ef200a7bcc7d694d02c43bd51d4697d6ffb Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 30 Apr 2024 11:34:32 +1000 Subject: [PATCH 09/13] Reformat using new black --- .../approaches/chatreadretrieveread.py | 16 +++++++++++---- app/backend/otlp_tracing.py | 20 +++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 372d5f4797..d182df224a 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -52,8 +52,12 @@ def __init__( self.query_language = query_language self.query_speller = query_speller self.chatgpt_token_limit = get_token_limit(chatgpt_model) - self.meter_ratelimit_remaining_tokens = metrics.get_meter(__name__).create_gauge("openai.ratelimit-remaining-tokens", description="OpenAI tokens remaining in limit") - self.meter_ratelimit_remaining_requests = metrics.get_meter(__name__).create_gauge("openai.ratelimit-remaining-requests", description="OpenAI remaining requests") + self.meter_ratelimit_remaining_tokens = metrics.get_meter(__name__).create_gauge( + "openai.ratelimit-remaining-tokens", description="OpenAI tokens remaining in limit" + ) + self.meter_ratelimit_remaining_requests = metrics.get_meter(__name__).create_gauge( + "openai.ratelimit-remaining-requests", description="OpenAI remaining requests" + ) @property def system_message_chat_conversation(self): @@ -143,8 +147,12 @@ async def run_until_final_call( tools=tools, tool_choice="auto", ) - self.meter_ratelimit_remaining_tokens.set(int(chat_completion_response.headers.get("x-ratelimit-remaining-tokens", 0))) - self.meter_ratelimit_remaining_requests.set(int(chat_completion_response.headers.get("x-ratelimit-remaining-requests", 0))) + self.meter_ratelimit_remaining_tokens.set( + int(chat_completion_response.headers.get("x-ratelimit-remaining-tokens", 0)) + ) + self.meter_ratelimit_remaining_requests.set( + int(chat_completion_response.headers.get("x-ratelimit-remaining-requests", 0)) + ) chat_completion = chat_completion_response.parse() query_text = self.get_search_query(chat_completion, original_user_query) diff --git a/app/backend/otlp_tracing.py b/app/backend/otlp_tracing.py index 40c007baa7..b481338b22 100644 --- a/app/backend/otlp_tracing.py +++ b/app/backend/otlp_tracing.py @@ -18,16 +18,14 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor -def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", endpoint=None, insecure=True, api_key=None): +def configure_oltp_grpc_tracing( + service_name: str = "azure-search-openai-demo", endpoint=None, insecure=True, api_key=None +): # Service name is required for most backends - resource = Resource(attributes={ - SERVICE_NAME: service_name - }) + resource = Resource(attributes={SERVICE_NAME: service_name}) if api_key: - headers = { - "x-otlp-api-key": api_key - } + headers = {"x-otlp-api-key": api_key} else: headers = None @@ -38,16 +36,12 @@ def configure_oltp_grpc_tracing(service_name: str = "azure-search-openai-demo", trace.set_tracer_provider(traceProvider) # Configure Metrics - reader = PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=endpoint, insecure=insecure, headers=headers) - ) + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint, insecure=insecure, headers=headers)) meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) metrics.set_meter_provider(meterProvider) # Configure Logging - logger_provider = LoggerProvider( - resource=resource - ) + logger_provider = LoggerProvider(resource=resource) set_logger_provider(logger_provider) exporter = OTLPLogExporter(endpoint=endpoint, insecure=insecure, headers=headers) From 9319a6e8eb79ae92be1de38a3f8a2fab8d4e8cc6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 1 May 2024 15:16:38 +1000 Subject: [PATCH 10/13] Link documents --- docs/localdev.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/localdev.md b/docs/localdev.md index 252cca0a8c..bcddbc53ff 100644 --- a/docs/localdev.md +++ b/docs/localdev.md @@ -65,3 +65,7 @@ If you're running inside a dev container, use this local URL instead: ```shell azd env set OPENAI_BASE_URL http://host.docker.internal:8080/v1 ``` + +## (Optional) Running with OpenTelemetry tracing + +To run the service with OpenTelemetry tracing, you can use the local Aspire Dashboard, see [OpenTelemetry support](../opentelemetry.md) for details on starting the dashboard. \ No newline at end of file From caac590c3bae3bebaec4d59014ccd2b5c1da9720 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 May 2024 09:52:48 +1000 Subject: [PATCH 11/13] Don't use metrics for now --- app/backend/approaches/chatreadretrieveread.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index d182df224a..f7159a453a 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -8,7 +8,6 @@ ChatCompletionChunk, ChatCompletionToolParam, ) -from opentelemetry import metrics from approaches.approach import ThoughtStep from approaches.chatapproach import ChatApproach @@ -52,12 +51,6 @@ def __init__( self.query_language = query_language self.query_speller = query_speller self.chatgpt_token_limit = get_token_limit(chatgpt_model) - self.meter_ratelimit_remaining_tokens = metrics.get_meter(__name__).create_gauge( - "openai.ratelimit-remaining-tokens", description="OpenAI tokens remaining in limit" - ) - self.meter_ratelimit_remaining_requests = metrics.get_meter(__name__).create_gauge( - "openai.ratelimit-remaining-requests", description="OpenAI remaining requests" - ) @property def system_message_chat_conversation(self): @@ -137,7 +130,7 @@ async def run_until_final_call( few_shots=self.query_prompt_few_shots, ) - chat_completion_response = await self.openai_client.chat.completions.with_raw_response.create( + chat_completion: ChatCompletion = await self.openai_client.chat.completions.create( messages=query_messages, # type: ignore # Azure OpenAI takes the deployment name as the model name model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model, @@ -147,13 +140,6 @@ async def run_until_final_call( tools=tools, tool_choice="auto", ) - self.meter_ratelimit_remaining_tokens.set( - int(chat_completion_response.headers.get("x-ratelimit-remaining-tokens", 0)) - ) - self.meter_ratelimit_remaining_requests.set( - int(chat_completion_response.headers.get("x-ratelimit-remaining-requests", 0)) - ) - chat_completion = chat_completion_response.parse() query_text = self.get_search_query(chat_completion, original_user_query) From 0661f48618164ca4b85bc7e861fc3a8040826a7a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 May 2024 09:54:09 +1000 Subject: [PATCH 12/13] rollback keyvault changes --- infra/core/security/keyvault.bicep | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/infra/core/security/keyvault.bicep b/infra/core/security/keyvault.bicep index 09b8ab27ea..8795cc3598 100644 --- a/infra/core/security/keyvault.bicep +++ b/infra/core/security/keyvault.bicep @@ -1,19 +1,6 @@ metadata description = 'Creates an Azure Key Vault.' param name string param location string = resourceGroup().location - -// Allow public network access to Key Vault -param allowPublicNetworkAccess bool = false - -// Allow all Azure services to bypass Key Vault network rules -param allowAzureServicesAccess bool = true - -param networkAcls object = { - bypass: allowAzureServicesAccess ? 'AzureServices' : 'None' - defaultAction: allowPublicNetworkAccess ? 'Allow' : 'Deny' - ipRules: [] - virtualNetworkRules: [] -} param tags object = {} param principalId string = '' @@ -27,7 +14,6 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { sku: { family: 'A', name: 'standard' } enablePurgeProtection: true enableSoftDelete: true - networkAcls: networkAcls accessPolicies: !empty(principalId) ? [ { objectId: principalId From 0ee02a0e65d8a43e9b5d1b3f61fff81be66d782f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 May 2024 09:55:37 +1000 Subject: [PATCH 13/13] Fix markdown reference --- docs/localdev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/localdev.md b/docs/localdev.md index bcddbc53ff..5657e50df3 100644 --- a/docs/localdev.md +++ b/docs/localdev.md @@ -68,4 +68,4 @@ azd env set OPENAI_BASE_URL http://host.docker.internal:8080/v1 ## (Optional) Running with OpenTelemetry tracing -To run the service with OpenTelemetry tracing, you can use the local Aspire Dashboard, see [OpenTelemetry support](../opentelemetry.md) for details on starting the dashboard. \ No newline at end of file +To run the service with OpenTelemetry tracing, you can use the local Aspire Dashboard, see [OpenTelemetry support](opentelemetry.md) for details on starting the dashboard. \ No newline at end of file