From 414f8b302516154d5758941b15266d63dd3c89e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 19 Jun 2024 11:53:52 -0400 Subject: [PATCH] [Metrics] New components to track HTTP failures and exceptions. (#3928) * New components to track http failures and generic errors. * Emitting metrics signals for every failed request. * Emiting metrics when exceptions are logged. * Cover one more scenario = errorslogger with no exceptions * Fix use of diferrent counters. --- .../Metrics/DefaultFailureMetricHandler.cs | 44 ++++++++++++++ .../Metrics/ExceptionMetricNotification.cs | 16 ++++++ .../Metrics/HttpErrorMetricNotification.cs | 12 ++++ .../Metrics/IExceptionMetricNotification.cs | 16 ++++++ .../Logging/Metrics/IFailureMetricHandler.cs | 14 +++++ .../Metrics/IHttpFailureMetricNotification.cs | 12 ++++ .../BaseExceptionMiddlewareTests.cs | 22 +++++-- .../Extensions/HttpRequestExtension.cs | 45 +++++++++++++++ .../Exceptions/BaseExceptionMiddleware.cs | 24 ++++++++ ...Microsoft.Health.Fhir.Shared.Api.projitems | 1 + .../FhirServerServiceCollectionExtensions.cs | 1 + ...ureMonitorOpenTelemetryLogEnricherTests.cs | 55 +++++++++++++++++- .../AzureMonitorOpenTelemetryLogEnricher.cs | 57 ++++++++++++------- .../Startup.cs | 4 +- 14 files changed, 295 insertions(+), 28 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultFailureMetricHandler.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/ExceptionMetricNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/HttpErrorMetricNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/IExceptionMetricNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/IFailureMetricHandler.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Logging/Metrics/IHttpFailureMetricNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Shared.Api/Extensions/HttpRequestExtension.cs diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultFailureMetricHandler.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultFailureMetricHandler.cs new file mode 100644 index 0000000000..3aa6107fac --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultFailureMetricHandler.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public sealed class DefaultFailureMetricHandler : BaseMeterMetricHandler, IFailureMetricHandler + { + private readonly Counter _counterExceptions; + private readonly Counter _counterHttpFailures; + + public DefaultFailureMetricHandler(IMeterFactory meterFactory) + : base(meterFactory) + { + _counterExceptions = MetricMeter.CreateCounter("Failures.Exceptions"); + _counterHttpFailures = MetricMeter.CreateCounter("Failures.HttpFailures"); + } + + public void EmitException(IExceptionMetricNotification notification) + { + EnsureArg.IsNotNull(notification, nameof(notification)); + + _counterExceptions.Add( + 1, + KeyValuePair.Create("OperationName", notification.OperationName), + KeyValuePair.Create("Severity", notification.Severity), + KeyValuePair.Create("ExceptionType", notification.ExceptionType)); + } + + public void EmitHttpFailure(IHttpFailureMetricNotification notification) + { + EnsureArg.IsNotNull(notification, nameof(notification)); + + _counterHttpFailures.Add( + 1, + KeyValuePair.Create("OperationName", notification.OperationName)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/ExceptionMetricNotification.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/ExceptionMetricNotification.cs new file mode 100644 index 0000000000..d52c4439b0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/ExceptionMetricNotification.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public sealed class ExceptionMetricNotification : IExceptionMetricNotification + { + public string OperationName { get; set; } + + public string ExceptionType { get; set; } + + public string Severity { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/HttpErrorMetricNotification.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/HttpErrorMetricNotification.cs new file mode 100644 index 0000000000..555eff469f --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/HttpErrorMetricNotification.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public sealed class HttpErrorMetricNotification : IHttpFailureMetricNotification + { + public string OperationName { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IExceptionMetricNotification.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IExceptionMetricNotification.cs new file mode 100644 index 0000000000..847c7cc6d1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IExceptionMetricNotification.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public interface IExceptionMetricNotification + { + string OperationName { get; set; } + + string ExceptionType { get; set; } + + string Severity { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IFailureMetricHandler.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IFailureMetricHandler.cs new file mode 100644 index 0000000000..379cef54a0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IFailureMetricHandler.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public interface IFailureMetricHandler + { + void EmitHttpFailure(IHttpFailureMetricNotification notification); + + void EmitException(IExceptionMetricNotification notification); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IHttpFailureMetricNotification.cs b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IHttpFailureMetricNotification.cs new file mode 100644 index 0000000000..55ff4c3a37 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Logging/Metrics/IHttpFailureMetricNotification.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Logging.Metrics +{ + public interface IHttpFailureMetricNotification + { + string OperationName { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Exceptions/BaseExceptionMiddlewareTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Exceptions/BaseExceptionMiddlewareTests.cs index df77f780ac..860a225f5a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Exceptions/BaseExceptionMiddlewareTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Exceptions/BaseExceptionMiddlewareTests.cs @@ -14,6 +14,7 @@ using Microsoft.Health.Fhir.Api.Features.Exceptions; using Microsoft.Health.Fhir.Api.Features.Formatters; using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Logging.Metrics; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; @@ -51,7 +52,9 @@ public BaseExceptionMiddlewareTests() [InlineData("The MetadataAddress or Authority must use HTTPS unless disabled for development by setting RequireHttpsMetadata=false.", "The security configuration requires the authority to be set to an https address.")] public async Task GivenAnHttpContextWithException_WhenExecutingBaseExceptionMiddleware_TheResponseShouldBeOperationOutcome(string exceptionMessage, string diagnosticMessage) { - var baseExceptionMiddleware = CreateBaseExceptionMiddleware(innerHttpContext => throw new Exception(exceptionMessage)); + IFailureMetricHandler failureMetricHandler = Substitute.For(); + + var baseExceptionMiddleware = CreateBaseExceptionMiddleware(innerHttpContext => throw new Exception(exceptionMessage), failureMetricHandler); baseExceptionMiddleware.ExecuteResultAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); @@ -64,23 +67,34 @@ await baseExceptionMiddleware Arg.Is(x => x.StatusCode == HttpStatusCode.InternalServerError && x.Result.Id == _correlationId && x.Result.Issue[0].Diagnostics == diagnosticMessage)); + + failureMetricHandler.Received(1).EmitHttpFailure(Arg.Any()); } [Fact] public async Task GivenAnHttpContextWithNoException_WhenExecutingBaseExceptionMiddleware_TheResponseShouldBeEmpty() { - var baseExceptionMiddleware = CreateBaseExceptionMiddleware(innerHttpContext => Task.CompletedTask); + IFailureMetricHandler failureMetricHandler = Substitute.For(); + + var baseExceptionMiddleware = CreateBaseExceptionMiddleware(innerHttpContext => Task.CompletedTask, failureMetricHandler); await baseExceptionMiddleware.Invoke(_context); Assert.Equal(200, _context.Response.StatusCode); Assert.Null(_context.Response.ContentType); Assert.Equal(0, _context.Response.Body.Length); + + failureMetricHandler.Received(0).EmitHttpFailure(Arg.Any()); } - private BaseExceptionMiddleware CreateBaseExceptionMiddleware(RequestDelegate nextDelegate) + private BaseExceptionMiddleware CreateBaseExceptionMiddleware(RequestDelegate nextDelegate, IFailureMetricHandler failureMetricHandler) { - return Substitute.ForPartsOf(nextDelegate, NullLogger.Instance, _fhirRequestContextAccessor, _formatParametersValidator); + return Substitute.ForPartsOf( + nextDelegate, + NullLogger.Instance, + failureMetricHandler, + _fhirRequestContextAccessor, + _formatParametersValidator); } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Extensions/HttpRequestExtension.cs b/src/Microsoft.Health.Fhir.Shared.Api/Extensions/HttpRequestExtension.cs new file mode 100644 index 0000000000..610d08fbc2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Api/Extensions/HttpRequestExtension.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.Health.Fhir.Core.Features.Telemetry; + +namespace Microsoft.Health.Fhir.Api.Extensions +{ + public static class HttpRequestExtension + { + public static string GetOperationName(this HttpRequest request, bool includeRouteValues = true) + { + if (request != null) + { + var name = request.Path.Value; + if (request.RouteValues != null + && request.RouteValues.TryGetValue(KnownHttpRequestProperties.RouteValueAction, out var action) + && request.RouteValues.TryGetValue(KnownHttpRequestProperties.RouteValueController, out var controller)) + { + name = $"{controller}/{action}"; + + if (includeRouteValues) + { + var parameterArray = request.RouteValues.Keys?.Where( + k => k.Contains(KnownHttpRequestProperties.RouteValueParameterSuffix, StringComparison.OrdinalIgnoreCase)) + .OrderBy(k => k, StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (parameterArray != null && parameterArray.Any()) + { + name += $" [{string.Join("/", parameterArray)}]"; + } + } + } + + return $"{request.Method} {name}".Trim(); + } + + return null; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Exceptions/BaseExceptionMiddleware.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Exceptions/BaseExceptionMiddleware.cs index d2733352a3..6d53abc136 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Exceptions/BaseExceptionMiddleware.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Exceptions/BaseExceptionMiddleware.cs @@ -14,10 +14,12 @@ using Microsoft.Extensions.Logging; using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Fhir.Api.Extensions; using Microsoft.Health.Fhir.Api.Features.ActionResults; using Microsoft.Health.Fhir.Api.Features.ContentTypes; using Microsoft.Health.Fhir.Api.Features.Formatters; using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Logging.Metrics; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Api.Features.Exceptions @@ -26,28 +28,33 @@ public class BaseExceptionMiddleware { private readonly RequestDelegate _next; private readonly ILogger _logger; + private readonly IFailureMetricHandler _failureMetricHandler; private readonly RequestContextAccessor _fhirRequestContextAccessor; private readonly IFormatParametersValidator _parametersValidator; public BaseExceptionMiddleware( RequestDelegate next, ILogger logger, + IFailureMetricHandler failureMetricHandler, RequestContextAccessor fhirRequestContextAccessor, IFormatParametersValidator parametersValidator) { EnsureArg.IsNotNull(next, nameof(next)); EnsureArg.IsNotNull(logger, nameof(logger)); + EnsureArg.IsNotNull(failureMetricHandler, nameof(failureMetricHandler)); EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); EnsureArg.IsNotNull(parametersValidator, nameof(parametersValidator)); _next = next; _logger = logger; + _failureMetricHandler = failureMetricHandler; _fhirRequestContextAccessor = fhirRequestContextAccessor; _parametersValidator = parametersValidator; } public async Task Invoke(HttpContext context) { + bool doesOperationOutcomeHaveError = false; try { await _next(context); @@ -105,13 +112,30 @@ public async Task Invoke(HttpContext context) operationOutcome, exception is ServiceUnavailableException ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.InternalServerError); + doesOperationOutcomeHaveError = true; + await ExecuteResultAsync(context, result); } + finally + { + EmitHttpFailureMetricInCaseOfError(context, doesOperationOutcomeHaveError); + } } protected internal virtual async Task ExecuteResultAsync(HttpContext context, IActionResult result) { await result.ExecuteResultAsync(new ActionContext { HttpContext = context }); } + + private void EmitHttpFailureMetricInCaseOfError(HttpContext context, bool doesOperationOutcomeHaveError) + { + if (context.Response.StatusCode >= 500 || doesOperationOutcomeHaveError) + { + string operationName = context.Request.GetOperationName(includeRouteValues: false); + + _failureMetricHandler.EmitHttpFailure( + new HttpErrorMetricNotification() { OperationName = operationName }); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems b/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems index e592778729..1e57932cf2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems @@ -20,6 +20,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs index b6f20099b7..64abbdcd71 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -166,6 +166,7 @@ private static void AddMetricEmitter(IServiceCollection services) services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); } private class FhirServerBuilder : IFhirServerBuilder diff --git a/src/Microsoft.Health.Fhir.Shared.Web.UnitTests/AzureMonitorOpenTelemetryLogEnricherTests.cs b/src/Microsoft.Health.Fhir.Shared.Web.UnitTests/AzureMonitorOpenTelemetryLogEnricherTests.cs index baf0e3f770..e8a01d8f88 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web.UnitTests/AzureMonitorOpenTelemetryLogEnricherTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Web.UnitTests/AzureMonitorOpenTelemetryLogEnricherTests.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Telemetry; +using Microsoft.Health.Fhir.Core.Logging.Metrics; using NSubstitute; using NSubstitute.ReturnsExtensions; using OpenTelemetry.Logs; @@ -30,6 +31,7 @@ public class AzureMonitorOpenTelemetryLogEnricherTests private readonly AzureMonitorOpenTelemetryLogEnricher _enricher; private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IFailureMetricHandler _failureMetricHandler; private readonly HttpContext _httpContext; public AzureMonitorOpenTelemetryLogEnricherTests() @@ -38,7 +40,8 @@ public AzureMonitorOpenTelemetryLogEnricherTests() _httpContext = Substitute.For(); _httpContextAccessor.HttpContext.Returns(_httpContext); _httpContext.Request.Returns(Substitute.For()); - _enricher = new AzureMonitorOpenTelemetryLogEnricher(_httpContextAccessor); + _failureMetricHandler = Substitute.For(); + _enricher = new AzureMonitorOpenTelemetryLogEnricher(_httpContextAccessor, _failureMetricHandler); } [Fact] @@ -58,6 +61,8 @@ public void GivenRequest_WhenOperationNameIsAbsent_ThenOperationNameShouldBeAdde Assert.Equal( operationName, log.Attributes.SingleOrDefault(kv => kv.Key == KnownApplicationInsightsDimensions.OperationName).Value); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); } [Fact] @@ -83,6 +88,8 @@ public void GivenRequest_WhenOperationNameIsAbsent_ThenOperationNameShouldBeAdde Assert.Equal( operationName, log.Attributes.SingleOrDefault(kv => kv.Key == KnownApplicationInsightsDimensions.OperationName).Value); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); } [Fact] @@ -112,6 +119,8 @@ public void GivenRequest_WhenOperationNameIsAbsent_ThenOperationNameShouldBeAdde Assert.Equal( operationName, log.Attributes.SingleOrDefault(kv => kv.Key == KnownApplicationInsightsDimensions.OperationName).Value); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); } [Fact] @@ -143,6 +152,8 @@ public void GivenRequest_WhenOperationNameIsPresentAndEmpty_ThenOperationNameSho Assert.Equal( operationName, log.Attributes.Where(kv => kv.Key == KnownApplicationInsightsDimensions.OperationName).Select(kv => kv.Value).FirstOrDefault()); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); } [Fact] @@ -167,6 +178,46 @@ public void GivenRequest_WhenOperationNameIsAlreadySet_ThenOperationNameShouldNo Assert.Equal( operationName, log.Attributes.Where(kv => kv.Key == KnownApplicationInsightsDimensions.OperationName).Select(kv => kv.Value).FirstOrDefault()); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); + } + + [Theory] + [InlineData(LogLevel.Information)] + [InlineData(LogLevel.Warning)] + [InlineData(LogLevel.Critical)] + [InlineData(LogLevel.Error)] + public void GivenARecord_WhenThereIsAnException_ThenGenerateTheMetricAccordingly(LogLevel logLevel) + { + LogRecord log = CreateLogRecord( + DateTime.UtcNow, + Guid.NewGuid().ToString(), + logLevel, + new EventId(1), + "Creating a log record", + new InvalidOperationException("Test"), + null); + + _enricher.OnEnd(log); + + _failureMetricHandler.Received(1).EmitException(Arg.Any()); + } + + [Fact] + public void GivenARecord_WhenAnErrorIsLogged_ThenGenerateTheMetricAccordingly() + { + LogRecord log = CreateLogRecord( + DateTime.UtcNow, + Guid.NewGuid().ToString(), + LogLevel.Error, + new EventId(1), + "Creating a log record", + null, + null); + + _enricher.OnEnd(log); + + _failureMetricHandler.Received(1).EmitException(Arg.Any()); } [Fact] @@ -177,6 +228,8 @@ public void GivenRequest_WhenOperatoinNameIsAbsentAndHttpContextIsNull_ThenOpera _enricher.OnEnd(log); Assert.DoesNotContain(log.Attributes, kv => kv.Key == KnownApplicationInsightsDimensions.OperationName); + + _failureMetricHandler.Received(0).EmitException(Arg.Any()); } public static LogRecord CreateLogRecord() diff --git a/src/Microsoft.Health.Fhir.Shared.Web/AzureMonitorOpenTelemetryLogEnricher.cs b/src/Microsoft.Health.Fhir.Shared.Web/AzureMonitorOpenTelemetryLogEnricher.cs index cb81f473b0..e87aaa5363 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/AzureMonitorOpenTelemetryLogEnricher.cs +++ b/src/Microsoft.Health.Fhir.Shared.Web/AzureMonitorOpenTelemetryLogEnricher.cs @@ -5,17 +5,14 @@ #nullable enable -using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; -using System.Reflection; using EnsureThat; -using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Api.Extensions; using Microsoft.Health.Fhir.Core.Features.Telemetry; +using Microsoft.Health.Fhir.Core.Logging.Metrics; using OpenTelemetry; using OpenTelemetry.Logs; @@ -24,12 +21,17 @@ namespace Microsoft.Health.Fhir.Shared.Web public class AzureMonitorOpenTelemetryLogEnricher : BaseProcessor { private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IFailureMetricHandler _failureMetricHandler; - public AzureMonitorOpenTelemetryLogEnricher(IHttpContextAccessor httpContextAccessor) + public AzureMonitorOpenTelemetryLogEnricher( + IHttpContextAccessor httpContextAccessor, + IFailureMetricHandler failureMetricHandler) { EnsureArg.IsNotNull(httpContextAccessor, nameof(httpContextAccessor)); + EnsureArg.IsNotNull(failureMetricHandler, nameof(failureMetricHandler)); _httpContextAccessor = httpContextAccessor; + _failureMetricHandler = failureMetricHandler; } public override void OnEnd(LogRecord data) @@ -47,6 +49,8 @@ public override void OnEnd(LogRecord data) AddOperationName(newAttributes); data.Attributes = newAttributes.ToList(); + + EmitMetricBasedOnLogs(data); } base.OnEnd(data!); @@ -57,26 +61,35 @@ public void AddOperationName(Dictionary attributes) var request = _httpContextAccessor.HttpContext?.Request; if (request != null) { - var name = request.Path.Value; - if (request.RouteValues != null - && request.RouteValues.TryGetValue(KnownHttpRequestProperties.RouteValueAction, out var action) - && request.RouteValues.TryGetValue(KnownHttpRequestProperties.RouteValueController, out var controller)) + string name = request.GetOperationName(); + + if (!string.IsNullOrWhiteSpace(name)) { - name = $"{controller}/{action}"; - var parameterArray = request.RouteValues.Keys?.Where( - k => k.Contains(KnownHttpRequestProperties.RouteValueParameterSuffix, StringComparison.OrdinalIgnoreCase)) - .OrderBy(k => k, StringComparer.OrdinalIgnoreCase) - .ToArray(); - if (parameterArray != null && parameterArray.Any()) - { - name += $" [{string.Join("/", parameterArray)}]"; - } + attributes[KnownApplicationInsightsDimensions.OperationName] = name; } + } + } - if (!string.IsNullOrWhiteSpace(name)) + private void EmitMetricBasedOnLogs(LogRecord data) + { + // Metrics should be emitted if there is an exception, or if an error is logged. + if (data.Exception != null || data.LogLevel == LogLevel.Error) + { + string operationName = string.Empty; + var request = _httpContextAccessor.HttpContext?.Request; + if (request != null) { - attributes[KnownApplicationInsightsDimensions.OperationName] = $"{request.Method} {name}"; + operationName = request.GetOperationName(includeRouteValues: false); } + + string exceptionType = data.Exception == null ? "ExceptionTypeNotDefined" : data.Exception.GetType().Name; + var notification = new ExceptionMetricNotification() + { + OperationName = operationName, + ExceptionType = exceptionType, + Severity = data.LogLevel.ToString(), + }; + _failureMetricHandler.EmitException(notification); } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs b/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs index a4d2fbe95e..4027c76395 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs +++ b/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs @@ -26,6 +26,7 @@ using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Telemetry; +using Microsoft.Health.Fhir.Core.Logging.Metrics; using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.Shared.Web; @@ -340,7 +341,8 @@ private static void AddAzureMonitorOpenTelemetry(IServiceCollection services, Te options.AddProcessor(sp => { var httpContextAccessor = sp.GetRequiredService(); - return new AzureMonitorOpenTelemetryLogEnricher(httpContextAccessor); + var failureMetricHandler = sp.GetRequiredService(); + return new AzureMonitorOpenTelemetryLogEnricher(httpContextAccessor, failureMetricHandler); }); }); }