From 36e9efda29811194a24d1366b6e134b766a284d5 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 03:51:31 +0100 Subject: [PATCH 1/7] Add OpenAI-compatible LLM provider Signed-off-by: Chris0Jeky # Conflicts: # backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs # docs/platform/CONFIGURATION_REFERENCE.md # docs/platform/LLM_PROVIDER_SETUP_GUIDE.md --- .../Extensions/LlmProviderRegistration.cs | 27 ++ .../Taskdeck.Api/appsettings.Development.json | 7 + backend/src/Taskdeck.Api/appsettings.json | 7 + .../Services/ILlmProvider.cs | 4 +- .../Services/LlmProviderSelectionPolicy.cs | 85 +++- .../Services/LlmProviderSettings.cs | 37 +- .../Services/OpenAiCompatibleLlmProvider.cs | 442 ++++++++++++++++++ .../LlmProviderRegistrationTests.cs | 25 + .../Services/LlmProviderResilienceTests.cs | 28 ++ .../LlmProviderSelectionPolicyTests.cs | 38 ++ .../OpenAiCompatibleLlmProviderTests.cs | 172 +++++++ .../RoadmapInvariantTests.cs | 1 + docs/platform/CONFIGURATION_REFERENCE.md | 11 +- docs/platform/LLM_PROVIDER_SETUP_GUIDE.md | 72 ++- 14 files changed, 945 insertions(+), 11 deletions(-) create mode 100644 backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs create mode 100644 backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs diff --git a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs index dfa0043a1..b267d7210 100644 --- a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs @@ -78,6 +78,8 @@ public static IServiceCollection AddLlmProviders( circuitBreakerTracker, "Gemini", circuitBreakerSettings); var ollamaCircuitBreakerPolicy = BuildCircuitBreakerPolicy( circuitBreakerTracker, "Ollama", circuitBreakerSettings); + var openAiCompatibleCircuitBreakerPolicy = BuildCircuitBreakerPolicy( + circuitBreakerTracker, "OpenAICompatible", circuitBreakerSettings); // Determine once at startup whether localhost LLM endpoints are permitted. // This is true only in development-like environments with AllowLiveProvidersInDevelopment. @@ -114,6 +116,30 @@ public static IServiceCollection AddLlmProviders( .AddPolicyHandler(openAiCircuitBreakerPolicy) .RemoveAllLoggers() .AddHttpMessageHandler(); + services.AddHttpClient((sp, client) => + { + var settings = sp.GetRequiredService(); + var timeoutSeconds = settings.OpenAiCompatible?.TimeoutSeconds > 0 ? settings.OpenAiCompatible.TimeoutSeconds : 30; + client.Timeout = TimeSpan.FromSeconds(timeoutSeconds); + }) + .ConfigurePrimaryHttpMessageHandler(serviceProvider => + { + return new SocketsHttpHandler + { + AllowAutoRedirect = false, + UseProxy = false, + ActivityHeadersPropagator = null, + MeterFactory = serviceProvider.GetRequiredService(), + ConnectCallback = (context, cancellationToken) => + OutboundWebhookConnectCallback.ConnectAsync( + context, + allowLocalhostEndpoints: localhostPolicy.AllowGeneralProviderLocalhost, + cancellationToken) + }; + }) + .AddPolicyHandler(openAiCompatibleCircuitBreakerPolicy) + .RemoveAllLoggers() + .AddHttpMessageHandler(); services.AddHttpClient((sp, client) => { var settings = sp.GetRequiredService(); @@ -182,6 +208,7 @@ public static IServiceCollection AddLlmProviders( return decision.ProviderKind switch { LlmProviderKind.OpenAi => sp.GetRequiredService(), + LlmProviderKind.OpenAiCompatible => sp.GetRequiredService(), LlmProviderKind.Gemini => sp.GetRequiredService(), LlmProviderKind.Ollama => sp.GetRequiredService(), _ => sp.GetRequiredService() diff --git a/backend/src/Taskdeck.Api/appsettings.Development.json b/backend/src/Taskdeck.Api/appsettings.Development.json index cc50763d0..46ccbffd2 100644 --- a/backend/src/Taskdeck.Api/appsettings.Development.json +++ b/backend/src/Taskdeck.Api/appsettings.Development.json @@ -35,6 +35,13 @@ "Model": "gpt-4o-mini", "TimeoutSeconds": 30 }, + "OpenAiCompatible": { + "ApiKey": "", + "BaseUrl": "", + "Model": "", + "TimeoutSeconds": 30, + "ExtraHeaders": {} + }, "Gemini": { "ApiKey": "", "BaseUrl": "https://generativelanguage.googleapis.com/v1beta", diff --git a/backend/src/Taskdeck.Api/appsettings.json b/backend/src/Taskdeck.Api/appsettings.json index 74fcdb3b1..9273cf739 100644 --- a/backend/src/Taskdeck.Api/appsettings.json +++ b/backend/src/Taskdeck.Api/appsettings.json @@ -38,6 +38,13 @@ "Model": "gpt-4o-mini", "TimeoutSeconds": 30 }, + "OpenAiCompatible": { + "ApiKey": "", + "BaseUrl": "", + "Model": "", + "TimeoutSeconds": 30, + "ExtraHeaders": {} + }, "Gemini": { "ApiKey": "", "BaseUrl": "https://generativelanguage.googleapis.com/v1beta", diff --git a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs index a5d3d6ca3..a26694d3a 100644 --- a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs @@ -71,7 +71,9 @@ public record LlmTokenEvent( string? Error = null, int? TokensUsed = null, string? Provider = null, - string? Model = null); + string? Model = null, + bool IsDegraded = false, + string? DegradedReason = null); public record LlmHealthStatus( bool IsAvailable, diff --git a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs index 8e4da2758..cbd625871 100644 --- a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs +++ b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs @@ -5,7 +5,8 @@ public enum LlmProviderKind Mock = 0, OpenAi = 1, Gemini = 2, - Ollama = 3 + Ollama = 3, + OpenAiCompatible = 4 } public sealed record LlmProviderDecision(LlmProviderKind ProviderKind, string Reason); @@ -76,6 +77,20 @@ public static LlmProviderDecision Evaluate(LlmProviderSettings settings, string? "Gemini provider selected."); } + if (requestedProvider.Value == LlmProviderKind.OpenAiCompatible) + { + if (!TryValidateOpenAiCompatibleSettings(settings, out var compatibleValidationError, allowDevelopmentLocalhostEndpoints)) + { + return new LlmProviderDecision( + LlmProviderKind.Mock, + $"OpenAI-compatible configuration is invalid: {compatibleValidationError}"); + } + + return new LlmProviderDecision( + LlmProviderKind.OpenAiCompatible, + "OpenAI-compatible provider selected."); + } + var allowOllamaLocalhostEndpoints = allowDevelopmentLocalhostEndpoints && settings.Ollama?.AllowLocalhostEndpoints == true; @@ -196,6 +211,69 @@ public static bool TryValidateGeminiSettings( return true; } + public static bool TryValidateOpenAiCompatibleSettings( + LlmProviderSettings settings, + out string error, + bool allowLocalhostEndpoints = false) + { + if (settings.OpenAiCompatible is null) + { + error = "OpenAI-compatible settings are required."; + return false; + } + + var compatible = settings.OpenAiCompatible; + if (string.IsNullOrWhiteSpace(compatible.ApiKey)) + { + error = "ApiKey is required."; + return false; + } + + if (string.IsNullOrWhiteSpace(compatible.Model)) + { + error = "Model is required."; + return false; + } + + if (!Uri.TryCreate(compatible.BaseUrl, UriKind.Absolute, out var baseUri) || + (baseUri.Scheme != Uri.UriSchemeHttps && baseUri.Scheme != Uri.UriSchemeHttp)) + { + error = "BaseUrl must be an absolute HTTP(S) URI."; + return false; + } + + var ssrfResult = SsrfProtectionService.ValidateLlmProviderUrl(compatible.BaseUrl, allowLocalhostEndpoints); + if (!ssrfResult.IsAllowed) + { + error = $"BaseUrl blocked by SSRF protection: {ssrfResult.ErrorMessage}"; + return false; + } + + if (compatible.TimeoutSeconds <= 0) + { + error = "TimeoutSeconds must be greater than zero."; + return false; + } + + foreach (var (name, value) in compatible.ExtraHeaders) + { + if (string.IsNullOrWhiteSpace(name) || string.Equals(name, "Authorization", StringComparison.OrdinalIgnoreCase)) + { + error = "ExtraHeaders must use non-Authorization header names."; + return false; + } + + if (value is null || value.Contains('\r') || value.Contains('\n')) + { + error = "ExtraHeaders values must not contain line breaks."; + return false; + } + } + + error = string.Empty; + return true; + } + public static bool TryValidateOllamaSettings( LlmProviderSettings settings, out string error, @@ -258,6 +336,11 @@ public static bool TryValidateOllamaSettings( return LlmProviderKind.OpenAi; } + if (normalized.Equals("OpenAICompatible", StringComparison.OrdinalIgnoreCase)) + { + return LlmProviderKind.OpenAiCompatible; + } + if (normalized.Equals("Gemini", StringComparison.OrdinalIgnoreCase)) { return LlmProviderKind.Gemini; diff --git a/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs b/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs index 0f84a3a6a..e3ec7e892 100644 --- a/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs +++ b/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs @@ -8,7 +8,7 @@ public sealed class LlmProviderSettings public bool AllowLiveProvidersInDevelopment { get; set; } [Required(AllowEmptyStrings = false)] - [RegularExpression("^(?i)(Mock|OpenAi|Gemini|Ollama)$", ErrorMessage = "Llm Provider must be 'Mock', 'OpenAi', 'Gemini', or 'Ollama' (case-insensitive).")] + [RegularExpression("^(?i)(Mock|OpenAi|OpenAiCompatible|Gemini|Ollama)$", ErrorMessage = "Llm Provider must be 'Mock', 'OpenAi', 'OpenAiCompatible', 'Gemini', or 'Ollama' (case-insensitive).")] public string Provider { get; set; } = "Mock"; [Required] @@ -17,6 +17,13 @@ public sealed class LlmProviderSettings [Required] public GeminiProviderSettings Gemini { get; set; } = new(); + /// + /// Settings for an OpenAI chat-completions compatible endpoint such as + /// OpenRouter, Groq, or DeepSeek. Unlike , this has + /// no vendor default: callers must select and configure an endpoint. + /// + public OpenAiCompatibleProviderSettings OpenAiCompatible { get; set; } = new(); + public OllamaProviderSettings Ollama { get; set; } = new(); } @@ -61,6 +68,34 @@ public sealed class GeminiProviderSettings public int TimeoutSeconds { get; set; } = 30; } +public sealed class OpenAiCompatibleProviderSettings +{ + /// + /// API key accepted by the configured OpenAI-compatible endpoint. + /// + public string ApiKey { get; set; } = string.Empty; + + /// + /// Required OpenAI-compatible API base URL (for example, https://openrouter.ai/api/v1). + /// + public string BaseUrl { get; set; } = string.Empty; + + /// + /// Required vendor model identifier. + /// + public string Model { get; set; } = string.Empty; + + [Range(1, 300, ErrorMessage = "TimeoutSeconds must be between 1 and 300.")] + public int TimeoutSeconds { get; set; } = 30; + + /// + /// Optional request headers required by a compatible gateway (for example, + /// OpenRouter's HTTP-Referer and X-Title headers). Authorization is always + /// derived from and cannot be supplied here. + /// + public Dictionary ExtraHeaders { get; set; } = new(StringComparer.OrdinalIgnoreCase); +} + public sealed class OllamaProviderSettings { [Required(AllowEmptyStrings = false)] diff --git a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs new file mode 100644 index 000000000..995b590ec --- /dev/null +++ b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs @@ -0,0 +1,442 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace Taskdeck.Application.Services; + +/// +/// OpenAI chat-completions-compatible provider for vendor gateways. This is kept +/// separate from so api.openai.com defaults and +/// its existing non-streaming semantics remain unchanged. +/// +public sealed class OpenAiCompatibleLlmProvider : ILlmProvider +{ + private const string ProviderName = "OpenAICompatible"; + private const string BufferedStreamingFallbackReason = + "Upstream endpoint rejected SSE streaming; the response was completed before emission."; + + private readonly HttpClient _httpClient; + private readonly LlmProviderSettings _settings; + private readonly ILogger _logger; + + public OpenAiCompatibleLlmProvider( + HttpClient httpClient, + LlmProviderSettings settings, + ILogger logger) + { + _httpClient = httpClient; + _settings = settings; + _logger = logger; + } + + public async Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) + { + var userMessage = GetLastUserMessage(request); + if (!LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var validationError)) + { + _logger.LogWarning("OpenAI-compatible provider configuration invalid: {Error}", validationError); + return BuildFallbackResult(userMessage, "Live provider configuration is invalid."); + } + + try + { + var useInstructionExtraction = request.SystemPrompt is null; + var response = await SendCompletionAsync(request, stream: false, includeResponseFormat: useInstructionExtraction, ct); + if (!response.IsSuccessStatusCode && useInstructionExtraction && IsResponseFormatRejection(response.StatusCode)) + { + response.Dispose(); + _logger.LogInformation("OpenAI-compatible endpoint rejected response_format; retrying with prompt-enforced JSON only."); + response = await SendCompletionAsync(request, stream: false, includeResponseFormat: false, ct); + } + + using (response) + { + var body = await response.Content.ReadAsStringAsync(ct); + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("OpenAI-compatible completion request failed with status code {StatusCode}.", (int)response.StatusCode); + return BuildFallbackResult(userMessage, "Live provider request failed."); + } + + if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason)) + { + _logger.LogWarning("OpenAI-compatible completion response could not be parsed."); + return BuildFallbackResult(userMessage, "Live provider response parsing failed."); + } + + if (string.Equals(finishReason, "length", StringComparison.OrdinalIgnoreCase) || + (useInstructionExtraction && OpenAiLlmProvider.LooksLikeTruncatedJson(content))) + { + return new LlmCompletionResult( + content, tokensUsed, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault(), + IsDegraded: true, DegradedReason: "Response was truncated"); + } + + if (useInstructionExtraction && LlmInstructionExtractionPrompt.TryParseStructuredResponse( + content, out var reply, out var actionable, out var instructions)) + { + return new LlmCompletionResult( + reply, tokensUsed, actionable, actionable ? "llm.extracted" : null, + ProviderName, GetConfiguredModelOrDefault(), Instructions: instructions.Count > 0 ? instructions : null); + } + + var (isActionable, actionIntent) = LlmIntentClassifier.Classify(userMessage); + var fallbackInstructions = isActionable + ? NaturalLanguageInstructionExtractor.Extract(userMessage, actionIntent) + : []; + return new LlmCompletionResult( + content, tokensUsed, isActionable, actionIntent, ProviderName, GetConfiguredModelOrDefault(), + Instructions: fallbackInstructions.Count > 0 ? fallbackInstructions : null); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogError( + "OpenAI-compatible completion request failed with unexpected error. {ExceptionSummary}", + SensitiveDataRedactor.SummarizeException(ex)); + return BuildFallbackResult(userMessage, "Live provider request errored."); + } + } + + public async IAsyncEnumerable StreamAsync( + ChatCompletionRequest request, + [EnumeratorCancellation] CancellationToken ct = default) + { + if (!LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var validationError)) + { + yield return new LlmTokenEvent(string.Empty, true, Error: $"Live provider configuration is invalid: {validationError}", + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + + HttpResponseMessage? response = null; + try + { + // Streaming chat is conversational text, not the buffered instruction-extraction + // contract. Suppress response_format and the JSON-only default system prompt so the + // client receives the vendor's real text deltas rather than a JSON envelope. + var streamingRequest = request.SystemPrompt is null ? request with { SystemPrompt = string.Empty } : request; + using var message = CreateRequestMessage(streamingRequest, stream: true, includeResponseFormat: false); + response = await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); + + if (!response.IsSuccessStatusCode) + { + var statusCode = response.StatusCode; + response.Dispose(); + response = null; + if (IsStreamingRejection(statusCode)) + { + await foreach (var fallback in EmitBufferedFallbackAsync(request, ct)) + yield return fallback; + yield break; + } + + yield return new LlmTokenEvent(string.Empty, true, + Error: $"OpenAI-compatible streaming request failed with status code {(int)statusCode}.", + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + + var mediaType = response.Content.Headers.ContentType?.MediaType; + if (!string.Equals(mediaType, "text/event-stream", StringComparison.OrdinalIgnoreCase)) + { + var body = await response.Content.ReadAsStringAsync(ct); + response.Dispose(); + response = null; + if (TryParseResponse(body, out var content, out var tokensUsed, out _)) + { + yield return new LlmTokenEvent(content, true, TokensUsed: tokensUsed, Provider: ProviderName, + Model: GetConfiguredModelOrDefault(), IsDegraded: true, + DegradedReason: BufferedStreamingFallbackReason); + } + else + { + yield return new LlmTokenEvent(string.Empty, true, + Error: "OpenAI-compatible endpoint returned a non-SSE response that could not be parsed.", + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + } + yield break; + } + + await using var stream = await response.Content.ReadAsStreamAsync(ct); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); + var streamedContent = new StringBuilder(); + var data = new StringBuilder(); + var completed = false; + + while (await reader.ReadLineAsync(ct) is { } line) + { + ct.ThrowIfCancellationRequested(); + if (line.Length == 0) + { + if (data.Length == 0) + continue; + + var eventData = data.ToString(); + data.Clear(); + var parsed = TryParseSseEvent(eventData); + if (parsed.Error is not null) + { + yield return new LlmTokenEvent(string.Empty, true, Error: parsed.Error, + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + + if (!string.IsNullOrEmpty(parsed.Delta)) + { + streamedContent.Append(parsed.Delta); + yield return new LlmTokenEvent(parsed.Delta, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + } + + if (parsed.IsDone) + { + completed = true; + yield return new LlmTokenEvent(string.Empty, true, + TokensUsed: parsed.TokensUsed ?? EstimateTokens(streamedContent.ToString()), + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + + continue; + } + + if (line.StartsWith("data:", StringComparison.Ordinal)) + { + if (data.Length > 0) + data.Append('\n'); + data.Append(line[5..].TrimStart()); + } + } + + // A compliant SSE sender terminates each event with a blank line, but + // process a final unterminated data field as well so a connection close + // cannot turn an otherwise complete response into a false parse error. + if (data.Length > 0) + { + var parsed = TryParseSseEvent(data.ToString()); + if (parsed.Error is not null) + { + yield return new LlmTokenEvent(string.Empty, true, Error: parsed.Error, + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + + if (!string.IsNullOrEmpty(parsed.Delta)) + { + streamedContent.Append(parsed.Delta); + yield return new LlmTokenEvent(parsed.Delta, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + } + + if (parsed.IsDone) + { + completed = true; + yield return new LlmTokenEvent(string.Empty, true, + TokensUsed: parsed.TokensUsed ?? EstimateTokens(streamedContent.ToString()), + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield break; + } + } + + if (!completed) + { + yield return new LlmTokenEvent(string.Empty, true, + Error: "OpenAI-compatible SSE stream ended before a completion marker.", + Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + } + } + finally + { + response?.Dispose(); + } + } + + public Task GetHealthAsync(CancellationToken ct = default) + { + var isValid = LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var error); + return Task.FromResult(new LlmHealthStatus(isValid, ProviderName, isValid ? null : error, GetConfiguredModelOrDefault())); + } + + public async Task ProbeAsync(CancellationToken ct = default) + { + var result = await CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("user", "Reply with exactly: OK")], MaxTokens: 4, Temperature: 0, SystemPrompt: string.Empty), ct); + return new LlmHealthStatus(!result.IsDegraded, ProviderName, + result.IsDegraded ? result.DegradedReason : null, GetConfiguredModelOrDefault(), IsProbed: true); + } + + private async IAsyncEnumerable EmitBufferedFallbackAsync( + ChatCompletionRequest request, + [EnumeratorCancellation] CancellationToken ct) + { + var result = await CompleteAsync(request, ct); + yield return new LlmTokenEvent(result.Content, true, TokensUsed: result.TokensUsed, Provider: ProviderName, + Model: result.Model, IsDegraded: true, + DegradedReason: result.IsDegraded + ? $"{BufferedStreamingFallbackReason} {result.DegradedReason}" + : BufferedStreamingFallbackReason); + } + + private async Task SendCompletionAsync( + ChatCompletionRequest request, + bool stream, + bool includeResponseFormat, + CancellationToken ct) + { + using var message = CreateRequestMessage(request, stream, includeResponseFormat); + return await _httpClient.SendAsync(message, ct); + } + + private HttpRequestMessage CreateRequestMessage(ChatCompletionRequest request, bool stream, bool includeResponseFormat) + { + var compatible = _settings.OpenAiCompatible; + var message = new HttpRequestMessage(HttpMethod.Post, BuildChatCompletionsEndpoint()); + message.Headers.Authorization = new AuthenticationHeaderValue("Bearer", compatible.ApiKey.Trim()); + LlmRequestAttributionMapper.AddAttributionHeaders(message, request.Attribution); + foreach (var (name, value) in compatible.ExtraHeaders) + { + message.Headers.Add(name, value); + } + message.Content = JsonContent.Create(BuildRequestPayload(request, stream, includeResponseFormat)); + return message; + } + + private object BuildRequestPayload(ChatCompletionRequest request, bool stream, bool includeResponseFormat) + { + var systemPrompt = LlmSystemPromptBuilder.BuildEffectiveSystemPrompt( + request.SystemPrompt ?? LlmInstructionExtractionPrompt.SystemPrompt, request.BoardContext); + var messages = new List(); + if (!string.IsNullOrEmpty(systemPrompt)) + messages.Add(new { role = "system", content = systemPrompt }); + messages.AddRange(request.Messages.Select(MapMessage)); + + var payload = new Dictionary + { + ["model"] = _settings.OpenAiCompatible.Model.Trim(), + ["messages"] = messages.ToArray(), + ["max_tokens"] = request.MaxTokens, + ["temperature"] = request.Temperature, + ["stream"] = stream + }; + if (includeResponseFormat) + payload["response_format"] = new { type = "json_object" }; + if (request.Attribution is not null) + payload["user"] = LlmRequestAttributionMapper.BuildUserToken(request.Attribution.UserId); + return payload; + } + + private static SseEventParseResult TryParseSseEvent(string eventData) + { + if (string.Equals(eventData.Trim(), "[DONE]", StringComparison.Ordinal)) + return new SseEventParseResult(IsDone: true); + + try + { + using var document = JsonDocument.Parse(eventData); + var root = document.RootElement; + if (root.TryGetProperty("error", out _)) + return new SseEventParseResult(Error: "OpenAI-compatible SSE stream reported an upstream error."); + if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array || choices.GetArrayLength() == 0) + return new SseEventParseResult(Error: "OpenAI-compatible SSE event did not contain choices."); + + var first = choices[0]; + var delta = first.TryGetProperty("delta", out var deltaElement) && + deltaElement.TryGetProperty("content", out var contentElement) && + contentElement.ValueKind == JsonValueKind.String + ? contentElement.GetString() + : null; + var finishReason = first.TryGetProperty("finish_reason", out var finish) && finish.ValueKind != JsonValueKind.Null + ? finish.GetString() + : null; + var tokensUsed = root.TryGetProperty("usage", out var usage) && + usage.TryGetProperty("total_tokens", out var total) && total.TryGetInt32(out var parsedTokens) + ? (int?)parsedTokens + : null; + return new SseEventParseResult(delta, !string.IsNullOrEmpty(finishReason), tokensUsed); + } + catch (JsonException) + { + return new SseEventParseResult(Error: "OpenAI-compatible SSE stream contained malformed JSON."); + } + } + + private static bool TryParseResponse(string body, out string content, out int tokensUsed, out string? finishReason) + { + content = string.Empty; + tokensUsed = 0; + finishReason = null; + try + { + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array || choices.GetArrayLength() == 0) + return false; + var choice = choices[0]; + if (!choice.TryGetProperty("message", out var message) || !message.TryGetProperty("content", out var contentElement)) + return false; + content = contentElement.GetString() ?? string.Empty; + if (string.IsNullOrWhiteSpace(content)) + return false; + finishReason = choice.TryGetProperty("finish_reason", out var finish) && finish.ValueKind == JsonValueKind.String + ? finish.GetString() + : null; + tokensUsed = root.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var total) && total.TryGetInt32(out var parsed) + ? parsed + : EstimateTokens(content); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static bool IsResponseFormatRejection(HttpStatusCode statusCode) => + statusCode is HttpStatusCode.BadRequest or HttpStatusCode.UnprocessableEntity; + + private static bool IsStreamingRejection(HttpStatusCode statusCode) => + statusCode is HttpStatusCode.BadRequest or HttpStatusCode.NotFound or HttpStatusCode.MethodNotAllowed or + HttpStatusCode.NotImplemented or HttpStatusCode.UnprocessableEntity; + + private string BuildChatCompletionsEndpoint() => $"{_settings.OpenAiCompatible.BaseUrl.TrimEnd('/')}/chat/completions"; + + private string GetConfiguredModelOrDefault() => string.IsNullOrWhiteSpace(_settings.OpenAiCompatible?.Model) + ? "openai-compatible-unknown-model" + : _settings.OpenAiCompatible.Model.Trim(); + + private static object MapMessage(ChatCompletionMessage message) => new + { + role = (message.Role ?? string.Empty).Trim().ToLowerInvariant() switch + { + "assistant" => "assistant", + "system" => "system", + _ => "user" + }, + content = message.Content + }; + + private static string GetLastUserMessage(ChatCompletionRequest request) => request.Messages + .LastOrDefault(message => string.Equals(message.Role, "User", StringComparison.OrdinalIgnoreCase))?.Content ?? string.Empty; + + private static int EstimateTokens(string text) => Math.Max(1, string.IsNullOrWhiteSpace(text) ? 1 : text.Length / 4); + + private LlmCompletionResult BuildFallbackResult(string userMessage, string reason) + { + var (isActionable, actionIntent) = LlmIntentClassifier.Classify(userMessage); + var instructions = isActionable ? NaturalLanguageInstructionExtractor.Extract(userMessage, actionIntent) : []; + var content = isActionable + ? $"I can help with that. I'll create a proposal to {actionIntent}. ({reason})" + : $"I can help with that request. ({reason})"; + return new LlmCompletionResult(content, EstimateTokens(userMessage) + EstimateTokens(content), isActionable, actionIntent, + ProviderName, GetConfiguredModelOrDefault(), true, reason, instructions.Count > 0 ? instructions : null); + } + + private sealed record SseEventParseResult(string? Delta = null, bool IsDone = false, int? TokensUsed = null, string? Error = null); +} diff --git a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs index 3d1c6fb57..6441f565a 100644 --- a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs @@ -13,6 +13,31 @@ namespace Taskdeck.Api.Tests; public class LlmProviderRegistrationTests { + [Fact] + public void AddLlmProviders_ResolvesOpenAiCompatibleProvider_WhenSelectionIsValid() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(new TestWebHostEnvironment("Production")); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Llm:EnableLiveProviders"] = "true", + ["Llm:Provider"] = "OpenAICompatible", + ["Llm:OpenAiCompatible:ApiKey"] = "test-compatible-key", + ["Llm:OpenAiCompatible:BaseUrl"] = "https://api.groq.com/openai/v1", + ["Llm:OpenAiCompatible:Model"] = "llama-3.1-8b-instant", + ["Llm:OpenAiCompatible:TimeoutSeconds"] = "30" + }) + .Build(); + + services.AddLlmProviders(configuration); + using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService() + .Should().BeOfType(); + } + [Theory] [InlineData(nameof(OpenAiLlmProvider))] [InlineData(nameof(GeminiLlmProvider))] diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderResilienceTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderResilienceTests.cs index 15f09c5b9..5f597d4ed 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderResilienceTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderResilienceTests.cs @@ -17,6 +17,34 @@ namespace Taskdeck.Application.Tests.Services; /// public class LlmProviderResilienceTests { + [Fact] + public async Task OpenAiCompatible_CompleteAsync_GarbageResponseBody_ReturnsDegradedResult() + { + var settings = new LlmProviderSettings + { + EnableLiveProviders = true, + Provider = "OpenAICompatible", + OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.example.test/v1", + Model = "vendor/model" + } + }; + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("not a compatible completion", Encoding.UTF8, "text/plain") + }); + var provider = new OpenAiCompatibleLlmProvider( + new HttpClient(handler), settings, NullLogger.Instance); + + var result = await provider.CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "create a card")])); + + result.IsDegraded.Should().BeTrue(); + result.Provider.Should().Be("OpenAICompatible"); + } + // ── OpenAI: Garbage Response (Invalid JSON Body) ───────────────── [Fact] diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs index 02241aa4b..4c0a51c69 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs @@ -88,6 +88,44 @@ public void Evaluate_ShouldSelectGemini_WhenProductionAndConfigurationIsValid() result.ProviderKind.Should().Be(LlmProviderKind.Gemini); } + [Fact] + public void Evaluate_ShouldSelectOpenAiCompatible_WhenProductionAndConfigurationIsValid() + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + ExtraHeaders = new Dictionary { ["X-Title"] = "Taskdeck" } + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.OpenAiCompatible); + } + + [Fact] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleBaseUrlTargetsPrivateHost() + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://127.0.0.1/v1", + Model = "local-model" + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("SSRF"); + } + [Fact] public void Evaluate_ShouldSelectMock_WhenGeminiConfigurationIsInvalid() { diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs new file mode 100644 index 000000000..de895fa11 --- /dev/null +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs @@ -0,0 +1,172 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Taskdeck.Application.Services; +using Taskdeck.Application.Tests.TestUtilities; +using Xunit; + +namespace Taskdeck.Application.Tests.Services; + +public class OpenAiCompatibleLlmProviderTests +{ + [Fact] + public async Task StreamAsync_ParsesRealSseDeltas_AndSendsStreamTrue() + { + var settings = BuildSettings(); + string? requestBody = null; + string? extraHeader = null; + var handler = new StubHttpMessageHandler(async (request, ct) => + { + requestBody = await request.Content!.ReadAsStringAsync(ct); + extraHeader = request.Headers.GetValues("X-Title").Single(); + return SseResponse( + "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"},\"finish_reason\":null}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"},\"finish_reason\":\"stop\"}],\"usage\":{\"total_tokens\":7}}\n\n"); + }); + var provider = CreateProvider(handler, settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Select(item => item.Token).Should().Equal("Hel", "lo", string.Empty); + events[^1].IsComplete.Should().BeTrue(); + events[^1].TokensUsed.Should().Be(7); + events[^1].IsDegraded.Should().BeFalse(); + using var payload = JsonDocument.Parse(requestBody!); + payload.RootElement.GetProperty("stream").GetBoolean().Should().BeTrue(); + payload.RootElement.TryGetProperty("response_format", out _).Should().BeFalse( + "streaming chat sends readable deltas rather than the buffered extraction envelope"); + extraHeader.Should().Be("Taskdeck tests"); + } + + [Fact] + public async Task StreamAsync_MalformedSse_EmitsExplicitCompletionError() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse("data: not-json\n\n")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].IsComplete.Should().BeTrue(); + events[0].Error.Should().Contain("malformed JSON"); + } + + [Fact] + public async Task StreamAsync_MidStreamError_PreservesDeliveredDelta_AndSignalsCompletionError() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}]}\n\n" + + "data: {\"error\":{\"message\":\"upstream failed\"}}\n\n")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().HaveCount(2); + events[0].Token.Should().Be("partial"); + events[0].IsComplete.Should().BeFalse(); + events[1].IsComplete.Should().BeTrue(); + events[1].Error.Should().Contain("upstream error"); + } + + [Fact] + public async Task StreamAsync_WhenEndpointRejectsStreaming_EmitsBufferedFallbackMetadata() + { + var settings = BuildSettings(); + var requests = new List(); + var handler = new StubHttpMessageHandler(async (request, ct) => + { + requests.Add(await request.Content!.ReadAsStringAsync(ct)); + return requests.Count == 1 + ? new HttpResponseMessage(HttpStatusCode.BadRequest) + : JsonResponse("""{"choices":[{"message":{"content":"fallback reply"}}],"usage":{"total_tokens":9}}"""); + }); + var provider = CreateProvider(handler, settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Token.Should().Be("fallback reply"); + events[0].IsComplete.Should().BeTrue(); + events[0].IsDegraded.Should().BeTrue(); + events[0].DegradedReason.Should().Contain("rejected SSE streaming"); + requests.Should().HaveCount(2); + using var streamedPayload = JsonDocument.Parse(requests[0]); + using var bufferedPayload = JsonDocument.Parse(requests[1]); + streamedPayload.RootElement.GetProperty("stream").GetBoolean().Should().BeTrue(); + bufferedPayload.RootElement.GetProperty("stream").GetBoolean().Should().BeFalse(); + } + + [Fact] + public async Task CompleteAsync_WhenResponseFormatIsRejected_RetriesWithPromptOnlyJson() + { + var requestBodies = new List(); + var handler = new StubHttpMessageHandler(async (request, ct) => + { + requestBodies.Add(await request.Content!.ReadAsStringAsync(ct)); + return requestBodies.Count == 1 + ? new HttpResponseMessage(HttpStatusCode.UnprocessableEntity) + : JsonResponse("""{"choices":[{"message":{"content":"plain fallback"}}],"usage":{"total_tokens":4}}"""); + }); + var provider = CreateProvider(handler, BuildSettings()); + + var result = await provider.CompleteAsync(Request()); + + result.Content.Should().Be("plain fallback"); + result.IsDegraded.Should().BeFalse(); + requestBodies.Should().HaveCount(2); + using var initialPayload = JsonDocument.Parse(requestBodies[0]); + using var fallbackPayload = JsonDocument.Parse(requestBodies[1]); + initialPayload.RootElement.TryGetProperty("response_format", out _).Should().BeTrue(); + fallbackPayload.RootElement.TryGetProperty("response_format", out _).Should().BeFalse(); + } + + [Fact] + public async Task StreamAsync_PropagatesCancellation() + { + var handler = new StubHttpMessageHandler((_, ct) => Task.FromCanceled(ct)); + var provider = CreateProvider(handler, BuildSettings()); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var act = async () => await CollectAsync(provider.StreamAsync(Request(), cancellation.Token)); + + await act.Should().ThrowAsync(); + } + + private static OpenAiCompatibleLlmProvider CreateProvider(HttpMessageHandler handler, LlmProviderSettings settings) => + new(new HttpClient(handler), settings, NullLogger.Instance); + + private static ChatCompletionRequest Request() => new([new ChatCompletionMessage("user", "hello")]); + + private static async Task> CollectAsync(IAsyncEnumerable stream) + { + var events = new List(); + await foreach (var item in stream) + events.Add(item); + return events; + } + + private static HttpResponseMessage SseResponse(string body) => new(HttpStatusCode.OK) + { + Content = new StringContent(body, Encoding.UTF8, "text/event-stream") + }; + + private static HttpResponseMessage JsonResponse(string body) => new(HttpStatusCode.OK) + { + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + + private static LlmProviderSettings BuildSettings() => new() + { + EnableLiveProviders = true, + Provider = "OpenAICompatible", + OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.example.test/v1", + Model = "vendor/model", + TimeoutSeconds = 30, + ExtraHeaders = new Dictionary { ["X-Title"] = "Taskdeck tests" } + } + }; +} diff --git a/backend/tests/Taskdeck.Architecture.Tests/RoadmapInvariantTests.cs b/backend/tests/Taskdeck.Architecture.Tests/RoadmapInvariantTests.cs index c4d42745f..41259e020 100644 --- a/backend/tests/Taskdeck.Architecture.Tests/RoadmapInvariantTests.cs +++ b/backend/tests/Taskdeck.Architecture.Tests/RoadmapInvariantTests.cs @@ -376,6 +376,7 @@ public void Invariant08_EgressEnvelope_OutboundHttpConstrained() var expectedSites = new HashSet(StringComparer.OrdinalIgnoreCase) { "OpenAiLlmProvider", + "OpenAiCompatibleLlmProvider", "GeminiLlmProvider", "OllamaLlmProvider", "OutboundWebhookDeliveryWorker", diff --git a/docs/platform/CONFIGURATION_REFERENCE.md b/docs/platform/CONFIGURATION_REFERENCE.md index f637839d2..16e78c8e8 100644 --- a/docs/platform/CONFIGURATION_REFERENCE.md +++ b/docs/platform/CONFIGURATION_REFERENCE.md @@ -208,20 +208,25 @@ Bound to `MfaPolicySettings`. Registered in `SettingsRegistration.cs`. ### `Llm` -Bound to `LlmProviderSettings` (nested: `OpenAi`, `Gemini`, `Ollama`). Registered in +Bound to `LlmProviderSettings` (nested: `OpenAi`, `OpenAiCompatible`, `Gemini`, `Ollama`). Registered in `LlmProviderRegistration.AddLlmProviders`. The Mock provider is always the default and the only one that ships enabled. See `docs/platform/LLM_PROVIDER_SETUP_GUIDE.md` for end-to-end provider setup. | Key | Type | Default | Description | Required? | | --- | --- | --- | --- | --- | -| `Llm:EnableLiveProviders` | `bool` | `false` | Master switch. Live providers (OpenAI, Gemini, Ollama) only run when this is true. | No | +| `Llm:EnableLiveProviders` | `bool` | `false` | Master switch. Live providers (OpenAI, OpenAICompatible, Gemini, Ollama) only run when this is true. | No | | `Llm:AllowLiveProvidersInDevelopment` | `bool` | `false` | Safety gate — live providers refuse to run in the `Development` environment unless this is also true. | No | -| `Llm:Provider` | `string` | `Mock` | Provider selector. `Mock`, `OpenAi`, `Gemini`, or `Ollama`. Resolved by `LlmProviderSelectionPolicy.Evaluate`. | No | +| `Llm:Provider` | `string` | `Mock` | Provider selector. `Mock`, `OpenAi`, `OpenAiCompatible`, `Gemini`, or `Ollama`. Resolved by `LlmProviderSelectionPolicy.Evaluate`. | No | | `Llm:OpenAi:ApiKey` | `string` | `""` | OpenAI API key. Required to use the OpenAI provider. Store as a secret. | Only for `Llm:Provider = OpenAi` | | `Llm:OpenAi:BaseUrl` | `string` | `https://api.openai.com/v1` | OpenAI API base URL. Override for compatible gateways. | No | | `Llm:OpenAi:Model` | `string` | `gpt-4o-mini` | Model identifier sent in chat requests. | No | | `Llm:OpenAi:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` applied to the OpenAI provider. Must be `> 0`: `LlmProviderSelectionPolicy.TryValidateOpenAiSettings` rejects values `<= 0` as invalid and the selection policy falls back to the Mock provider. (The `HttpClient` registration also substitutes `30` when the value is `<= 0`, but only as a safety net — the provider will still not be selected.) | No | +| `Llm:OpenAiCompatible:ApiKey` | `string` | `""` | API key for the configured OpenAI-compatible endpoint. Store as a secret. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. It passes the same HTTP(S), private-network, metadata-host, and DNS connection-time SSRF controls as OpenAI. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:Model` | `string` | `""` | Required model identifier supplied by the compatible vendor. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` for the compatible provider. Must be `> 0`; invalid values select Mock. | No | +| `Llm:OpenAiCompatible:ExtraHeaders:` | `object` (map) | `{}` | Optional non-secret gateway headers, for example `HTTP-Referer` and `X-Title` for OpenRouter. `Authorization` cannot be set here because Taskdeck derives it from `ApiKey`; values with line breaks are rejected. | No | | `Llm:Gemini:ApiKey` | `string` | `""` | Gemini API key. Required to use the Gemini provider. Store as a secret. | Only for `Llm:Provider = Gemini` | | `Llm:Gemini:BaseUrl` | `string` | `https://generativelanguage.googleapis.com/v1beta` | Gemini API base URL. | No | | `Llm:Gemini:Model` | `string` | `gemini-2.5-flash` | Model identifier. | No | diff --git a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md index 9572b0ce4..afd3f1ade 100644 --- a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md +++ b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md @@ -6,7 +6,7 @@ Scope: Provider runtime setup for chat/capture automation and safe local demo op ## Purpose Taskdeck keeps application services provider-agnostic through `ILlmProvider`, while retaining a safe default posture. -This guide defines what is now shipped and how to run OpenAI/Gemini/Ollama demos without code changes. +This guide defines what is now shipped and how to run configured LLM demos without code changes. ## Current Shipped State @@ -14,6 +14,7 @@ Backend provider runtime now supports: - `Mock` provider (default) - `OpenAI` provider (config-gated) +- `OpenAICompatible` provider (config-gated; OpenRouter, Groq, and DeepSeek-compatible chat endpoints) - `Gemini` provider (config-gated) - `Ollama` provider (config-gated) - managed-key attribution baseline for provider-bound chat/capture requests (`#236`): @@ -24,13 +25,13 @@ Backend provider runtime now supports: Selection is deterministic through `LlmProviderSelectionPolicy`: -- to use live providers (`OpenAI`/`Gemini`/`Ollama`), live providers must be enabled (`EnableLiveProviders=true`) +- to use live providers (`OpenAI`, `OpenAICompatible`, `Gemini`, or `Ollama`), live providers must be enabled (`EnableLiveProviders=true`) - to use live providers in development-like environments, explicit live mode is required (`AllowLiveProvidersInDevelopment=true`) -- provider mode may be explicitly set to `Mock`, `OpenAI`, `Gemini`, or `Ollama`; this guide's config example intentionally uses `Mock` as the safe default +- provider mode may be explicitly set to `Mock`, `OpenAI`, `OpenAICompatible`, `Gemini`, or `Ollama`; this guide's config example intentionally uses `Mock` as the safe default - unknown provider values also fall back deterministically to `Mock` - selected provider config must pass provider-specific validation (`BaseUrl`, `Model`, `TimeoutSeconds`, and an API key where required) - `BaseUrl` is additionally validated by `SsrfProtectionService.ValidateLlmProviderUrl` (SEC-26 PR `#905`): private IPv4 (`127/8`, `10/8`, `172.16/12`, `192.168/16`), IPv6 ranges (`::1`, `fc00::/7`, `fe80::/10`), IPv4-mapped IPv6, cloud metadata hostnames (`metadata.google.internal`, `metadata.goog`, AWS IMDS `169.254.169.254`, AWS IMDSv2 IPv6 `fd00:ec2::254`, Alibaba `100.100.100.200`), and non-HTTPS URLs are rejected; the selection policy falls back to Mock when validation fails -- the OpenAI, Gemini, and Ollama primary `HttpClient` handlers use `OutboundWebhookConnectCallback` for DNS-level SSRF protection and set both `AllowAutoRedirect = false` and `UseProxy = false`; ambient/system proxy settings are ignored so the configured provider origin remains the host validated by the connect callback +- the OpenAI, OpenAICompatible, Gemini, and Ollama primary `HttpClient` handlers use `OutboundWebhookConnectCallback` for DNS-level SSRF protection and set both `AllowAutoRedirect = false` and `UseProxy = false`; ambient/system proxy settings are ignored so the configured provider origin remains the host validated by the connect callback These provider transports are direct-only. Taskdeck has no proxy-aware outbound LLM mode, so a deployment that can reach providers only through a corporate proxy fails closed. This is a dedicated connect-callback boundary, not `EgressEnvelopeHandler` enforcement, and it does not change the existing redirect or audit posture. Registered protected clients mask their configured URI immediately before send, then an inner handler restores it for transport; this keeps path/query data out of outer .NET HTTP EventSource payloads. Public caller-owned provider clients do not opt into masking. Protected registrations remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private metric scope that Taskdeck's configured OpenTelemetry pipeline drops alongside marked HTTP activities. Enabled Sentry removes its outbound handler from these protected client pipelines only, so it cannot add separate `sentry-trace`/baggage headers or capture protected URLs and 5xx responses; unrelated clients retain instrumentation and server-side Sentry tracking remains active. Independently installed process-global `ActivityListener`/`MeterListener` and transport-stage host/IP observation remain outside the guarantee. @@ -69,6 +70,13 @@ in Development. "Model": "gpt-4o-mini", "TimeoutSeconds": 30 }, + "OpenAiCompatible": { + "ApiKey": "", + "BaseUrl": "", + "Model": "", + "TimeoutSeconds": 30, + "ExtraHeaders": {} + }, "Gemini": { "ApiKey": "", "BaseUrl": "https://generativelanguage.googleapis.com/v1beta", @@ -134,6 +142,59 @@ The Ollama localhost exception is effective only in Development/Test/Testing and only for the exact `localhost` hostname. Production, literal loopback addresses, and other private/link-local origins remain blocked. +## OpenAI-Compatible Providers (OpenRouter, Groq, DeepSeek) + +`OpenAICompatible` is the named provider for public HTTPS endpoints using the +OpenAI Chat Completions wire format. It is distinct from `OpenAI`: OpenAI keeps +its `api.openai.com` defaults, while compatible endpoints require an explicit +base URL and model. The provider sends real upstream SSE requests (`stream:true`) +for chat streams and forwards delta events as they arrive. + +Set the common safety gates and provider name: + +- `Llm__EnableLiveProviders=true` +- `Llm__AllowLiveProvidersInDevelopment=true` (only for development-like environments) +- `Llm__Provider=OpenAICompatible` + +The endpoint must be public HTTP(S) and pass the same URL and DNS-level SSRF +checks as OpenAI. Keep keys in a secret store; never commit them. Compatible +gateways may require optional non-secret headers such as `HTTP-Referer` or +`X-Title`; use `Llm__OpenAiCompatible__ExtraHeaders__` for those. +`Authorization` is reserved for the configured API key and cannot be overridden. + +### OpenRouter + +```powershell +$env:Llm__OpenAiCompatible__ApiKey = '' +$env:Llm__OpenAiCompatible__BaseUrl = 'https://openrouter.ai/api/v1' +$env:Llm__OpenAiCompatible__Model = 'openai/gpt-4o-mini' +Set-Item -Path 'Env:Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer' -Value 'https://your-app.example' +Set-Item -Path 'Env:Llm__OpenAiCompatible__ExtraHeaders__X-Title' -Value 'Taskdeck' +``` + +### Groq + +```powershell +$env:Llm__OpenAiCompatible__ApiKey = '' +$env:Llm__OpenAiCompatible__BaseUrl = 'https://api.groq.com/openai/v1' +$env:Llm__OpenAiCompatible__Model = 'llama-3.1-8b-instant' +``` + +### DeepSeek + +```powershell +$env:Llm__OpenAiCompatible__ApiKey = '' +$env:Llm__OpenAiCompatible__BaseUrl = 'https://api.deepseek.com/v1' +$env:Llm__OpenAiCompatible__Model = 'deepseek-chat' +``` + +If a gateway rejects SSE, Taskdeck retries as a normal completion and emits one +final event with explicit `isDegraded`/`degradedReason` metadata rather than +pretending the buffered response was incremental. Some compatible gateways also +reject `response_format: { type: json_object }`; non-streaming extraction retries +without that field while retaining the JSON-only instruction prompt and robust +response parsing. + ## Playwright Demo Auto-Enable For full Playwright-backed demos (`npm run demo:director` or `TASKDECK_RUN_DEMO=1 npx playwright test tests/e2e/stakeholder-demo.spec.ts --headed`): @@ -200,9 +261,10 @@ This is intentionally separate from the broader demo tooling so an operator can ## Test Coverage Expectations (Implemented) -- selection-policy unit coverage for `Mock`/`OpenAI`/`Gemini` and invalid-config fallback +- selection-policy unit coverage for `Mock`/`OpenAI`/`OpenAICompatible`/`Gemini` and invalid-config fallback - provider adapter unit coverage: - OpenAI: success/failure + metadata checks + - OpenAICompatible: true SSE delta parsing, malformed/mid-stream error, cancellation, and explicit buffered-fallback metadata - Gemini: success/failure/invalid-response/invalid-config/cancellation + health + attribution header mapping - API integration coverage: - capture triage provenance includes provider/model From a95e0598ede96f5681a0fb6058a69c40dadc2ec9 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 03:58:25 +0100 Subject: [PATCH 2/7] Harden compatible provider validation Signed-off-by: Chris0Jeky --- .../Services/LlmProviderSelectionPolicy.cs | 20 +++++++++ .../Services/OpenAiCompatibleLlmProvider.cs | 15 +++++-- .../LlmProviderSelectionPolicyTests.cs | 42 +++++++++++++++++++ .../OpenAiCompatibleLlmProviderTests.cs | 18 ++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs index cbd625871..6c2466304 100644 --- a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs +++ b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs @@ -255,6 +255,12 @@ public static bool TryValidateOpenAiCompatibleSettings( return false; } + if (compatible.ExtraHeaders is null) + { + error = "ExtraHeaders must be a header map when configured."; + return false; + } + foreach (var (name, value) in compatible.ExtraHeaders) { if (string.IsNullOrWhiteSpace(name) || string.Equals(name, "Authorization", StringComparison.OrdinalIgnoreCase)) @@ -268,6 +274,20 @@ public static bool TryValidateOpenAiCompatibleSettings( error = "ExtraHeaders values must not contain line breaks."; return false; } + + // Use the framework's request-header parser so malformed and + // content-only/restricted names select the deterministic Mock + // provider instead of throwing when the client constructs a request. + using var headerProbe = new HttpRequestMessage(); + try + { + headerProbe.Headers.Add(name, value); + } + catch (Exception ex) when (ex is ArgumentException or FormatException or InvalidOperationException) + { + error = $"ExtraHeaders contains an invalid or restricted request header '{name}'."; + return false; + } } error = string.Empty; diff --git a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs index 995b590ec..68cace4f3 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs @@ -36,7 +36,7 @@ public OpenAiCompatibleLlmProvider( public async Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) { var userMessage = GetLastUserMessage(request); - if (!LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var validationError)) + if (!TryValidateSettings(out var validationError)) { _logger.LogWarning("OpenAI-compatible provider configuration invalid: {Error}", validationError); return BuildFallbackResult(userMessage, "Live provider configuration is invalid."); @@ -110,7 +110,7 @@ public async IAsyncEnumerable StreamAsync( ChatCompletionRequest request, [EnumeratorCancellation] CancellationToken ct = default) { - if (!LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var validationError)) + if (!TryValidateSettings(out var validationError)) { yield return new LlmTokenEvent(string.Empty, true, Error: $"Live provider configuration is invalid: {validationError}", Provider: ProviderName, Model: GetConfiguredModelOrDefault()); @@ -260,7 +260,7 @@ public async IAsyncEnumerable StreamAsync( public Task GetHealthAsync(CancellationToken ct = default) { - var isValid = LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings(_settings, out var error); + var isValid = TryValidateSettings(out var error); return Task.FromResult(new LlmHealthStatus(isValid, ProviderName, isValid ? null : error, GetConfiguredModelOrDefault())); } @@ -294,6 +294,15 @@ private async Task SendCompletionAsync( return await _httpClient.SendAsync(message, ct); } + private bool TryValidateSettings(out string error) => + LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings( + _settings, + out error, + // The selected provider can use the narrow development localhost + // exception already granted by the selection policy. In production, + // selection and the DNS-level ConnectCallback still reject it. + allowLocalhostEndpoints: _settings.AllowLiveProvidersInDevelopment); + private HttpRequestMessage CreateRequestMessage(ChatCompletionRequest request, bool stream, bool includeResponseFormat) { var compatible = _settings.OpenAiCompatible; diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs index 4c0a51c69..2548b92ea 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs @@ -126,6 +126,48 @@ public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleBaseUrlTargetsPrivateH result.Reason.Should().Contain("SSRF"); } + [Fact] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleExtraHeadersIsNull() + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + ExtraHeaders = null! + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("ExtraHeaders"); + } + + [Theory] + [InlineData("Content-Type")] + [InlineData("X Invalid")] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleExtraHeaderIsInvalidOrRestricted(string headerName) + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + ExtraHeaders = new Dictionary { [headerName] = "value" } + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("invalid or restricted"); + } + [Fact] public void Evaluate_ShouldSelectMock_WhenGeminiConfigurationIsInvalid() { diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs index de895fa11..6a521642f 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs @@ -133,6 +133,24 @@ public async Task StreamAsync_PropagatesCancellation() await act.Should().ThrowAsync(); } + [Fact] + public async Task CompleteAsync_AllowsTheDevelopmentLocalhostEndpointSelectedByPolicy() + { + var settings = BuildSettings(); + settings.AllowLiveProvidersInDevelopment = true; + settings.OpenAiCompatible.BaseUrl = "http://localhost:11434/v1"; + var selection = LlmProviderSelectionPolicy.Evaluate(settings, "Development"); + selection.ProviderKind.Should().Be(LlmProviderKind.OpenAiCompatible); + + var provider = CreateProvider(new StubHttpMessageHandler(_ => + JsonResponse("""{"choices":[{"message":{"content":"local response"}}],"usage":{"total_tokens":3}}""")), settings); + + var result = await provider.CompleteAsync(Request()); + + result.IsDegraded.Should().BeFalse(); + result.Content.Should().Be("local response"); + } + private static OpenAiCompatibleLlmProvider CreateProvider(HttpMessageHandler handler, LlmProviderSettings settings) => new(new HttpClient(handler), settings, NullLogger.Instance); From 83a3d3902390dc5d219e1578e1f040f90b9f464d Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 04:52:43 +0100 Subject: [PATCH 3/7] Harden OpenAI-compatible streaming Signed-off-by: Chris0Jeky # Conflicts: # backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs # backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs # backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs --- .../Extensions/LlmProviderRegistration.cs | 93 ++- .../Extensions/WorkerRegistration.cs | 1 + .../Taskdeck.Api/appsettings.Development.json | 3 + backend/src/Taskdeck.Api/appsettings.json | 3 + .../Services/ChatService.cs | 36 +- .../Services/CircuitBreakerStateTracker.cs | 87 +++ .../Services/GeminiLlmProvider.cs | 4 + .../Services/ILlmProvider.cs | 11 +- .../Services/LlmProviderSelectionPolicy.cs | 64 +- .../Services/LlmProviderSettings.cs | 16 + .../Services/OllamaLlmProvider.cs | 4 + .../Services/OpenAiCompatibleLlmProvider.cs | 667 +++++++++++++----- .../Services/OpenAiLlmProvider.cs | 4 + .../LlmProviderRegistrationTests.cs | 28 + .../Services/ChatServiceTests.cs | 125 ++++ .../Services/GeminiLlmProviderTests.cs | 19 + .../LlmProviderSelectionPolicyTests.cs | 100 +++ .../Services/OllamaLlmProviderTests.cs | 19 + .../OpenAiCompatibleLlmProviderTests.cs | 362 +++++++++- .../Services/OpenAiLlmProviderTests.cs | 19 + docs/platform/CONFIGURATION_REFERENCE.md | 9 +- docs/platform/LLM_PROVIDER_SETUP_GUIDE.md | 13 +- 22 files changed, 1500 insertions(+), 187 deletions(-) diff --git a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs index b267d7210..c39012c27 100644 --- a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs @@ -11,6 +11,8 @@ namespace Taskdeck.Api.Extensions; public static class LlmProviderRegistration { + internal const string OpenAiCompatibleHttpClientName = "OpenAiCompatibleLlmProvider"; + public static IServiceCollection AddLlmProviders( this IServiceCollection services, IConfiguration configuration) @@ -87,6 +89,11 @@ public static IServiceCollection AddLlmProviders( services.AddSingleton(localhostPolicy); services.TryAddTransient(); services.TryAddSingleton(); + var egressRegistry = GetOrCreateEgressRegistry(services); + RegisterOpenAiCompatibleEgress( + egressRegistry, + llmProviderSettings, + localhostPolicy.AllowGeneralProviderLocalhost); services.AddHttpClient((sp, client) => { @@ -116,7 +123,7 @@ public static IServiceCollection AddLlmProviders( .AddPolicyHandler(openAiCircuitBreakerPolicy) .RemoveAllLoggers() .AddHttpMessageHandler(); - services.AddHttpClient((sp, client) => + services.AddHttpClient(OpenAiCompatibleHttpClientName, (sp, client) => { var settings = sp.GetRequiredService(); var timeoutSeconds = settings.OpenAiCompatible?.TimeoutSeconds > 0 ? settings.OpenAiCompatible.TimeoutSeconds : 30; @@ -137,6 +144,10 @@ public static IServiceCollection AddLlmProviders( cancellationToken) }; }) + .AddHttpMessageHandler(sp => new EgressEnvelopeHandler( + sp.GetRequiredService(), + sp.GetRequiredService>(), + nameof(OpenAiCompatibleLlmProvider))) .AddPolicyHandler(openAiCompatibleCircuitBreakerPolicy) .RemoveAllLoggers() .AddHttpMessageHandler(); @@ -208,7 +219,12 @@ public static IServiceCollection AddLlmProviders( return decision.ProviderKind switch { LlmProviderKind.OpenAi => sp.GetRequiredService(), - LlmProviderKind.OpenAiCompatible => sp.GetRequiredService(), + LlmProviderKind.OpenAiCompatible => CreateOpenAiCompatibleProvider( + sp, + settings, + circuitBreakerTracker, + circuitBreakerSettings, + localhostPolicy.AllowGeneralProviderLocalhost), LlmProviderKind.Gemini => sp.GetRequiredService(), LlmProviderKind.Ollama => sp.GetRequiredService(), _ => sp.GetRequiredService() @@ -218,6 +234,79 @@ public static IServiceCollection AddLlmProviders( return services; } + internal static SocketsHttpHandler CreateProtectedSocketsHttpHandler(bool allowLocalhostEndpoints) + { + return new SocketsHttpHandler + { + AllowAutoRedirect = false, + // A configured system proxy changes ConnectCallback's destination to the + // proxy and can tunnel to an unvalidated target. Protected clients must + // connect directly so the DNS-level callback validates the real origin. + UseProxy = false, + ConnectCallback = (context, cancellationToken) => + OutboundWebhookConnectCallback.ConnectAsync( + context, + allowLocalhostEndpoints, + cancellationToken) + }; + } + + private static OpenAiCompatibleLlmProvider CreateOpenAiCompatibleProvider( + IServiceProvider services, + LlmProviderSettings settings, + CircuitBreakerStateTracker circuitBreakerTracker, + CircuitBreakerSettings circuitBreakerSettings, + bool allowLocalhostEndpoints) + { + RegisterOpenAiCompatibleEgress( + services.GetRequiredService(), + settings, + allowLocalhostEndpoints); + return new OpenAiCompatibleLlmProvider( + services.GetRequiredService().CreateClient(OpenAiCompatibleHttpClientName), + settings, + services.GetRequiredService>(), + circuitBreakerTracker, + circuitBreakerSettings, + allowLocalhostEndpoints); + } + + private static IEgressRegistry GetOrCreateEgressRegistry(IServiceCollection services) + { + var existing = services.LastOrDefault(descriptor => descriptor.ServiceType == typeof(IEgressRegistry)); + if (existing?.ImplementationInstance is IEgressRegistry registry) + return registry; + + var created = new EgressRegistry(); + if (existing is null) + services.AddSingleton(created); + return created; + } + + internal static void RegisterOpenAiCompatibleEgress( + IEgressRegistry registry, + LlmProviderSettings settings, + bool allowLocalhostEndpoints) + { + if (!LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings( + settings, + out _, + allowLocalhostEndpoints) || + !Uri.TryCreate(settings.OpenAiCompatible.BaseUrl, UriKind.Absolute, out var endpoint)) + return; + + if (registry.GetAllEntries().Any(entry => + string.Equals(entry.Host.TrimEnd('.'), endpoint.Host.TrimEnd('.'), StringComparison.OrdinalIgnoreCase) && + string.Equals(entry.ToolOrAgentName, nameof(OpenAiCompatibleLlmProvider), StringComparison.Ordinal))) + return; + + registry.Register(new EgressEntry( + endpoint.Host, + "LLM prompt with board context and user input", + nameof(OpenAiCompatibleLlmProvider), + EgressDataClassification.UserContent)); + } + /// /// Builds a Polly circuit breaker policy for an external HTTP client. /// The circuit opens after diff --git a/backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs b/backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs index 63ce0ddf3..5e5ef3a0d 100644 --- a/backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs @@ -63,4 +63,5 @@ public static IServiceCollection AddTaskdeckWorkers( return services; } + } diff --git a/backend/src/Taskdeck.Api/appsettings.Development.json b/backend/src/Taskdeck.Api/appsettings.Development.json index 46ccbffd2..c531ab45c 100644 --- a/backend/src/Taskdeck.Api/appsettings.Development.json +++ b/backend/src/Taskdeck.Api/appsettings.Development.json @@ -40,6 +40,9 @@ "BaseUrl": "", "Model": "", "TimeoutSeconds": 30, + "MaxResponseBytes": 1048576, + "MaxSseLineBytes": 65536, + "MaxSseEventBytes": 131072, "ExtraHeaders": {} }, "Gemini": { diff --git a/backend/src/Taskdeck.Api/appsettings.json b/backend/src/Taskdeck.Api/appsettings.json index 9273cf739..cfb78731a 100644 --- a/backend/src/Taskdeck.Api/appsettings.json +++ b/backend/src/Taskdeck.Api/appsettings.json @@ -43,6 +43,9 @@ "BaseUrl": "", "Model": "", "TimeoutSeconds": 30, + "MaxResponseBytes": 1048576, + "MaxSseLineBytes": 65536, + "MaxSseEventBytes": 131072, "ExtraHeaders": {} }, "Gemini": { diff --git a/backend/src/Taskdeck.Application/Services/ChatService.cs b/backend/src/Taskdeck.Application/Services/ChatService.cs index f8704f3ff..f7180d26d 100644 --- a/backend/src/Taskdeck.Application/Services/ChatService.cs +++ b/backend/src/Taskdeck.Application/Services/ChatService.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; @@ -13,6 +14,7 @@ namespace Taskdeck.Application.Services; public class ChatService : IChatService { private const int MaxPromptLength = 4000; + private const int MaxStreamedAssistantBytes = 1_048_576; private const int MaxChecklistItemCount = 30; private static readonly Regex MentionRegex = new(@"(?[A-Za-z0-9_.-]{3,50})", RegexOptions.Compiled); private static readonly string[] PromptInjectionDenylist = @@ -599,10 +601,13 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, // Accumulate streamed content and capture usage from the final token event // so we can persist an assistant message and record quota usage after the // stream completes. - var contentBuilder = new System.Text.StringBuilder(); + var contentBuilder = new StringBuilder(); + var streamedContentBytes = 0; int? tokensUsed = null; string? provider = null; string? model = null; + var streamIsDegraded = false; + string? streamDegradedReason = null; // True once the provider has delivered at least one non-error event: the LLM call was made and // tokens flowed, so the reservation is billable even if the final usage event never arrives. var providerStreamed = false; @@ -633,6 +638,21 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, if (token.Error == null) providerStreamed = true; + var tokenBytes = Encoding.UTF8.GetByteCount(token.Token); + if (tokenBytes > MaxStreamedAssistantBytes - streamedContentBytes) + { + streamIsDegraded = true; + streamDegradedReason = "Streamed assistant response exceeded the safety limit."; + yield return new LlmTokenEvent( + string.Empty, + true, + Error: streamDegradedReason, + Provider: token.Provider ?? provider, + Model: token.Model ?? model); + break; + } + + streamedContentBytes += tokenBytes; contentBuilder.Append(token.Token); // Best-known provider/model: any event may carry them; the final usage event is // authoritative and overwrites earlier values. @@ -642,6 +662,16 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, model = token.Model; if (token.IsComplete) tokensUsed = token.TokensUsed; + if (token.IsDegraded) + { + streamIsDegraded = true; + streamDegradedReason = token.DegradedReason ?? "Provider returned a degraded stream."; + } + if (!string.IsNullOrWhiteSpace(token.Error)) + { + streamIsDegraded = true; + streamDegradedReason = token.Error; + } yield return token; } @@ -655,7 +685,9 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, sessionId, ChatMessageRole.Assistant, streamedContent, - tokenUsage: tokensUsed); + messageType: streamIsDegraded ? "degraded" : "text", + tokenUsage: tokensUsed, + degradedReason: streamIsDegraded ? streamDegradedReason : null); session.AddMessage(assistantMessage); await _unitOfWork.ChatMessages.AddAsync(assistantMessage, ct); await _unitOfWork.SaveChangesAsync(ct); diff --git a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs index a964b7188..725c89c9b 100644 --- a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs +++ b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs @@ -10,6 +10,7 @@ namespace Taskdeck.Application.Services; public sealed class CircuitBreakerStateTracker { private readonly ConcurrentDictionary _states = new(); + private readonly ConcurrentDictionary _streamingFailures = new(); /// /// Records a circuit state transition. Called by the Polly onBreak, @@ -40,6 +41,92 @@ public IReadOnlyDictionary GetAll() { return _states.TryGetValue(circuitName, out var snapshot) ? snapshot : null; } + + /// + /// Applies the configured circuit posture to failures that occur after HTTP + /// response headers. Polly cannot observe body-read, SSE-parse, or idle-timeout + /// failures when callers use ResponseHeadersRead, so providers report those + /// failures explicitly through this companion gate. + /// + public bool TryEnterStreamingRequest( + string circuitName, + CircuitBreakerSettings settings, + out string? error) + { + while (true) + { + if (!_streamingFailures.TryGetValue(circuitName, out var state)) + { + error = null; + return true; + } + + if (state.HalfOpenProbeInFlight) + { + error = $"{circuitName} streaming circuit is half-open and its probe is already in progress."; + return false; + } + + if (state.OpenUntilUtc is null) + { + error = null; + return true; + } + + var now = DateTimeOffset.UtcNow; + if (state.OpenUntilUtc > now) + { + error = $"{circuitName} streaming circuit is open after repeated response-body failures."; + return false; + } + + var halfOpen = new StreamingFailureState( + Math.Max(0, settings.FailureThreshold - 1), + null, + HalfOpenProbeInFlight: true); + if (_streamingFailures.TryUpdate(circuitName, halfOpen, state)) + { + RecordState(circuitName, CircuitState.HalfOpen); + error = null; + return true; + } + } + } + + public void RecordStreamingFailure( + string circuitName, + CircuitBreakerSettings settings, + string reason) + { + var now = DateTimeOffset.UtcNow; + var next = _streamingFailures.AddOrUpdate( + circuitName, + _ => new StreamingFailureState(1, null, HalfOpenProbeInFlight: false), + (_, existing) => existing.OpenUntilUtc is not null && existing.OpenUntilUtc > now + ? existing + : new StreamingFailureState(existing.ConsecutiveFailures + 1, null, HalfOpenProbeInFlight: false)); + + if (next.ConsecutiveFailures < settings.FailureThreshold && next.OpenUntilUtc is null) + return; + + var opened = new StreamingFailureState( + next.ConsecutiveFailures, + now.AddSeconds(settings.BreakDurationSeconds), + HalfOpenProbeInFlight: false); + _streamingFailures[circuitName] = opened; + RecordState(circuitName, CircuitState.Open, reason); + } + + public void RecordStreamingSuccess(string circuitName) + { + if (_streamingFailures.TryRemove(circuitName, out _)) + RecordState(circuitName, CircuitState.Closed); + } + + private sealed record StreamingFailureState( + int ConsecutiveFailures, + DateTimeOffset? OpenUntilUtc, + bool HalfOpenProbeInFlight); } public enum CircuitState diff --git a/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs b/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs index 5d0098b59..5e8f812b3 100644 --- a/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs @@ -444,6 +444,10 @@ public async IAsyncEnumerable StreamAsync(ChatCompletionRequest r var isLast = i == tokens.Length - 1; yield return isLast ? new LlmTokenEvent(token, true, TokensUsed: result.TokensUsed, Provider: result.Provider, Model: result.Model) + { + IsDegraded = result.IsDegraded, + DegradedReason = result.DegradedReason + } : new LlmTokenEvent(token, false); } } diff --git a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs index a26694d3a..c35e52daf 100644 --- a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs @@ -71,9 +71,14 @@ public record LlmTokenEvent( string? Error = null, int? TokensUsed = null, string? Provider = null, - string? Model = null, - bool IsDegraded = false, - string? DegradedReason = null); + string? Model = null) +{ + // Keep these out of the positional constructor. LlmTokenEvent is part of the + // public Application contract, so extending its existing constructor would + // break already-compiled consumers even though in-tree source still builds. + public bool IsDegraded { get; init; } + public string? DegradedReason { get; init; } +} public record LlmHealthStatus( bool IsAvailable, diff --git a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs index 6c2466304..408f0e300 100644 --- a/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs +++ b/backend/src/Taskdeck.Application/Services/LlmProviderSelectionPolicy.cs @@ -13,6 +13,12 @@ public sealed record LlmProviderDecision(LlmProviderKind ProviderKind, string Re public static class LlmProviderSelectionPolicy { + private static readonly HashSet ForbiddenCompatibleHeaders = new(StringComparer.OrdinalIgnoreCase) + { + "Host", "Connection", "Transfer-Encoding", "Cookie", "Cookie2", "Keep-Alive", + "TE", "Trailer", "Upgrade", "Via", "Forwarded", "Content-Length", "HTTP2-Settings" + }; + public static LlmProviderDecision Evaluate(LlmProviderSettings settings, string? environmentName) { var requestedProvider = ResolveRequestedProviderKind(settings.Provider); @@ -229,6 +235,22 @@ public static bool TryValidateOpenAiCompatibleSettings( return false; } + if (compatible.ApiKey.Contains('\r') || compatible.ApiKey.Contains('\n')) + { + error = "ApiKey must not contain line breaks."; + return false; + } + + try + { + _ = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", compatible.ApiKey.Trim()); + } + catch (FormatException) + { + error = "ApiKey cannot be serialized as a Bearer credential."; + return false; + } + if (string.IsNullOrWhiteSpace(compatible.Model)) { error = "Model is required."; @@ -242,6 +264,13 @@ public static bool TryValidateOpenAiCompatibleSettings( return false; } + if (!string.IsNullOrEmpty(baseUri.Query) || !string.IsNullOrEmpty(baseUri.Fragment) || + !string.IsNullOrEmpty(baseUri.UserInfo)) + { + error = "BaseUrl must not contain user information, a query, or a fragment."; + return false; + } + var ssrfResult = SsrfProtectionService.ValidateLlmProviderUrl(compatible.BaseUrl, allowLocalhostEndpoints); if (!ssrfResult.IsAllowed) { @@ -249,9 +278,24 @@ public static bool TryValidateOpenAiCompatibleSettings( return false; } - if (compatible.TimeoutSeconds <= 0) + if (Uri.CheckHostName(baseUri.Host) != UriHostNameType.Dns) { - error = "TimeoutSeconds must be greater than zero."; + error = "BaseUrl host must be a DNS name so it can be disclosed and enforced by the egress registry."; + return false; + } + + if (compatible.TimeoutSeconds is < 1 or > 300) + { + error = "TimeoutSeconds must be between 1 and 300."; + return false; + } + + if (compatible.MaxResponseBytes is < 1024 or > 4_194_304 || + compatible.MaxSseLineBytes is < 256 or > 262_144 || + compatible.MaxSseEventBytes is < 512 or > 524_288 || + compatible.MaxSseEventBytes > compatible.MaxResponseBytes) + { + error = "OpenAI-compatible response budgets are invalid or inconsistent."; return false; } @@ -263,9 +307,21 @@ public static bool TryValidateOpenAiCompatibleSettings( foreach (var (name, value) in compatible.ExtraHeaders) { - if (string.IsNullOrWhiteSpace(name) || string.Equals(name, "Authorization", StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrWhiteSpace(name) || + name.Contains("authorization", StringComparison.OrdinalIgnoreCase) || + name.Contains("authenticate", StringComparison.OrdinalIgnoreCase) || + name.Contains("authentication", StringComparison.OrdinalIgnoreCase) || + name.Contains("cookie", StringComparison.OrdinalIgnoreCase) || + name.Equals("X-Api-Key", StringComparison.OrdinalIgnoreCase) || + name.Equals("Api-Key", StringComparison.OrdinalIgnoreCase) || + name.Contains("Auth-Token", StringComparison.OrdinalIgnoreCase) || + name.StartsWith("Proxy-", StringComparison.OrdinalIgnoreCase) || + name.StartsWith("X-Forwarded-", StringComparison.OrdinalIgnoreCase) || + name.StartsWith("Forwarded-", StringComparison.OrdinalIgnoreCase) || + name.StartsWith("X-Taskdeck-", StringComparison.OrdinalIgnoreCase) || + ForbiddenCompatibleHeaders.Contains(name)) { - error = "ExtraHeaders must use non-Authorization header names."; + error = $"ExtraHeaders contains dangerous or reserved request header '{name}'."; return false; } diff --git a/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs b/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs index e3ec7e892..34cfa760c 100644 --- a/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs +++ b/backend/src/Taskdeck.Application/Services/LlmProviderSettings.cs @@ -70,6 +70,10 @@ public sealed class GeminiProviderSettings public sealed class OpenAiCompatibleProviderSettings { + public const int DefaultMaxResponseBytes = 1_048_576; + public const int DefaultMaxSseLineBytes = 65_536; + public const int DefaultMaxSseEventBytes = 131_072; + /// /// API key accepted by the configured OpenAI-compatible endpoint. /// @@ -88,6 +92,18 @@ public sealed class OpenAiCompatibleProviderSettings [Range(1, 300, ErrorMessage = "TimeoutSeconds must be between 1 and 300.")] public int TimeoutSeconds { get; set; } = 30; + /// Maximum UTF-8 bytes accepted from one buffered response or stream. + [Range(1024, 4_194_304)] + public int MaxResponseBytes { get; set; } = DefaultMaxResponseBytes; + + /// Maximum UTF-8 bytes accepted in one SSE line. + [Range(256, 262_144)] + public int MaxSseLineBytes { get; set; } = DefaultMaxSseLineBytes; + + /// Maximum UTF-8 bytes accepted in one assembled SSE event. + [Range(512, 524_288)] + public int MaxSseEventBytes { get; set; } = DefaultMaxSseEventBytes; + /// /// Optional request headers required by a compatible gateway (for example, /// OpenRouter's HTTP-Referer and X-Title headers). Authorization is always diff --git a/backend/src/Taskdeck.Application/Services/OllamaLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OllamaLlmProvider.cs index 7365e4553..87fec608b 100644 --- a/backend/src/Taskdeck.Application/Services/OllamaLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OllamaLlmProvider.cs @@ -148,6 +148,10 @@ public async IAsyncEnumerable StreamAsync(ChatCompletionRequest r var isLast = i == tokens.Length - 1; yield return isLast ? new LlmTokenEvent(token, true, TokensUsed: result.TokensUsed, Provider: result.Provider, Model: result.Model) + { + IsDegraded = result.IsDegraded, + DegradedReason = result.DegradedReason + } : new LlmTokenEvent(token, false); } } diff --git a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs index 68cace4f3..967316b34 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs @@ -16,21 +16,40 @@ namespace Taskdeck.Application.Services; public sealed class OpenAiCompatibleLlmProvider : ILlmProvider { private const string ProviderName = "OpenAICompatible"; + private const string CircuitName = "OpenAICompatible"; private const string BufferedStreamingFallbackReason = "Upstream endpoint rejected SSE streaming; the response was completed before emission."; private readonly HttpClient _httpClient; private readonly LlmProviderSettings _settings; private readonly ILogger _logger; + private readonly CircuitBreakerStateTracker? _circuitBreakerTracker; + private readonly CircuitBreakerSettings? _circuitBreakerSettings; + private readonly bool _allowLocalhostEndpoints; + // Retain the original public constructor for already-compiled consumers. public OpenAiCompatibleLlmProvider( HttpClient httpClient, LlmProviderSettings settings, ILogger logger) + : this(httpClient, settings, logger, null, null, false) + { + } + + public OpenAiCompatibleLlmProvider( + HttpClient httpClient, + LlmProviderSettings settings, + ILogger logger, + CircuitBreakerStateTracker? circuitBreakerTracker, + CircuitBreakerSettings? circuitBreakerSettings, + bool allowLocalhostEndpoints) { _httpClient = httpClient; _settings = settings; _logger = logger; + _circuitBreakerTracker = circuitBreakerTracker; + _circuitBreakerSettings = circuitBreakerSettings; + _allowLocalhostEndpoints = allowLocalhostEndpoints; } public async Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) @@ -42,20 +61,24 @@ public async Task CompleteAsync(ChatCompletionRequest reque return BuildFallbackResult(userMessage, "Live provider configuration is invalid."); } + using var timeout = CreateProviderTimeoutTokenSource(ct); try { var useInstructionExtraction = request.SystemPrompt is null; - var response = await SendCompletionAsync(request, stream: false, includeResponseFormat: useInstructionExtraction, ct); + var response = await SendCompletionAsync(request, includeResponseFormat: useInstructionExtraction, timeout.Token); if (!response.IsSuccessStatusCode && useInstructionExtraction && IsResponseFormatRejection(response.StatusCode)) { response.Dispose(); _logger.LogInformation("OpenAI-compatible endpoint rejected response_format; retrying with prompt-enforced JSON only."); - response = await SendCompletionAsync(request, stream: false, includeResponseFormat: false, ct); + response = await SendCompletionAsync(request, includeResponseFormat: false, timeout.Token); } using (response) { - var body = await response.Content.ReadAsStringAsync(ct); + var body = await ReadBoundedContentAsync( + response.Content, + _settings.OpenAiCompatible.MaxResponseBytes, + timeout.Token); if (!response.IsSuccessStatusCode) { _logger.LogWarning("OpenAI-compatible completion request failed with status code {StatusCode}.", (int)response.StatusCode); @@ -68,12 +91,14 @@ public async Task CompleteAsync(ChatCompletionRequest reque return BuildFallbackResult(userMessage, "Live provider response parsing failed."); } - if (string.Equals(finishReason, "length", StringComparison.OrdinalIgnoreCase) || + var finishDegradation = GetFinishReasonDegradation(finishReason); + if (finishDegradation is not null || (useInstructionExtraction && OpenAiLlmProvider.LooksLikeTruncatedJson(content))) { return new LlmCompletionResult( content, tokensUsed, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault(), - IsDegraded: true, DegradedReason: "Response was truncated"); + IsDegraded: true, + DegradedReason: finishDegradation ?? "Response was truncated."); } if (useInstructionExtraction && LlmInstructionExtractionPrompt.TryParseStructuredResponse( @@ -93,10 +118,20 @@ public async Task CompleteAsync(ChatCompletionRequest reque Instructions: fallbackInstructions.Count > 0 ? fallbackInstructions : null); } } - catch (OperationCanceledException) + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } + catch (OperationCanceledException) + { + _logger.LogWarning("OpenAI-compatible completion exceeded its configured response deadline."); + return BuildFallbackResult(userMessage, "Live provider request timed out."); + } + catch (LlmProviderResponseLimitException ex) + { + _logger.LogWarning("OpenAI-compatible completion exceeded a response budget: {Limit}", ex.Message); + return BuildFallbackResult(userMessage, "Live provider response exceeded a safety limit."); + } catch (Exception ex) { _logger.LogError( @@ -112,150 +147,227 @@ public async IAsyncEnumerable StreamAsync( { if (!TryValidateSettings(out var validationError)) { - yield return new LlmTokenEvent(string.Empty, true, Error: $"Live provider configuration is invalid: {validationError}", - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield return TerminalError($"Live provider configuration is invalid: {validationError}"); yield break; } - HttpResponseMessage? response = null; - try + if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null && + !_circuitBreakerTracker.TryEnterStreamingRequest(CircuitName, _circuitBreakerSettings, out var circuitError)) + { + yield return TerminalError(circuitError ?? "OpenAI-compatible streaming circuit is open."); + yield break; + } + + using var timeout = CreateProviderTimeoutTokenSource(ct); + await using var enumerator = StreamCoreAsync(request, timeout.Token).GetAsyncEnumerator(timeout.Token); + var emittedTerminal = false; + + while (true) { - // Streaming chat is conversational text, not the buffered instruction-extraction - // contract. Suppress response_format and the JSON-only default system prompt so the - // client receives the vendor's real text deltas rather than a JSON envelope. - var streamingRequest = request.SystemPrompt is null ? request with { SystemPrompt = string.Empty } : request; - using var message = CreateRequestMessage(streamingRequest, stream: true, includeResponseFormat: false); - response = await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); - - if (!response.IsSuccessStatusCode) + LlmTokenEvent? failure = null; + bool moved; + try { - var statusCode = response.StatusCode; - response.Dispose(); - response = null; - if (IsStreamingRejection(statusCode)) - { - await foreach (var fallback in EmitBufferedFallbackAsync(request, ct)) - yield return fallback; - yield break; - } + moved = await enumerator.MoveNextAsync(); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) + { + moved = false; + failure = TerminalError("OpenAI-compatible SSE response timed out after headers were received."); + } + catch (Exception ex) + { + _logger.LogWarning( + "OpenAI-compatible SSE response failed after request dispatch. {ExceptionSummary}", + SensitiveDataRedactor.SummarizeException(ex)); + moved = false; + failure = TerminalError(ex is LlmProviderResponseLimitException + ? "OpenAI-compatible SSE response exceeded a safety limit." + : "OpenAI-compatible SSE transport failed before completion."); + } - yield return new LlmTokenEvent(string.Empty, true, - Error: $"OpenAI-compatible streaming request failed with status code {(int)statusCode}.", - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + if (failure is not null) + { + RecordStreamingFailure(failure.Error!); + yield return failure; yield break; } - var mediaType = response.Content.Headers.ContentType?.MediaType; - if (!string.Equals(mediaType, "text/event-stream", StringComparison.OrdinalIgnoreCase)) + if (!moved) + break; + + var current = enumerator.Current; + if (current.IsComplete) { - var body = await response.Content.ReadAsStringAsync(ct); - response.Dispose(); - response = null; - if (TryParseResponse(body, out var content, out var tokensUsed, out _)) - { - yield return new LlmTokenEvent(content, true, TokensUsed: tokensUsed, Provider: ProviderName, - Model: GetConfiguredModelOrDefault(), IsDegraded: true, - DegradedReason: BufferedStreamingFallbackReason); - } + emittedTerminal = true; + if (IsStreamingFailure(current)) + RecordStreamingFailure(current.Error ?? current.DegradedReason ?? "Degraded SSE completion."); else - { - yield return new LlmTokenEvent(string.Empty, true, - Error: "OpenAI-compatible endpoint returned a non-SSE response that could not be parsed.", - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); - } - yield break; + RecordStreamingSuccess(); } - await using var stream = await response.Content.ReadAsStreamAsync(ct); - using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); - var streamedContent = new StringBuilder(); - var data = new StringBuilder(); - var completed = false; + yield return current; + if (current.IsComplete) + yield break; + } - while (await reader.ReadLineAsync(ct) is { } line) + if (!emittedTerminal) + { + var failure = TerminalError("OpenAI-compatible SSE stream ended before a completion marker."); + RecordStreamingFailure(failure.Error!); + yield return failure; + } + } + + private async IAsyncEnumerable StreamCoreAsync( + ChatCompletionRequest request, + [EnumeratorCancellation] CancellationToken ct) + { + // Streaming chat is conversational text, not the buffered instruction-extraction + // contract. Use this same effective request for a rejected-stream fallback. + var streamingRequest = request.SystemPrompt is null ? request with { SystemPrompt = string.Empty } : request; + using var message = CreateRequestMessage(streamingRequest, stream: true, includeResponseFormat: false); + using var response = await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); + + if (!response.IsSuccessStatusCode) + { + if (IsStreamingRejection(response.StatusCode)) { - ct.ThrowIfCancellationRequested(); - if (line.Length == 0) - { - if (data.Length == 0) - continue; - - var eventData = data.ToString(); - data.Clear(); - var parsed = TryParseSseEvent(eventData); - if (parsed.Error is not null) - { - yield return new LlmTokenEvent(string.Empty, true, Error: parsed.Error, - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); - yield break; - } - - if (!string.IsNullOrEmpty(parsed.Delta)) - { - streamedContent.Append(parsed.Delta); - yield return new LlmTokenEvent(parsed.Delta, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault()); - } - - if (parsed.IsDone) - { - completed = true; - yield return new LlmTokenEvent(string.Empty, true, - TokensUsed: parsed.TokensUsed ?? EstimateTokens(streamedContent.ToString()), - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); - yield break; - } + await foreach (var fallback in EmitBufferedFallbackAsync(streamingRequest, ct)) + yield return fallback; + yield break; + } - continue; - } + yield return TerminalError( + $"OpenAI-compatible streaming request failed with status code {(int)response.StatusCode}."); + yield break; + } - if (line.StartsWith("data:", StringComparison.Ordinal)) + var compatible = _settings.OpenAiCompatible; + var mediaType = response.Content.Headers.ContentType?.MediaType; + if (!string.Equals(mediaType, "text/event-stream", StringComparison.OrdinalIgnoreCase)) + { + var body = await ReadBoundedContentAsync(response.Content, compatible.MaxResponseBytes, ct); + if (TryParseResponse(body, out var content, out var tokensUsed, out var finishReason)) + { + var finishDegradation = GetFinishReasonDegradation(finishReason); + yield return new LlmTokenEvent( + content, + true, + TokensUsed: tokensUsed, + Provider: ProviderName, + Model: GetConfiguredModelOrDefault()) { - if (data.Length > 0) - data.Append('\n'); - data.Append(line[5..].TrimStart()); - } + IsDegraded = true, + DegradedReason = finishDegradation is null + ? BufferedStreamingFallbackReason + : $"{BufferedStreamingFallbackReason} {finishDegradation}" + }; + } + else + { + yield return TerminalError( + "OpenAI-compatible endpoint returned a non-SSE response that could not be parsed."); } + yield break; + } + + await using var stream = await response.Content.ReadAsStreamAsync(ct); + using var reader = new StreamReader( + stream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true), + detectEncodingFromByteOrderMarks: true, + bufferSize: 4096, + leaveOpen: false); + var eventData = new StringBuilder(); + var eventBytes = 0; + var responseBytes = 0; + string? finishReasonSeen = null; + int? tokensUsedSeen = null; + + while (true) + { + var line = await ReadBoundedLineAsync(reader, compatible.MaxSseLineBytes, ct); + if (line.Text is null) + break; + + responseBytes = checked(responseBytes + line.Utf8Bytes + 1); + if (responseBytes > compatible.MaxResponseBytes) + throw new LlmProviderResponseLimitException("aggregate SSE response byte budget exceeded"); - // A compliant SSE sender terminates each event with a blank line, but - // process a final unterminated data field as well so a connection close - // cannot turn an otherwise complete response into a false parse error. - if (data.Length > 0) + if (line.Text.Length == 0) { - var parsed = TryParseSseEvent(data.ToString()); + if (eventData.Length == 0) + continue; + + var parsed = TryParseSseEvent(eventData.ToString()); + eventData.Clear(); + eventBytes = 0; + if (parsed.Error is not null) { - yield return new LlmTokenEvent(string.Empty, true, Error: parsed.Error, - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield return TerminalError(parsed.Error); yield break; } + if (parsed.TokensUsed is not null) + tokensUsedSeen = parsed.TokensUsed; + if (parsed.FinishReason is not null) + finishReasonSeen = parsed.FinishReason; if (!string.IsNullOrEmpty(parsed.Delta)) - { - streamedContent.Append(parsed.Delta); yield return new LlmTokenEvent(parsed.Delta, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault()); - } - if (parsed.IsDone) + if (parsed.IsDoneMarker) { - completed = true; - yield return new LlmTokenEvent(string.Empty, true, - TokensUsed: parsed.TokensUsed ?? EstimateTokens(streamedContent.ToString()), - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + yield return BuildTerminalCompletion(finishReasonSeen, tokensUsedSeen); yield break; } - } - if (!completed) - { - yield return new LlmTokenEvent(string.Empty, true, - Error: "OpenAI-compatible SSE stream ended before a completion marker.", - Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + continue; } + + if (!line.Text.StartsWith("data:", StringComparison.Ordinal)) + continue; + + var dataField = line.Text[5..].TrimStart(); + var fieldBytes = Encoding.UTF8.GetByteCount(dataField); + eventBytes = checked(eventBytes + fieldBytes + (eventData.Length > 0 ? 1 : 0)); + if (eventBytes > compatible.MaxSseEventBytes) + throw new LlmProviderResponseLimitException("single SSE event byte budget exceeded"); + if (eventData.Length > 0) + eventData.Append('\n'); + eventData.Append(dataField); } - finally + + // Accept a final data field without the conventional blank separator. + if (eventData.Length > 0) { - response?.Dispose(); + var parsed = TryParseSseEvent(eventData.ToString()); + if (parsed.Error is not null) + { + yield return TerminalError(parsed.Error); + yield break; + } + if (parsed.TokensUsed is not null) + tokensUsedSeen = parsed.TokensUsed; + if (parsed.FinishReason is not null) + finishReasonSeen = parsed.FinishReason; + if (!string.IsNullOrEmpty(parsed.Delta)) + yield return new LlmTokenEvent(parsed.Delta, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault()); + if (parsed.IsDoneMarker) + { + yield return BuildTerminalCompletion(finishReasonSeen, tokensUsedSeen); + yield break; + } } + + yield return TerminalError( + "OpenAI-compatible SSE stream ended before a completion marker.", + tokensUsedSeen); } public Task GetHealthAsync(CancellationToken ct = default) @@ -266,10 +378,29 @@ public Task GetHealthAsync(CancellationToken ct = default) public async Task ProbeAsync(CancellationToken ct = default) { - var result = await CompleteAsync(new ChatCompletionRequest( - [new ChatCompletionMessage("user", "Reply with exactly: OK")], MaxTokens: 4, Temperature: 0, SystemPrompt: string.Empty), ct); - return new LlmHealthStatus(!result.IsDegraded, ProviderName, - result.IsDegraded ? result.DegradedReason : null, GetConfiguredModelOrDefault(), IsProbed: true); + try + { + var result = await CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("user", "Reply with exactly: OK")], + MaxTokens: 4, + Temperature: 0, + SystemPrompt: string.Empty), ct); + return new LlmHealthStatus( + !result.IsDegraded, + ProviderName, + result.IsDegraded ? result.DegradedReason : null, + GetConfiguredModelOrDefault(), + IsProbed: true); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return new LlmHealthStatus( + false, + ProviderName, + "Provider probe timed out.", + GetConfiguredModelOrDefault(), + IsProbed: true); + } } private async IAsyncEnumerable EmitBufferedFallbackAsync( @@ -277,31 +408,46 @@ private async IAsyncEnumerable EmitBufferedFallbackAsync( [EnumeratorCancellation] CancellationToken ct) { var result = await CompleteAsync(request, ct); - yield return new LlmTokenEvent(result.Content, true, TokensUsed: result.TokensUsed, Provider: ProviderName, - Model: result.Model, IsDegraded: true, - DegradedReason: result.IsDegraded + yield return new LlmTokenEvent( + result.Content, + true, + TokensUsed: result.TokensUsed, + Provider: ProviderName, + Model: result.Model) + { + IsDegraded = true, + DegradedReason = result.IsDegraded ? $"{BufferedStreamingFallbackReason} {result.DegradedReason}" - : BufferedStreamingFallbackReason); + : BufferedStreamingFallbackReason + }; } private async Task SendCompletionAsync( ChatCompletionRequest request, - bool stream, bool includeResponseFormat, CancellationToken ct) { - using var message = CreateRequestMessage(request, stream, includeResponseFormat); - return await _httpClient.SendAsync(message, ct); + using var message = CreateRequestMessage(request, stream: false, includeResponseFormat); + return await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); } - private bool TryValidateSettings(out string error) => - LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings( + private bool TryValidateSettings(out string error) + { + if (!_settings.EnableLiveProviders) + { + error = "Live providers are disabled."; + return false; + } + if (!string.Equals(_settings.Provider, "OpenAiCompatible", StringComparison.OrdinalIgnoreCase)) + { + error = "OpenAI-compatible is not the selected provider."; + return false; + } + return LlmProviderSelectionPolicy.TryValidateOpenAiCompatibleSettings( _settings, out error, - // The selected provider can use the narrow development localhost - // exception already granted by the selection policy. In production, - // selection and the DNS-level ConnectCallback still reject it. - allowLocalhostEndpoints: _settings.AllowLiveProvidersInDevelopment); + _allowLocalhostEndpoints); + } private HttpRequestMessage CreateRequestMessage(ChatCompletionRequest request, bool stream, bool includeResponseFormat) { @@ -310,9 +456,7 @@ private HttpRequestMessage CreateRequestMessage(ChatCompletionRequest request, b message.Headers.Authorization = new AuthenticationHeaderValue("Bearer", compatible.ApiKey.Trim()); LlmRequestAttributionMapper.AddAttributionHeaders(message, request.Attribution); foreach (var (name, value) in compatible.ExtraHeaders) - { message.Headers.Add(name, value); - } message.Content = JsonContent.Create(BuildRequestPayload(request, stream, includeResponseFormat)); return message; } @@ -320,7 +464,8 @@ private HttpRequestMessage CreateRequestMessage(ChatCompletionRequest request, b private object BuildRequestPayload(ChatCompletionRequest request, bool stream, bool includeResponseFormat) { var systemPrompt = LlmSystemPromptBuilder.BuildEffectiveSystemPrompt( - request.SystemPrompt ?? LlmInstructionExtractionPrompt.SystemPrompt, request.BoardContext); + request.SystemPrompt ?? LlmInstructionExtractionPrompt.SystemPrompt, + request.BoardContext); var messages = new List(); if (!string.IsNullOrEmpty(systemPrompt)) messages.Add(new { role = "system", content = systemPrompt }); @@ -334,6 +479,8 @@ private object BuildRequestPayload(ChatCompletionRequest request, bool stream, b ["temperature"] = request.Temperature, ["stream"] = stream }; + if (stream) + payload["stream_options"] = new { include_usage = true }; if (includeResponseFormat) payload["response_format"] = new { type = "json_object" }; if (request.Attribution is not null) @@ -341,34 +488,70 @@ private object BuildRequestPayload(ChatCompletionRequest request, bool stream, b return payload; } - private static SseEventParseResult TryParseSseEvent(string eventData) + private static SseEventParseResult TryParseSseEvent(string data) { - if (string.Equals(eventData.Trim(), "[DONE]", StringComparison.Ordinal)) - return new SseEventParseResult(IsDone: true); + if (string.Equals(data.Trim(), "[DONE]", StringComparison.Ordinal)) + return new SseEventParseResult(IsDoneMarker: true); try { - using var document = JsonDocument.Parse(eventData); + using var document = JsonDocument.Parse(data); var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + return new SseEventParseResult(Error: "OpenAI-compatible SSE event root was not an object."); if (root.TryGetProperty("error", out _)) return new SseEventParseResult(Error: "OpenAI-compatible SSE stream reported an upstream error."); - if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array || choices.GetArrayLength() == 0) - return new SseEventParseResult(Error: "OpenAI-compatible SSE event did not contain choices."); + + int? tokensUsed = null; + if (root.TryGetProperty("usage", out var usage)) + { + if (usage.ValueKind != JsonValueKind.Null && + (usage.ValueKind != JsonValueKind.Object || + !usage.TryGetProperty("total_tokens", out var total) || + total.ValueKind != JsonValueKind.Number || + !total.TryGetInt32(out var parsedTokens) || parsedTokens < 0)) + { + return new SseEventParseResult(Error: "OpenAI-compatible SSE usage metadata was malformed."); + } + if (usage.ValueKind == JsonValueKind.Object) + tokensUsed = usage.GetProperty("total_tokens").GetInt32(); + } + + if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array) + return new SseEventParseResult(Error: "OpenAI-compatible SSE event did not contain a choices array."); + if (choices.GetArrayLength() == 0) + return tokensUsed is not null + ? new SseEventParseResult(TokensUsed: tokensUsed) + : new SseEventParseResult(Error: "OpenAI-compatible SSE event contained no choices or usage."); var first = choices[0]; - var delta = first.TryGetProperty("delta", out var deltaElement) && - deltaElement.TryGetProperty("content", out var contentElement) && - contentElement.ValueKind == JsonValueKind.String - ? contentElement.GetString() - : null; - var finishReason = first.TryGetProperty("finish_reason", out var finish) && finish.ValueKind != JsonValueKind.Null - ? finish.GetString() - : null; - var tokensUsed = root.TryGetProperty("usage", out var usage) && - usage.TryGetProperty("total_tokens", out var total) && total.TryGetInt32(out var parsedTokens) - ? (int?)parsedTokens - : null; - return new SseEventParseResult(delta, !string.IsNullOrEmpty(finishReason), tokensUsed); + if (first.ValueKind != JsonValueKind.Object) + return new SseEventParseResult(Error: "OpenAI-compatible SSE choice was not an object."); + + string? delta = null; + if (first.TryGetProperty("delta", out var deltaElement)) + { + if (deltaElement.ValueKind != JsonValueKind.Object) + return new SseEventParseResult(Error: "OpenAI-compatible SSE delta was not an object."); + if (deltaElement.TryGetProperty("content", out var contentElement)) + { + if (contentElement.ValueKind == JsonValueKind.String) + delta = contentElement.GetString(); + else if (contentElement.ValueKind != JsonValueKind.Null) + return new SseEventParseResult(Error: "OpenAI-compatible SSE delta content was not text."); + } + } + + string? finishReason = null; + if (first.TryGetProperty("finish_reason", out var finish)) + { + if (finish.ValueKind == JsonValueKind.String) + finishReason = finish.GetString(); + else if (finish.ValueKind != JsonValueKind.Null) + return new SseEventParseResult(Error: "OpenAI-compatible SSE finish_reason was not text or null."); + } + + return new SseEventParseResult(delta, FinishReason: finishReason, TokensUsed: tokensUsed); } catch (JsonException) { @@ -385,20 +568,42 @@ private static bool TryParseResponse(string body, out string content, out int to { using var document = JsonDocument.Parse(body); var root = document.RootElement; - if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array || choices.GetArrayLength() == 0) + if (root.ValueKind != JsonValueKind.Object || + !root.TryGetProperty("choices", out var choices) || + choices.ValueKind != JsonValueKind.Array || + choices.GetArrayLength() == 0 || + choices[0].ValueKind != JsonValueKind.Object) return false; var choice = choices[0]; - if (!choice.TryGetProperty("message", out var message) || !message.TryGetProperty("content", out var contentElement)) + if (!choice.TryGetProperty("message", out var message) || + message.ValueKind != JsonValueKind.Object || + !message.TryGetProperty("content", out var contentElement) || + contentElement.ValueKind != JsonValueKind.String) return false; content = contentElement.GetString() ?? string.Empty; if (string.IsNullOrWhiteSpace(content)) return false; - finishReason = choice.TryGetProperty("finish_reason", out var finish) && finish.ValueKind == JsonValueKind.String - ? finish.GetString() - : null; - tokensUsed = root.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var total) && total.TryGetInt32(out var parsed) - ? parsed - : EstimateTokens(content); + + if (choice.TryGetProperty("finish_reason", out var finish)) + { + if (finish.ValueKind == JsonValueKind.String) + finishReason = finish.GetString(); + else if (finish.ValueKind != JsonValueKind.Null) + return false; + } + + if (root.TryGetProperty("usage", out var usage)) + { + if (usage.ValueKind != JsonValueKind.Object || + !usage.TryGetProperty("total_tokens", out var total) || + total.ValueKind != JsonValueKind.Number || + !total.TryGetInt32(out tokensUsed) || tokensUsed < 0) + return false; + } + else + { + tokensUsed = EstimateTokens(content); + } return true; } catch (JsonException) @@ -407,6 +612,119 @@ private static bool TryParseResponse(string body, out string content, out int to } } + private static async Task ReadBoundedContentAsync( + HttpContent content, + int maxBytes, + CancellationToken ct) + { + if (content.Headers.ContentLength is long length && length > maxBytes) + throw new LlmProviderResponseLimitException("Content-Length exceeded the response byte budget"); + + await using var stream = await content.ReadAsStreamAsync(ct); + using var buffer = new MemoryStream(Math.Min(maxBytes, 16_384)); + var chunk = new byte[8192]; + while (true) + { + var read = await stream.ReadAsync(chunk.AsMemory(), ct); + if (read == 0) + break; + if (buffer.Length + read > maxBytes) + throw new LlmProviderResponseLimitException("response body exceeded the response byte budget"); + buffer.Write(chunk, 0, read); + } + + return new UTF8Encoding(false, true).GetString(buffer.GetBuffer(), 0, checked((int)buffer.Length)); + } + + private static async Task ReadBoundedLineAsync( + StreamReader reader, + int maxBytes, + CancellationToken ct) + { + var line = new StringBuilder(); + var one = new char[1]; + var bytes = 0; + while (true) + { + var read = await reader.ReadAsync(one.AsMemory(), ct); + if (read == 0) + return line.Length == 0 ? new BoundedLine(null, 0) : new BoundedLine(line.ToString().TrimEnd('\r'), bytes); + if (one[0] == '\n') + return new BoundedLine(line.ToString().TrimEnd('\r'), bytes); + + bytes = checked(bytes + Encoding.UTF8.GetByteCount(one)); + if (bytes > maxBytes) + throw new LlmProviderResponseLimitException("single SSE line byte budget exceeded"); + line.Append(one[0]); + } + } + + private LlmTokenEvent BuildTerminalCompletion(string? finishReason, int? tokensUsed) + { + if (string.IsNullOrWhiteSpace(finishReason)) + return TerminalError("OpenAI-compatible SSE stream completed without a finish reason.", tokensUsed); + + var degradation = GetFinishReasonDegradation(finishReason); + return new LlmTokenEvent( + string.Empty, + true, + TokensUsed: tokensUsed, + Provider: ProviderName, + Model: GetConfiguredModelOrDefault()) + { + IsDegraded = degradation is not null, + DegradedReason = degradation + }; + } + + private LlmTokenEvent TerminalError(string error, int? tokensUsed = null) => new( + string.Empty, + true, + Error: error, + TokensUsed: tokensUsed, + Provider: ProviderName, + Model: GetConfiguredModelOrDefault()); + + private CancellationTokenSource CreateProviderTimeoutTokenSource(CancellationToken ct) + { + var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(_settings.OpenAiCompatible.TimeoutSeconds)); + return timeout; + } + + private void RecordStreamingFailure(string reason) + { + if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null) + _circuitBreakerTracker.RecordStreamingFailure(CircuitName, _circuitBreakerSettings, reason); + } + + private void RecordStreamingSuccess() + { + _circuitBreakerTracker?.RecordStreamingSuccess(CircuitName); + } + + private static bool IsStreamingFailure(LlmTokenEvent terminal) => + terminal.Error is not null || + (terminal.IsDegraded && !string.Equals( + terminal.DegradedReason, + BufferedStreamingFallbackReason, + StringComparison.Ordinal)); + + private static string? GetFinishReasonDegradation(string? finishReason) + { + if (string.Equals(finishReason, "stop", StringComparison.OrdinalIgnoreCase)) + return null; + return finishReason?.Trim().ToLowerInvariant() switch + { + "length" => "Response was truncated because the upstream token limit was reached.", + "content_filter" => "Response was stopped by the upstream content filter.", + "tool_calls" => "Response ended with unsupported upstream tool calls.", + "function_call" => "Response ended with an unsupported upstream function call.", + null or "" => null, + _ => "Response ended with a non-standard upstream finish reason." + }; + } + private static bool IsResponseFormatRejection(HttpStatusCode statusCode) => statusCode is HttpStatusCode.BadRequest or HttpStatusCode.UnprocessableEntity; @@ -414,7 +732,11 @@ private static bool IsStreamingRejection(HttpStatusCode statusCode) => statusCode is HttpStatusCode.BadRequest or HttpStatusCode.NotFound or HttpStatusCode.MethodNotAllowed or HttpStatusCode.NotImplemented or HttpStatusCode.UnprocessableEntity; - private string BuildChatCompletionsEndpoint() => $"{_settings.OpenAiCompatible.BaseUrl.TrimEnd('/')}/chat/completions"; + private Uri BuildChatCompletionsEndpoint() + { + var baseUrl = _settings.OpenAiCompatible.BaseUrl.Trim().TrimEnd('/') + "/"; + return new Uri(new Uri(baseUrl, UriKind.Absolute), "chat/completions"); + } private string GetConfiguredModelOrDefault() => string.IsNullOrWhiteSpace(_settings.OpenAiCompatible?.Model) ? "openai-compatible-unknown-model" @@ -443,9 +765,26 @@ private LlmCompletionResult BuildFallbackResult(string userMessage, string reaso var content = isActionable ? $"I can help with that. I'll create a proposal to {actionIntent}. ({reason})" : $"I can help with that request. ({reason})"; - return new LlmCompletionResult(content, EstimateTokens(userMessage) + EstimateTokens(content), isActionable, actionIntent, - ProviderName, GetConfiguredModelOrDefault(), true, reason, instructions.Count > 0 ? instructions : null); + return new LlmCompletionResult( + content, + EstimateTokens(userMessage) + EstimateTokens(content), + isActionable, + actionIntent, + ProviderName, + GetConfiguredModelOrDefault(), + true, + reason, + instructions.Count > 0 ? instructions : null); } - private sealed record SseEventParseResult(string? Delta = null, bool IsDone = false, int? TokensUsed = null, string? Error = null); + private sealed record SseEventParseResult( + string? Delta = null, + bool IsDoneMarker = false, + string? FinishReason = null, + int? TokensUsed = null, + string? Error = null); + + private sealed record BoundedLine(string? Text, int Utf8Bytes); + + private sealed class LlmProviderResponseLimitException(string message) : IOException(message); } diff --git a/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs index 19998109e..422091cd9 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs @@ -423,6 +423,10 @@ public async IAsyncEnumerable StreamAsync(ChatCompletionRequest r var isLast = i == tokens.Length - 1; yield return isLast ? new LlmTokenEvent(token, true, TokensUsed: result.TokensUsed, Provider: result.Provider, Model: result.Model) + { + IsDegraded = result.IsDegraded, + DegradedReason = result.DegradedReason + } : new LlmTokenEvent(token, false); } } diff --git a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs index 6441f565a..cfdd05d75 100644 --- a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Http; using Taskdeck.Api.Extensions; using Taskdeck.Application.Services; using Xunit; @@ -36,6 +37,28 @@ public void AddLlmProviders_ResolvesOpenAiCompatibleProvider_WhenSelectionIsVali provider.GetRequiredService() .Should().BeOfType(); + provider.GetService().Should().BeNull( + "the selector, not a directly resolvable concrete transport, owns the live-provider decision"); + provider.GetRequiredService().GetAllEntries() + .Should().ContainSingle(entry => + entry.Host == "api.groq.com" && + entry.ToolOrAgentName == nameof(OpenAiCompatibleLlmProvider)); + var handler = provider.GetRequiredService() + .CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); + EnumeratePipeline(handler).Should().Contain(item => item is EgressEnvelopeHandler, + "the configured disclosure entry must also be enforced on the compatible client"); + } + + [Fact] + public void ProtectedSocketHandlers_DisableSystemProxyBypass() + { + using var llmHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler(false); + using var webhookHandler = WorkerRegistration.CreateProtectedWebhookHandler(false); + + llmHandler.UseProxy.Should().BeFalse(); + llmHandler.AllowAutoRedirect.Should().BeFalse(); + webhookHandler.UseProxy.Should().BeFalse(); + webhookHandler.AllowAutoRedirect.Should().BeFalse(); } [Theory] @@ -364,4 +387,9 @@ private sealed class TestWebHostEnvironment(string environmentName) : IWebHostEn public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider(); } + private static IEnumerable EnumeratePipeline(HttpMessageHandler root) + { + for (var current = root; current is not null; current = (current as DelegatingHandler)?.InnerHandler) + yield return current; + } } diff --git a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs index 6c2232b8b..2b5b725f0 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs @@ -1987,12 +1987,137 @@ public async Task StreamResponseAsync_ProviderYieldsOnlyErrorEvent_ShouldRelease Times.Never); } + [Theory] + [InlineData(false, "Response was stopped by the upstream content filter.")] + [InlineData(true, "OpenAI-compatible SSE transport failed before completion.")] + public async Task StreamResponseAsync_TerminalDegradedOrError_PersistsPartialHistoryAsDegraded( + bool terminalError, + string expectedReason) + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Stream terminal state persistence"); + ChatMessage? persisted = null; + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, default)) + .ReturnsAsync(session); + _chatMessageRepoMock + .Setup(r => r.AddAsync(It.IsAny(), default)) + .ReturnsAsync((ChatMessage message, CancellationToken _) => + { + persisted = message; + return message; + }); + _llmProviderMock + .Setup(p => p.StreamAsync(It.IsAny(), default)) + .Returns(TerminalStateStream(terminalError, expectedReason)); + + await foreach (var _ in _service.StreamResponseAsync(session.Id, userId, default)) { } + + persisted.Should().NotBeNull(); + persisted!.Content.Should().Be("partial response"); + persisted.MessageType.Should().Be("degraded"); + persisted.DegradedReason.Should().Be(expectedReason); + } + + [Fact] + public async Task StreamResponseAsync_UsageAbsent_CommitsReservationEstimate() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Stream missing usage quota"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, default)) + .ReturnsAsync(session); + _chatMessageRepoMock + .Setup(r => r.AddAsync(It.IsAny(), default)) + .ReturnsAsync((ChatMessage message, CancellationToken _) => message); + _llmProviderMock + .Setup(p => p.StreamAsync(It.IsAny(), default)) + .Returns(StreamWithoutUsage()); + + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, default)) + .ReturnsAsync(new DTOs.QuotaReservationDto( + true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var serviceWithQuota = new ChatService( + _unitOfWorkMock.Object, + _llmProviderMock.Object, + _plannerMock.Object, + _proposalServiceMock.Object, + _policyEngineMock.Object, + _notificationServiceMock.Object, + _authorizationServiceMock.Object, + quotaService: quotaMock.Object); + + await foreach (var _ in serviceWithQuota.StreamResponseAsync(session.Id, userId, default)) { } + + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 2000, + 0, + CancellationToken.None), Times.Once); + quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); + } + private static async IAsyncEnumerable ErrorOnlyStream() { yield return new LlmTokenEvent(string.Empty, true, Error: "provider unavailable"); await Task.CompletedTask; } + private static async IAsyncEnumerable TerminalStateStream( + bool terminalError, + string reason) + { + yield return new LlmTokenEvent( + "partial response", + false, + Provider: "OpenAICompatible", + Model: "vendor/model"); + if (terminalError) + { + yield return new LlmTokenEvent( + string.Empty, + true, + Error: reason, + Provider: "OpenAICompatible", + Model: "vendor/model"); + } + else + { + yield return new LlmTokenEvent( + string.Empty, + true, + TokensUsed: 7, + Provider: "OpenAICompatible", + Model: "vendor/model") + { + IsDegraded = true, + DegradedReason = reason + }; + } + await Task.CompletedTask; + } + + private static async IAsyncEnumerable StreamWithoutUsage() + { + yield return new LlmTokenEvent( + "hello", + false, + Provider: "OpenAICompatible", + Model: "vendor/model"); + yield return new LlmTokenEvent( + string.Empty, + true, + Provider: "OpenAICompatible", + Model: "vendor/model"); + await Task.CompletedTask; + } + private static async IAsyncEnumerable StreamEvents() { yield return new LlmTokenEvent("token", true, TokensUsed: 10, Provider: "Mock", Model: "mock-default"); diff --git a/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs index 5a67a1c83..44b0fdb96 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs @@ -470,6 +470,25 @@ [new ChatCompletionMessage("User", "tell me something")], result.IsActionable.Should().BeFalse(); } + [Fact] + public async Task StreamAsync_PropagatesBufferedCompletionDegradationMetadata() + { + var settings = BuildSettings(); + settings.Gemini.ApiKey = string.Empty; + var provider = new GeminiLlmProvider( + new HttpClient(), settings, NullLogger.Instance); + + var events = new List(); + await foreach (var item in provider.StreamAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "hello")], + SystemPrompt: string.Empty))) + events.Add(item); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].IsDegraded.Should().BeTrue(); + events[^1].DegradedReason.Should().Contain("configuration"); + } + [Theory] [InlineData("{\"reply\":\"incomplete", true)] [InlineData("{}", false)] diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs index 2548b92ea..c2a01e024 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmProviderSelectionPolicyTests.cs @@ -168,6 +168,106 @@ public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleExtraHeaderIsInvalidOr result.Reason.Should().Contain("invalid or restricted"); } + [Theory] + [InlineData("Host")] + [InlineData("Proxy-Authorization")] + [InlineData("Connection")] + [InlineData("Transfer-Encoding")] + [InlineData("Cookie")] + [InlineData("Set-Cookie")] + [InlineData("X-Authorization")] + [InlineData("Authentication-Info")] + [InlineData("WWW-Authenticate")] + [InlineData("X-Api-Key")] + [InlineData("X-Auth-Token")] + [InlineData("X-Forwarded-Host")] + [InlineData("X-Taskdeck-Correlation-Id")] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleExtraHeaderIsDangerous(string headerName) + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + ExtraHeaders = new Dictionary { [headerName] = "value" } + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("dangerous or reserved"); + } + + [Theory] + [InlineData("test\r\nX-Evil: yes", "ApiKey", null)] + [InlineData("test-compatible-key", "BaseUrl", "https://api.groq.com/openai/v1?target=other")] + [InlineData("test-compatible-key", "BaseUrl", "https://api.groq.com/openai/v1#fragment")] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleCredentialOrBaseUrlIsAmbiguous( + string apiKey, + string expectedReason, + string? baseUrl = null) + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = apiKey, + BaseUrl = baseUrl ?? "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant" + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain(expectedReason); + } + + [Fact] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleTimeoutExceedsDeclaredMaximum() + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + TimeoutSeconds = 301 + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("between 1 and 300"); + } + + [Fact] + public void Evaluate_ShouldSelectMock_WhenOpenAiCompatibleResponseBudgetsAreInconsistent() + { + var settings = BuildValidSettings(); + settings.EnableLiveProviders = true; + settings.Provider = "OpenAICompatible"; + settings.OpenAiCompatible = new OpenAiCompatibleProviderSettings + { + ApiKey = "test-compatible-key", + BaseUrl = "https://api.groq.com/openai/v1", + Model = "llama-3.1-8b-instant", + MaxResponseBytes = 1024, + MaxSseLineBytes = 512, + MaxSseEventBytes = 2048 + }; + + var result = LlmProviderSelectionPolicy.Evaluate(settings, "Production"); + + result.ProviderKind.Should().Be(LlmProviderKind.Mock); + result.Reason.Should().Contain("budgets"); + } + [Fact] public void Evaluate_ShouldSelectMock_WhenGeminiConfigurationIsInvalid() { diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OllamaLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OllamaLlmProviderTests.cs index 106d57dac..8c17844a1 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OllamaLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OllamaLlmProviderTests.cs @@ -756,6 +756,25 @@ private static LlmProviderSettings BuildSettings() }; } + [Fact] + public async Task StreamAsync_PropagatesBufferedCompletionDegradationMetadata() + { + var settings = BuildSettings(); + settings.Ollama.Model = string.Empty; + var provider = new OllamaLlmProvider( + new HttpClient(), settings, NullLogger.Instance); + + var events = new List(); + await foreach (var item in provider.StreamAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "hello")], + SystemPrompt: string.Empty))) + events.Add(item); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].IsDegraded.Should().BeTrue(); + events[^1].DegradedReason.Should().Contain("configuration"); + } + /// /// Settings used for selection-policy tests. The Ollama BaseUrl defaults to /// http://localhost:11434, which requires allowLocalhostEndpoints: true diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs index 6a521642f..7e4ce5495 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs @@ -22,8 +22,10 @@ public async Task StreamAsync_ParsesRealSseDeltas_AndSendsStreamTrue() requestBody = await request.Content!.ReadAsStringAsync(ct); extraHeader = request.Headers.GetValues("X-Title").Single(); return SseResponse( - "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"},\"finish_reason\":null}]}\n\n" + - "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"},\"finish_reason\":\"stop\"}],\"usage\":{\"total_tokens\":7}}\n\n"); + "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"},\"finish_reason\":null}],\"usage\":null}\n\n" + + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"},\"finish_reason\":\"stop\"}],\"usage\":null}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":7}}\n\n" + + "data: [DONE]\n\n"); }); var provider = CreateProvider(handler, settings); @@ -35,6 +37,7 @@ public async Task StreamAsync_ParsesRealSseDeltas_AndSendsStreamTrue() events[^1].IsDegraded.Should().BeFalse(); using var payload = JsonDocument.Parse(requestBody!); payload.RootElement.GetProperty("stream").GetBoolean().Should().BeTrue(); + payload.RootElement.GetProperty("stream_options").GetProperty("include_usage").GetBoolean().Should().BeTrue(); payload.RootElement.TryGetProperty("response_format", out _).Should().BeFalse( "streaming chat sends readable deltas rather than the buffered extraction envelope"); extraHeader.Should().Be("Taskdeck tests"); @@ -94,6 +97,9 @@ public async Task StreamAsync_WhenEndpointRejectsStreaming_EmitsBufferedFallback using var bufferedPayload = JsonDocument.Parse(requests[1]); streamedPayload.RootElement.GetProperty("stream").GetBoolean().Should().BeTrue(); bufferedPayload.RootElement.GetProperty("stream").GetBoolean().Should().BeFalse(); + bufferedPayload.RootElement.TryGetProperty("response_format", out _).Should().BeFalse(); + bufferedPayload.RootElement.GetProperty("messages").EnumerateArray() + .Should().OnlyContain(item => item.GetProperty("role").GetString() != "system"); } [Fact] @@ -143,7 +149,8 @@ public async Task CompleteAsync_AllowsTheDevelopmentLocalhostEndpointSelectedByP selection.ProviderKind.Should().Be(LlmProviderKind.OpenAiCompatible); var provider = CreateProvider(new StubHttpMessageHandler(_ => - JsonResponse("""{"choices":[{"message":{"content":"local response"}}],"usage":{"total_tokens":3}}""")), settings); + JsonResponse("""{"choices":[{"message":{"content":"local response"}}],"usage":{"total_tokens":3}}""")), settings, + allowLocalhostEndpoints: true); var result = await provider.CompleteAsync(Request()); @@ -151,10 +158,312 @@ public async Task CompleteAsync_AllowsTheDevelopmentLocalhostEndpointSelectedByP result.Content.Should().Be("local response"); } - private static OpenAiCompatibleLlmProvider CreateProvider(HttpMessageHandler handler, LlmProviderSettings settings) => - new(new HttpClient(handler), settings, NullLogger.Instance); + [Fact] + public async Task StreamAsync_UsageAbsent_LeavesTerminalUsageUnknown() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].Error.Should().BeNull(); + events[^1].TokensUsed.Should().BeNull( + "the quota layer must settle the reservation estimate when upstream usage is absent"); + } + + [Theory] + [InlineData("length", "token limit")] + [InlineData("content_filter", "content filter")] + [InlineData("tool_calls", "tool calls")] + [InlineData("function_call", "function call")] + [InlineData("vendor_reason", "non-standard")] + public async Task StreamAsync_NonStopFinishReason_IsTerminalDegraded(string finishReason, string expectedReason) + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + $"data: {{\"choices\":[{{\"delta\":{{\"content\":\"partial\"}},\"finish_reason\":\"{finishReason}\"}}]}}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":5}}\n\n" + + "data: [DONE]\n\n")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].IsDegraded.Should().BeTrue(); + events[^1].DegradedReason.Should().Contain(expectedReason); + events[^1].TokensUsed.Should().Be(5); + } + + [Theory] + [InlineData("{\"choices\":[1]}")] + [InlineData("{\"choices\":[{\"delta\":\"bad\",\"finish_reason\":null}]}")] + [InlineData("{\"choices\":[{\"delta\":{\"content\":1},\"finish_reason\":null}]}")] + [InlineData("{\"choices\":[{\"delta\":{},\"finish_reason\":1}]}")] + [InlineData("{\"choices\":[],\"usage\":{\"total_tokens\":\"many\"}}")] + public async Task StreamAsync_HostileSseSchema_EmitsExplicitTerminalError(string data) + { + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseResponse($"data: {data}\n\n")), + BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].IsComplete.Should().BeTrue(); + events[0].Error.Should().NotBeNullOrWhiteSpace(); + } + + [Fact] + public async Task StreamAsync_HttpTransportFailure_EmitsExplicitTerminalError() + { + var provider = CreateProvider( + new StubHttpMessageHandler((_, _) => Task.FromException(new HttpRequestException("boom"))), + BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].IsComplete.Should().BeTrue(); + events[0].Error.Should().Contain("transport failed"); + } + + [Fact] + public async Task StreamAsync_ResponseBodyIoFailure_EmitsExplicitTerminalError() + { + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseStreamResponse(new ThrowingReadStream())), + BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); - private static ChatCompletionRequest Request() => new([new ChatCompletionMessage("user", "hello")]); + events.Should().ContainSingle(); + events[0].Error.Should().Contain("transport failed"); + } + + [Fact] + public async Task StreamAsync_StalledBodyAfterHeaders_EmitsTimeoutTerminalEvent() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.TimeoutSeconds = 1; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseStreamResponse(new BlockingReadStream())), + settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].IsComplete.Should().BeTrue(); + events[0].Error.Should().Contain("timed out"); + } + + [Fact] + public async Task StreamAsync_OversizedLine_EmitsBoundedTerminalError() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxSseLineBytes = 256; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + settings.OpenAiCompatible.MaxResponseBytes = 1024; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseResponse("data: " + new string('x', 300) + "\n\n")), + settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Error.Should().Contain("safety limit"); + } + + [Fact] + public async Task StreamAsync_OversizedEvent_EmitsBoundedTerminalError() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxSseLineBytes = 1024; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + settings.OpenAiCompatible.MaxResponseBytes = 4096; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseResponse("data: " + new string('x', 600) + "\n\n")), + settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Error.Should().Contain("safety limit"); + } + + [Fact] + public async Task StreamAsync_AggregateResponseBudget_EmitsBoundedTerminalError() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxSseLineBytes = 256; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + settings.OpenAiCompatible.MaxResponseBytes = 1024; + var oneEvent = "data: {\"choices\":[{\"delta\":{\"content\":\"" + new string('x', 80) + "\"},\"finish_reason\":null}]}\n\n"; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseResponse(string.Concat(Enumerable.Repeat(oneEvent, 12)))), + settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].Error.Should().Contain("safety limit"); + } + + [Fact] + public async Task StreamAsync_OversizedNonSseBody_EmitsBoundedTerminalError() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxResponseBytes = 1024; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => JsonResponse(new string('x', 1025))), + settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Error.Should().Contain("safety limit"); + } + + [Fact] + public async Task CompleteAsync_OversizedJsonBody_ReturnsDegradedFallback() + { + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxResponseBytes = 1024; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => JsonResponse(new string('x', 1025))), + settings); + + var result = await provider.CompleteAsync(Request()); + + result.IsDegraded.Should().BeTrue(); + result.DegradedReason.Should().Contain("safety limit"); + } + + [Fact] + public async Task StreamAsync_BodyFailuresOpenCompanionCircuit() + { + var settings = BuildSettings(); + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 60 }; + var dispatches = 0; + var provider = CreateProvider( + new StubHttpMessageHandler(_ => + { + dispatches++; + return SseResponse("data: not-json\n\n"); + }), + settings, + tracker, + circuitSettings); + + var first = await CollectAsync(provider.StreamAsync(Request())); + var second = await CollectAsync(provider.StreamAsync(Request())); + + first[^1].Error.Should().Contain("malformed JSON"); + second.Should().ContainSingle(); + second[0].Error.Should().Contain("circuit is open"); + dispatches.Should().Be(1); + tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Open); + } + + [Fact] + public async Task ProbeAsync_ProviderTimeout_ReturnsUnavailableInsteadOfThrowing() + { + var provider = CreateProvider( + new StubHttpMessageHandler((_, _) => + Task.FromException(new TaskCanceledException("upstream timeout"))), + BuildSettings()); + + var health = await provider.ProbeAsync(); + + health.IsAvailable.Should().BeFalse(); + health.IsProbed.Should().BeTrue(); + health.ErrorMessage.Should().Contain("timed out"); + } + + [Fact] + public async Task CompleteAsync_WhenProviderIsNotAuthoritativelySelected_DoesNotDispatch() + { + var settings = BuildSettings(); + settings.Provider = "Mock"; + var dispatched = false; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatched = true; + return JsonResponse("{}"); + }), settings); + + var result = await provider.CompleteAsync(Request()); + + result.IsDegraded.Should().BeTrue(); + dispatched.Should().BeFalse(); + } + + [Fact] + public async Task StreamAsync_EmitsEachServerDerivedAttributionHeaderExactlyOnce() + { + HttpRequestMessage? observed = null; + var provider = CreateProvider(new StubHttpMessageHandler(request => + { + observed = request; + return SseResponse( + "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n"); + }), BuildSettings()); + var attribution = new LlmRequestAttribution( + Guid.NewGuid(), "corr-1", LlmRequestSourceSurface.Chat, Guid.NewGuid(), Guid.NewGuid()); + + await CollectAsync(provider.StreamAsync(Request(attribution))); + + observed.Should().NotBeNull(); + foreach (var header in new[] + { + LlmRequestAttributionMapper.CorrelationHeader, + LlmRequestAttributionMapper.SourceSurfaceHeader, + LlmRequestAttributionMapper.UserTokenHeader, + LlmRequestAttributionMapper.BoardTokenHeader, + LlmRequestAttributionMapper.SessionTokenHeader + }) + { + observed!.Headers.GetValues(header).Should().ContainSingle(); + } + observed!.Headers.Authorization!.Scheme.Should().Be("Bearer"); + observed.Headers.GetValues("X-Title").Should().ContainSingle(); + } + + [Fact] + public void LlmTokenEvent_PreservesSixParameterConstructorAndPascalCaseWireShape() + { + typeof(LlmTokenEvent).GetConstructors() + .Should().ContainSingle(constructor => constructor.GetParameters().Length == 6); + var json = JsonSerializer.Serialize(new LlmTokenEvent("", true) + { + IsDegraded = true, + DegradedReason = "reason" + }); + + json.Should().Contain("\"IsDegraded\":true"); + json.Should().Contain("\"DegradedReason\":\"reason\""); + json.Should().NotContain("\"isDegraded\""); + } + + private static OpenAiCompatibleLlmProvider CreateProvider( + HttpMessageHandler handler, + LlmProviderSettings settings, + CircuitBreakerStateTracker? tracker = null, + CircuitBreakerSettings? circuitSettings = null, + bool allowLocalhostEndpoints = false) => + new( + new HttpClient(handler), + settings, + NullLogger.Instance, + tracker, + circuitSettings, + allowLocalhostEndpoints); + + private static ChatCompletionRequest Request(LlmRequestAttribution? attribution = null) => + new([new ChatCompletionMessage("user", "hello")], Attribution: attribution); private static async Task> CollectAsync(IAsyncEnumerable stream) { @@ -169,6 +478,16 @@ private static async Task> CollectAsync(IAsyncEnumerable new(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") @@ -187,4 +506,35 @@ private static async Task> CollectAsync(IAsyncEnumerable { ["X-Title"] = "Taskdeck tests" } } }; + + private abstract class TestReadStream : Stream + { + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + private sealed class ThrowingReadStream : TestReadStream + { + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + ValueTask.FromException(new IOException("broken SSE body")); + } + + private sealed class BlockingReadStream : TestReadStream + { + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return 0; + } + } } diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs index 188cdfceb..ebfe00874 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs @@ -342,6 +342,25 @@ public async Task CompleteAsync_ShouldReturnDegraded_WhenJsonModeResponseIsInval result.DegradedReason.Should().Be("Response was truncated"); } + [Fact] + public async Task StreamAsync_PropagatesBufferedCompletionDegradationMetadata() + { + var settings = BuildSettings(); + settings.OpenAi.ApiKey = string.Empty; + var provider = new OpenAiLlmProvider( + new HttpClient(), settings, NullLogger.Instance); + + var events = new List(); + await foreach (var item in provider.StreamAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "hello")], + SystemPrompt: string.Empty))) + events.Add(item); + + events[^1].IsComplete.Should().BeTrue(); + events[^1].IsDegraded.Should().BeTrue(); + events[^1].DegradedReason.Should().Contain("configuration"); + } + [Theory] [InlineData("{\"reply\":\"incomplete", true)] [InlineData("{}", false)] diff --git a/docs/platform/CONFIGURATION_REFERENCE.md b/docs/platform/CONFIGURATION_REFERENCE.md index 16e78c8e8..76e67cdca 100644 --- a/docs/platform/CONFIGURATION_REFERENCE.md +++ b/docs/platform/CONFIGURATION_REFERENCE.md @@ -223,10 +223,13 @@ default and the only one that ships enabled. See | `Llm:OpenAi:Model` | `string` | `gpt-4o-mini` | Model identifier sent in chat requests. | No | | `Llm:OpenAi:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` applied to the OpenAI provider. Must be `> 0`: `LlmProviderSelectionPolicy.TryValidateOpenAiSettings` rejects values `<= 0` as invalid and the selection policy falls back to the Mock provider. (The `HttpClient` registration also substitutes `30` when the value is `<= 0`, but only as a safety net — the provider will still not be selected.) | No | | `Llm:OpenAiCompatible:ApiKey` | `string` | `""` | API key for the configured OpenAI-compatible endpoint. Store as a secret. | Only for `Llm:Provider = OpenAiCompatible` | -| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. It passes the same HTTP(S), private-network, metadata-host, and DNS connection-time SSRF controls as OpenAI. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. It may contain a path but not user information, a query, or a fragment, and passes HTTP(S), private-network, metadata-host, egress-envelope, and DNS connection-time SSRF controls. | Only for `Llm:Provider = OpenAiCompatible` | | `Llm:OpenAiCompatible:Model` | `string` | `""` | Required model identifier supplied by the compatible vendor. | Only for `Llm:Provider = OpenAiCompatible` | -| `Llm:OpenAiCompatible:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` for the compatible provider. Must be `> 0`; invalid values select Mock. | No | -| `Llm:OpenAiCompatible:ExtraHeaders:` | `object` (map) | `{}` | Optional non-secret gateway headers, for example `HTTP-Referer` and `X-Title` for OpenRouter. `Authorization` cannot be set here because Taskdeck derives it from `ApiKey`; values with line breaks are rejected. | No | +| `Llm:OpenAiCompatible:TimeoutSeconds` | `int` | `30` | Full compatible-provider response deadline, including response-body/SSE reads after headers. Valid range: 1--300; invalid values select Mock. | No | +| `Llm:OpenAiCompatible:MaxResponseBytes` | `int` | `1048576` | Maximum UTF-8 bytes accepted from a buffered response or aggregate SSE response. Valid range: 1024--4194304. | No | +| `Llm:OpenAiCompatible:MaxSseLineBytes` | `int` | `65536` | Maximum UTF-8 bytes in one SSE line. Valid range: 256--262144. | No | +| `Llm:OpenAiCompatible:MaxSseEventBytes` | `int` | `131072` | Maximum UTF-8 bytes in one assembled SSE event. Valid range: 512--524288 and cannot exceed `MaxResponseBytes`. | No | +| `Llm:OpenAiCompatible:ExtraHeaders:` | `object` (map) | `{}` | Optional non-secret gateway headers, for example `HTTP-Referer` and `X-Title` for OpenRouter. Authorization, proxy, hop-by-hop, cookie, host-routing, forwarding, and `x-taskdeck-*` headers are reserved; values with line breaks are rejected. | No | | `Llm:Gemini:ApiKey` | `string` | `""` | Gemini API key. Required to use the Gemini provider. Store as a secret. | Only for `Llm:Provider = Gemini` | | `Llm:Gemini:BaseUrl` | `string` | `https://generativelanguage.googleapis.com/v1beta` | Gemini API base URL. | No | | `Llm:Gemini:Model` | `string` | `gemini-2.5-flash` | Model identifier. | No | diff --git a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md index afd3f1ade..cd15bb50e 100644 --- a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md +++ b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md @@ -75,6 +75,9 @@ in Development. "BaseUrl": "", "Model": "", "TimeoutSeconds": 30, + "MaxResponseBytes": 1048576, + "MaxSseLineBytes": 65536, + "MaxSseEventBytes": 131072, "ExtraHeaders": {} }, "Gemini": { @@ -160,7 +163,9 @@ The endpoint must be public HTTP(S) and pass the same URL and DNS-level SSRF checks as OpenAI. Keep keys in a secret store; never commit them. Compatible gateways may require optional non-secret headers such as `HTTP-Referer` or `X-Title`; use `Llm__OpenAiCompatible__ExtraHeaders__` for those. -`Authorization` is reserved for the configured API key and cannot be overridden. +Authorization, proxy, hop-by-hop, cookie, host-routing, forwarding, and +`x-taskdeck-*` headers are reserved and cannot be overridden. The base URL may +contain a path but not user information, a query, or a fragment. ### OpenRouter @@ -189,11 +194,13 @@ $env:Llm__OpenAiCompatible__Model = 'deepseek-chat' ``` If a gateway rejects SSE, Taskdeck retries as a normal completion and emits one -final event with explicit `isDegraded`/`degradedReason` metadata rather than +final event with explicit `IsDegraded`/`DegradedReason` metadata rather than pretending the buffered response was incremental. Some compatible gateways also reject `response_format: { type: json_object }`; non-streaming extraction retries without that field while retaining the JSON-only instruction prompt and robust -response parsing. +response parsing. `TimeoutSeconds` is a full response deadline, including stream +body reads after headers. `MaxResponseBytes`, `MaxSseLineBytes`, and +`MaxSseEventBytes` bound buffered bodies and streaming parser memory. ## Playwright Demo Auto-Enable From b3c5ca67e3760b23a02ee0df9ea77fee85531575 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 05:54:17 +0100 Subject: [PATCH 4/7] Address OpenAI-compatible review findings Signed-off-by: Chris0Jeky # Conflicts: # backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs # backend/src/Taskdeck.Api/Extensions/WorkerRegistration.cs # backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs # docs/platform/LLM_PROVIDER_SETUP_GUIDE.md --- .../Extensions/LlmProviderRegistration.cs | 15 +- .../Services/ChatService.cs | 28 +- .../Services/CircuitBreakerStateTracker.cs | 141 ++++-- .../Services/EgressEnvelopeHandler.cs | 115 ++++- .../Services/ILlmProvider.cs | 43 +- .../Services/LlmCaptureTriageExtractor.cs | 30 +- .../Services/OpenAiCompatibleLlmProvider.cs | 454 ++++++++++++------ .../Taskdeck.Application.csproj | 1 + .../Taskdeck.Domain/Agents/EgressViolation.cs | 5 +- .../LlmProviderRegistrationTests.cs | 118 ++++- .../Services/ChatServiceTests.cs | 121 +++++ .../Services/EgressEnvelopeHandlerTests.cs | 45 ++ .../LlmCaptureTriageExtractorTests.cs | 38 ++ .../OpenAiCompatibleLlmProviderTests.cs | 272 ++++++++++- ...enAiCompatibleProviderArchitectureTests.cs | 88 ++++ deploy/.env.example | 12 + deploy/.env.production.template | 13 +- deploy/docker-compose.yml | 9 + deploy/render.yaml | 19 + docs/ops/CLOUD_REFERENCE_ARCHITECTURE.md | 10 +- docs/platform/CLOUD_DEPLOYMENT_GUIDE.md | 10 +- docs/platform/CONFIGURATION_REFERENCE.md | 20 +- docs/platform/LLM_PROVIDER_SETUP_GUIDE.md | 24 +- 23 files changed, 1395 insertions(+), 236 deletions(-) create mode 100644 backend/tests/Taskdeck.Architecture.Tests/OpenAiCompatibleProviderArchitectureTests.cs diff --git a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs index c39012c27..ab31d223b 100644 --- a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs @@ -147,7 +147,8 @@ public static IServiceCollection AddLlmProviders( .AddHttpMessageHandler(sp => new EgressEnvelopeHandler( sp.GetRequiredService(), sp.GetRequiredService>(), - nameof(OpenAiCompatibleLlmProvider))) + nameof(OpenAiCompatibleLlmProvider), + followRedirects: false)) .AddPolicyHandler(openAiCompatibleCircuitBreakerPolicy) .RemoveAllLoggers() .AddHttpMessageHandler(); @@ -234,15 +235,17 @@ public static IServiceCollection AddLlmProviders( return services; } - internal static SocketsHttpHandler CreateProtectedSocketsHttpHandler(bool allowLocalhostEndpoints) + internal static SocketsHttpHandler CreateProtectedSocketsHttpHandler( + bool allowLocalhostEndpoints, + bool disableSystemProxy = false) { return new SocketsHttpHandler { AllowAutoRedirect = false, - // A configured system proxy changes ConnectCallback's destination to the - // proxy and can tunnel to an unvalidated target. Protected clients must - // connect directly so the DNS-level callback validates the real origin. - UseProxy = false, + // The compatible client opts out of system proxies because an arbitrary + // configured origin must be the endpoint validated by ConnectCallback. + // Existing fixed-origin providers retain their established proxy behavior. + UseProxy = !disableSystemProxy, ConnectCallback = (context, cancellationToken) => OutboundWebhookConnectCallback.ConnectAsync( context, diff --git a/backend/src/Taskdeck.Application/Services/ChatService.cs b/backend/src/Taskdeck.Application/Services/ChatService.cs index f7180d26d..438db904b 100644 --- a/backend/src/Taskdeck.Application/Services/ChatService.cs +++ b/backend/src/Taskdeck.Application/Services/ChatService.cs @@ -139,6 +139,7 @@ public async Task> SendMessageAsync(Guid sessionId, Guid string? quotaBilledProvider = null; string? quotaBilledModel = null; var quotaBilledTokens = 0; + var quotaEstimatedTokens = 0; try { if (string.IsNullOrWhiteSpace(dto.Content)) @@ -192,6 +193,7 @@ public async Task> SendMessageAsync(Guid sessionId, Guid if (!reservation.Allowed) return Result.Failure(ErrorCodes.LlmQuotaExceeded, reservation.DeniedReason ?? "LLM quota exceeded"); quotaReservationId = reservation.ReservationId; + quotaEstimatedTokens = reservation.EstimatedTokens; } if (dto.RequestProposal && LooksLikeChecklistBootstrapRequest(dto.Content)) @@ -379,28 +381,33 @@ await _quotaService.CommitReservationAsync( SystemPrompt: clarificationPrompt); llmResult = await _llmProvider.CompleteAsync(completionRequest, ct); - // Finalize the quota reservation with the actual token count. The provider - // reports a combined TokensUsed total without an input/output split. Record - // the full total as input tokens and 0 for output until providers surface - // separate counts. - if (_quotaService != null && quotaReservationId is Guid singleResId && llmResult.TokensUsed > 0) + // Finalize with authoritative combined usage when present. If the provider + // omitted usage, commit the reservation estimate so a short output cannot + // undercharge a large input. Record the chosen total as input tokens and 0 + // for output until providers surface a reliable split. + var quotaTokens = llmResult.HasAuthoritativeTokenUsage + ? llmResult.TokensUsed + : llmResult.ShouldSettleQuotaReservation + ? quotaEstimatedTokens + : 0; + if (_quotaService != null && quotaReservationId is Guid singleResId && quotaTokens > 0) { // CancellationToken.None (M1, #1427 review): see the tool-result commit above. quotaBilledProvider = llmResult.Provider; quotaBilledModel = llmResult.Model; - quotaBilledTokens = llmResult.TokensUsed; + quotaBilledTokens = quotaTokens; await _quotaService.CommitReservationAsync( singleResId, userId, Domain.Enums.LlmSurface.Chat, llmResult.Provider, llmResult.Model, - llmResult.TokensUsed, 0, + quotaTokens, 0, CancellationToken.None); quotaCommitted = true; } } assistantContent = llmResult.Content; - tokenUsage = llmResult.TokensUsed; + tokenUsage = llmResult.HasAuthoritativeTokenUsage ? llmResult.TokensUsed : null; degradedReason = llmResult.DegradedReason; if (llmResult.IsDegraded) @@ -608,6 +615,7 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, string? model = null; var streamIsDegraded = false; string? streamDegradedReason = null; + var persistEmptyDegradedTerminal = false; // True once the provider has delivered at least one non-error event: the LLM call was made and // tokens flowed, so the reservation is billable even if the final usage event never arrives. var providerStreamed = false; @@ -666,6 +674,8 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, { streamIsDegraded = true; streamDegradedReason = token.DegradedReason ?? "Provider returned a degraded stream."; + if (token.IsComplete && token.Error is null) + persistEmptyDegradedTerminal = true; } if (!string.IsNullOrWhiteSpace(token.Error)) { @@ -679,6 +689,8 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, // Persist the streamed assistant message with token usage so the streaming // path is consistent with the non-streaming SendMessageAsync path. var streamedContent = contentBuilder.ToString(); + if (streamedContent.Length == 0 && persistEmptyDegradedTerminal) + streamedContent = "The provider ended the response without returning text."; if (!string.IsNullOrEmpty(streamedContent)) { var assistantMessage = new ChatMessage( diff --git a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs index 725c89c9b..ea7c0f283 100644 --- a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs +++ b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs @@ -10,7 +10,7 @@ namespace Taskdeck.Application.Services; public sealed class CircuitBreakerStateTracker { private readonly ConcurrentDictionary _states = new(); - private readonly ConcurrentDictionary _streamingFailures = new(); + private readonly ConcurrentDictionary _providerFailures = new(); /// /// Records a circuit state transition. Called by the Polly onBreak, @@ -48,27 +48,31 @@ public IReadOnlyDictionary GetAll() /// failures when callers use ResponseHeadersRead, so providers report those /// failures explicitly through this companion gate. /// - public bool TryEnterStreamingRequest( + internal bool TryEnterProviderRequest( string circuitName, CircuitBreakerSettings settings, + out CircuitRequestLease lease, out string? error) { while (true) { - if (!_streamingFailures.TryGetValue(circuitName, out var state)) + if (!_providerFailures.TryGetValue(circuitName, out var state)) { + lease = default; error = null; return true; } - if (state.HalfOpenProbeInFlight) + if (state.HalfOpenProbeId is not null) { - error = $"{circuitName} streaming circuit is half-open and its probe is already in progress."; + lease = default; + error = $"{circuitName} provider circuit is half-open and its probe is already in progress."; return false; } if (state.OpenUntilUtc is null) { + lease = default; error = null; return true; } @@ -76,57 +80,134 @@ public bool TryEnterStreamingRequest( var now = DateTimeOffset.UtcNow; if (state.OpenUntilUtc > now) { - error = $"{circuitName} streaming circuit is open after repeated response-body failures."; + lease = default; + error = $"{circuitName} provider circuit is open after repeated transport, body, or protocol failures."; return false; } - var halfOpen = new StreamingFailureState( + var probeId = Guid.NewGuid(); + var halfOpen = new ProviderFailureState( Math.Max(0, settings.FailureThreshold - 1), null, - HalfOpenProbeInFlight: true); - if (_streamingFailures.TryUpdate(circuitName, halfOpen, state)) + probeId); + if (_providerFailures.TryUpdate(circuitName, halfOpen, state)) { RecordState(circuitName, CircuitState.HalfOpen); + lease = new CircuitRequestLease(probeId); error = null; return true; } } } - public void RecordStreamingFailure( + internal void RecordProviderFailure( string circuitName, CircuitBreakerSettings settings, - string reason) + string reason, + CircuitRequestLease lease) { var now = DateTimeOffset.UtcNow; - var next = _streamingFailures.AddOrUpdate( - circuitName, - _ => new StreamingFailureState(1, null, HalfOpenProbeInFlight: false), - (_, existing) => existing.OpenUntilUtc is not null && existing.OpenUntilUtc > now - ? existing - : new StreamingFailureState(existing.ConsecutiveFailures + 1, null, HalfOpenProbeInFlight: false)); + while (true) + { + if (!_providerFailures.TryGetValue(circuitName, out var existing)) + { + if (lease.IsHalfOpenProbe) + return; - if (next.ConsecutiveFailures < settings.FailureThreshold && next.OpenUntilUtc is null) - return; + var initial = new ProviderFailureState(1, null, null); + if (!_providerFailures.TryAdd(circuitName, initial)) + continue; + existing = initial; + } + else + { + // Ignore stale completions from requests that pre-date a half-open probe, + // and ignore outcomes from a superseded probe lease. + if (existing.HalfOpenProbeId is not null && + existing.HalfOpenProbeId != lease.HalfOpenProbeId) + return; + if (lease.HalfOpenProbeId is not null && + existing.HalfOpenProbeId != lease.HalfOpenProbeId) + return; + if (existing.OpenUntilUtc is not null && existing.OpenUntilUtc > now && + lease.HalfOpenProbeId is null) + return; + + var failureCount = lease.IsHalfOpenProbe + ? settings.FailureThreshold + : existing.ConsecutiveFailures + 1; + var next = new ProviderFailureState(failureCount, null, null); + if (!_providerFailures.TryUpdate(circuitName, next, existing)) + continue; + existing = next; + } + + if (existing.ConsecutiveFailures < settings.FailureThreshold) + return; + + var opened = new ProviderFailureState( + existing.ConsecutiveFailures, + now.AddSeconds(settings.BreakDurationSeconds), + null); + if (_providerFailures.TryUpdate(circuitName, opened, existing)) + { + RecordState(circuitName, CircuitState.Open, reason); + return; + } + } + } - var opened = new StreamingFailureState( - next.ConsecutiveFailures, - now.AddSeconds(settings.BreakDurationSeconds), - HalfOpenProbeInFlight: false); - _streamingFailures[circuitName] = opened; - RecordState(circuitName, CircuitState.Open, reason); + internal void RecordProviderSuccess(string circuitName, CircuitRequestLease lease) + { + while (_providerFailures.TryGetValue(circuitName, out var existing)) + { + if (lease.HalfOpenProbeId is not null && existing.HalfOpenProbeId != lease.HalfOpenProbeId) + return; + if (lease.HalfOpenProbeId is null && existing.HalfOpenProbeId is not null) + return; + if (((ICollection>)_providerFailures).Remove( + new KeyValuePair(circuitName, existing))) + { + RecordState(circuitName, CircuitState.Closed); + return; + } + } } - public void RecordStreamingSuccess(string circuitName) + internal void AbandonProviderRequest(string circuitName, CircuitRequestLease lease) { - if (_streamingFailures.TryRemove(circuitName, out _)) - RecordState(circuitName, CircuitState.Closed); + if (!lease.IsHalfOpenProbe) + return; + + while (_providerFailures.TryGetValue(circuitName, out var existing)) + { + if (existing.HalfOpenProbeId != lease.HalfOpenProbeId) + return; + + // Release the exclusive probe immediately while retaining the open posture. + // The next caller can acquire a fresh half-open lease without waiting through + // another break duration after cancellation or iterator disposal. + var released = new ProviderFailureState( + existing.ConsecutiveFailures, + DateTimeOffset.UtcNow, + null); + if (_providerFailures.TryUpdate(circuitName, released, existing)) + { + RecordState(circuitName, CircuitState.Open, "Half-open provider probe was abandoned."); + return; + } + } } - private sealed record StreamingFailureState( + private sealed record ProviderFailureState( int ConsecutiveFailures, DateTimeOffset? OpenUntilUtc, - bool HalfOpenProbeInFlight); + Guid? HalfOpenProbeId); +} + +internal readonly record struct CircuitRequestLease(Guid? HalfOpenProbeId) +{ + public bool IsHalfOpenProbe => HalfOpenProbeId is not null; } public enum CircuitState diff --git a/backend/src/Taskdeck.Application/Services/EgressEnvelopeHandler.cs b/backend/src/Taskdeck.Application/Services/EgressEnvelopeHandler.cs index 8257bb032..41cecc26e 100644 --- a/backend/src/Taskdeck.Application/Services/EgressEnvelopeHandler.cs +++ b/backend/src/Taskdeck.Application/Services/EgressEnvelopeHandler.cs @@ -5,8 +5,9 @@ namespace Taskdeck.Application.Services; /// /// DelegatingHandler that enforces the egress envelope for all outbound HTTP requests. -/// Rejects requests to hosts not in the EgressRegistry and blocks redirects to -/// out-of-envelope destinations. +/// Rejects requests to hosts not in the EgressRegistry, blocks redirects to +/// out-of-envelope destinations, and can fail closed on every redirect for +/// clients whose destination must remain fixed. /// GP-10: EgressViolations are loud, structured, and never swallowed. /// public sealed class EgressEnvelopeHandler : DelegatingHandler @@ -16,15 +17,18 @@ public sealed class EgressEnvelopeHandler : DelegatingHandler private readonly IEgressRegistry _egressRegistry; private readonly ILogger? _logger; private readonly string? _sourceComponent; + private readonly bool _followRedirects; public EgressEnvelopeHandler( IEgressRegistry egressRegistry, ILogger? logger = null, - string? sourceComponent = null) + string? sourceComponent = null, + bool followRedirects = true) { _egressRegistry = egressRegistry ?? throw new ArgumentNullException(nameof(egressRegistry)); _logger = logger; _sourceComponent = sourceComponent; + _followRedirects = followRedirects; } /// Maximum number of redirects to follow manually. @@ -48,32 +52,48 @@ protected override async Task SendAsync( var currentRequest = request; var response = await base.SendAsync(currentRequest, cancellationToken); + if (!_followRedirects && IsRedirect(response)) + { + var redirectUri = response.Headers.Location; + var resolvedRedirectUri = redirectUri is null + ? null + : TryResolveRedirectUri(currentRequest.RequestUri, redirectUri); + response.Dispose(); + ThrowRedirectViolation( + currentRequest.RequestUri, + resolvedRedirectUri, + EgressViolationType.RedirectNotAllowed, + "Redirects are disabled for this outbound client. Redirect blocked."); + } + // Manually follow redirects, validating each target against the egress envelope var redirectCount = 0; while (IsRedirect(response) && response.Headers.Location is { } redirectUri && redirectCount < MaxRedirects) { redirectCount++; - var resolvedRedirectUri = redirectUri.IsAbsoluteUri - ? redirectUri - : new Uri(currentRequest.RequestUri!, redirectUri); + var resolvedRedirectUri = TryResolveRedirectUri(currentRequest.RequestUri, redirectUri); + + if (resolvedRedirectUri is null) + { + response.Dispose(); + ThrowRedirectViolation( + currentRequest.RequestUri, + null, + EgressViolationType.RedirectToUnknownHost, + "Redirect target was not a valid absolute or relative URI. Redirect blocked."); + } var redirectHost = resolvedRedirectUri.Host; if (string.IsNullOrWhiteSpace(redirectHost) || !_egressRegistry.IsHostAllowed(redirectHost)) { - var violation = new EgressViolation( - attemptedHost: redirectHost ?? "(empty)", - requestUri: resolvedRedirectUri.ToString(), - violationType: EgressViolationType.RedirectToUnknownHost, - reason: $"Redirect to host '{redirectHost}' is not in the egress envelope. Redirect blocked.", - sourceComponent: _sourceComponent); - - _logger?.LogError( - "EgressViolation: redirect to '{Host}' not in egress envelope. OriginalURI={OriginalUri}, RedirectURI={RedirectUri}, Source={Source}", - redirectHost, currentRequest.RequestUri, resolvedRedirectUri, _sourceComponent); - - throw new EgressViolationException(violation); + response.Dispose(); + ThrowRedirectViolation( + currentRequest.RequestUri, + resolvedRedirectUri, + EgressViolationType.RedirectToUnknownHost, + $"Redirect to host '{redirectHost}' is not in the egress envelope. Redirect blocked."); } // Follow the redirect: create a new request preserving the method for 307/308 @@ -129,32 +149,34 @@ private void ValidateHost(Uri? requestUri) var host = requestUri?.Host; if (string.IsNullOrWhiteSpace(host)) { + var sanitizedUri = SanitizeUriForAudit(requestUri); var violation = new EgressViolation( attemptedHost: "(empty)", - requestUri: requestUri?.ToString() ?? "(null)", + requestUri: sanitizedUri, violationType: EgressViolationType.UnknownHost, reason: "Request has no host specified.", sourceComponent: _sourceComponent); _logger?.LogError( "EgressViolation: request with no host. URI={Uri}, Source={Source}", - requestUri, _sourceComponent); + sanitizedUri, _sourceComponent); throw new EgressViolationException(violation); } if (!_egressRegistry.IsHostAllowed(host)) { + var sanitizedUri = SanitizeUriForAudit(requestUri); var violation = new EgressViolation( attemptedHost: host, - requestUri: requestUri!.ToString(), + requestUri: sanitizedUri, violationType: EgressViolationType.UnknownHost, reason: $"Host '{host}' is not in the egress envelope. Request blocked.", sourceComponent: _sourceComponent); _logger?.LogError( "EgressViolation: host '{Host}' not in egress envelope. URI={Uri}, Source={Source}", - host, requestUri, _sourceComponent); + host, sanitizedUri, _sourceComponent); throw new EgressViolationException(violation); } @@ -166,6 +188,55 @@ private static bool IsRedirect(HttpResponseMessage response) return statusCode is >= 300 and < 400; } + private static Uri? TryResolveRedirectUri(Uri? currentUri, Uri redirectUri) + { + if (redirectUri.IsAbsoluteUri) + return redirectUri; + + return currentUri is not null && Uri.TryCreate(currentUri, redirectUri, out var resolved) + ? resolved + : null; + } + + [System.Diagnostics.CodeAnalysis.DoesNotReturn] + private void ThrowRedirectViolation( + Uri? originalUri, + Uri? redirectUri, + EgressViolationType violationType, + string reason) + { + var redirectHost = redirectUri?.Host; + var sanitizedOriginalUri = SanitizeUriForAudit(originalUri); + var sanitizedRedirectUri = SanitizeUriForAudit(redirectUri); + var violation = new EgressViolation( + attemptedHost: string.IsNullOrWhiteSpace(redirectHost) ? "(empty)" : redirectHost, + requestUri: sanitizedRedirectUri, + violationType: violationType, + reason: reason, + sourceComponent: _sourceComponent); + + _logger?.LogError( + "EgressViolation: redirect blocked. Host={Host}, OriginalOrigin={OriginalOrigin}, RedirectOrigin={RedirectOrigin}, Source={Source}", + redirectHost ?? "(empty)", + sanitizedOriginalUri, + sanitizedRedirectUri, + _sourceComponent); + + throw new EgressViolationException(violation); + } + + private static string SanitizeUriForAudit(Uri? uri) + { + if (uri is null) + return "(null)"; + if (!uri.IsAbsoluteUri || string.IsNullOrWhiteSpace(uri.Host)) + return "(invalid-origin)"; + + return new UriBuilder(uri.Scheme, uri.Host, uri.IsDefaultPort ? -1 : uri.Port) + .Uri + .GetLeftPart(UriPartial.Authority); + } + private static async Task PrepareReplayableContentAsync( HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs index c35e52daf..925eb6ab3 100644 --- a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Serialization; namespace Taskdeck.Application.Services; @@ -63,7 +64,28 @@ public record LlmCompletionResult( string? DegradedReason = null, List? Instructions = null, bool IsClarificationRequest = false -); +) +{ + /// + /// True only when came from authoritative upstream + /// usage metadata. A false value tells quota consumers to settle the reserved + /// estimate instead of treating a local output-only estimate as total usage. + /// + [JsonIgnore] + public bool HasAuthoritativeTokenUsage { get; init; } = true; + + // Internal billing/circuit metadata must not become part of the API wire shape. + // Providers set this false when selection/configuration rejected a request before + // any upstream dispatch, so quota callers can release rather than charge. + [JsonIgnore] + internal bool ShouldSettleQuotaReservation { get; init; } = true; + + [JsonIgnore] + internal LlmProviderFailureKind ProviderFailureKind { get; init; } + + [JsonIgnore] + internal bool CountsAsProviderFailure => ProviderFailureKind != LlmProviderFailureKind.None; +} public record LlmTokenEvent( string Token, @@ -76,8 +98,27 @@ public record LlmTokenEvent( // Keep these out of the positional constructor. LlmTokenEvent is part of the // public Application contract, so extending its existing constructor would // break already-compiled consumers even though in-tree source still builds. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool IsDegraded { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? DegradedReason { get; init; } + + [JsonIgnore] + internal LlmProviderFailureKind ProviderFailureKind { get; init; } + + [JsonIgnore] + internal bool CountsAsProviderFailure => ProviderFailureKind != LlmProviderFailureKind.None; +} + +internal enum LlmProviderFailureKind +{ + None = 0, + Transport = 1, + ResponseBody = 2, + Protocol = 3, + Timeout = 4, + ResponseLimit = 5 } public record LlmHealthStatus( diff --git a/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs b/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs index b9bd1dc84..7935fa09f 100644 --- a/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs +++ b/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs @@ -73,6 +73,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell // Atomic quota reservation (issue #1313): reserve before the completion, then commit with the // actual tokens or release. This serializes concurrent triage/chat calls at the boundary. Guid? quotaReservationId = null; + var quotaEstimatedTokens = 0; if (_quotaService is not null) { var reservation = await _quotaService.ReserveAsync(userId, LlmSurface.CaptureTriage, cancellationToken); @@ -83,6 +84,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell Detail: reservation.DeniedReason); } quotaReservationId = reservation.ReservationId; + quotaEstimatedTokens = reservation.EstimatedTokens; } var request = new ChatCompletionRequest( @@ -106,13 +108,13 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell result = await _llmProvider.CompleteAsync(request, cancellationToken); completed = result; - // Settle the reservation. Tokens are consumed whether or not the content is usable - // (truncation is the clearest case: degraded AND billed), so a call that used tokens commits - // before any quality checks; a zero-token call releases (mirrors the prior "record only if - // > 0" behavior). + // Settle before quality checks because degraded output can still be billed. Use + // authoritative combined usage when present; otherwise commit the reservation estimate + // for a dispatched call so short output cannot undercharge a large transcript. if (quotaReservationId is Guid reservationId) { - if (result.TokensUsed > 0) + var quotaTokens = ResolveQuotaTokens(result, quotaEstimatedTokens); + if (quotaTokens > 0) { // CancellationToken.None (M1, #1427 review): tokens are already billed at this // point, so finalization must not be cancellable — a cancelled commit would trip @@ -123,7 +125,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell LlmSurface.CaptureTriage, result.Provider, result.Model, - result.TokensUsed, + quotaTokens, 0, CancellationToken.None); } @@ -145,7 +147,10 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell { try { - if (completed is { TokensUsed: > 0 } billed) + var quotaTokens = completed is null + ? 0 + : ResolveQuotaTokens(completed, quotaEstimatedTokens); + if (completed is { } billed && quotaTokens > 0) { await _quotaService!.CommitReservationAsync( unsettledId, @@ -153,7 +158,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell LlmSurface.CaptureTriage, billed.Provider, billed.Model, - billed.TokensUsed, + quotaTokens, 0, CancellationToken.None); } @@ -169,7 +174,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell "Quota reservation {ReservationId} settle failed in transcript triage (billed tokens: {Tokens}); " + "the row stays Reserved until the TTL sweep.", unsettledId, - completed?.TokensUsed ?? 0); + completed is null ? 0 : ResolveQuotaTokens(completed, quotaEstimatedTokens)); } } } @@ -274,4 +279,11 @@ private static List SanitizeTasks(IReadOnlyList + result.HasAuthoritativeTokenUsage + ? result.TokensUsed + : result.ShouldSettleQuotaReservation + ? reservationEstimate + : 0; } diff --git a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs index 967316b34..673daf943 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs @@ -13,7 +13,7 @@ namespace Taskdeck.Application.Services; /// separate from so api.openai.com defaults and /// its existing non-streaming semantics remain unchanged. /// -public sealed class OpenAiCompatibleLlmProvider : ILlmProvider +internal sealed class OpenAiCompatibleLlmProvider : ILlmProvider { private const string ProviderName = "OpenAICompatible"; private const string CircuitName = "OpenAICompatible"; @@ -27,8 +27,7 @@ public sealed class OpenAiCompatibleLlmProvider : ILlmProvider private readonly CircuitBreakerSettings? _circuitBreakerSettings; private readonly bool _allowLocalhostEndpoints; - // Retain the original public constructor for already-compiled consumers. - public OpenAiCompatibleLlmProvider( + internal OpenAiCompatibleLlmProvider( HttpClient httpClient, LlmProviderSettings settings, ILogger logger) @@ -36,7 +35,7 @@ public OpenAiCompatibleLlmProvider( { } - public OpenAiCompatibleLlmProvider( + internal OpenAiCompatibleLlmProvider( HttpClient httpClient, LlmProviderSettings settings, ILogger logger, @@ -52,13 +51,34 @@ public OpenAiCompatibleLlmProvider( _allowLocalhostEndpoints = allowLocalhostEndpoints; } - public async Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) + public Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) => + CompleteCoreAsync(request, ct, participateInCircuit: true); + + private async Task CompleteCoreAsync( + ChatCompletionRequest request, + CancellationToken ct, + bool participateInCircuit) { var userMessage = GetLastUserMessage(request); if (!TryValidateSettings(out var validationError)) { _logger.LogWarning("OpenAI-compatible provider configuration invalid: {Error}", validationError); - return BuildFallbackResult(userMessage, "Live provider configuration is invalid."); + return BuildFallbackResult( + userMessage, + "Live provider configuration is invalid.", + shouldSettleQuotaReservation: false, + failureKind: LlmProviderFailureKind.None); + } + + var lease = default(CircuitRequestLease); + var circuitSettled = !participateInCircuit; + if (participateInCircuit && !TryEnterProviderRequest(out lease, out var circuitError)) + { + return BuildFallbackResult( + userMessage, + circuitError ?? "OpenAI-compatible provider circuit is open.", + shouldSettleQuotaReservation: false, + failureKind: LlmProviderFailureKind.None); } using var timeout = CreateProviderTimeoutTokenSource(ct); @@ -82,40 +102,66 @@ public async Task CompleteAsync(ChatCompletionRequest reque if (!response.IsSuccessStatusCode) { _logger.LogWarning("OpenAI-compatible completion request failed with status code {StatusCode}.", (int)response.StatusCode); - return BuildFallbackResult(userMessage, "Live provider request failed."); + return RecordFailureAndBuildFallback( + $"OpenAI-compatible completion returned HTTP {(int)response.StatusCode}.", + "Live provider request failed.", + LlmProviderFailureKind.Protocol); } if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason)) { _logger.LogWarning("OpenAI-compatible completion response could not be parsed."); - return BuildFallbackResult(userMessage, "Live provider response parsing failed."); + return RecordFailureAndBuildFallback( + "OpenAI-compatible completion response could not be parsed.", + "Live provider response parsing failed.", + LlmProviderFailureKind.Protocol); } var finishDegradation = GetFinishReasonDegradation(finishReason); - if (finishDegradation is not null || - (useInstructionExtraction && OpenAiLlmProvider.LooksLikeTruncatedJson(content))) + if (finishDegradation is not null) + { + return RecordSuccess(new LlmCompletionResult( + content, tokensUsed ?? 0, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault(), + IsDegraded: true, + DegradedReason: finishDegradation) + { + HasAuthoritativeTokenUsage = tokensUsed.HasValue + }); + } + + if (useInstructionExtraction && OpenAiLlmProvider.LooksLikeTruncatedJson(content)) { - return new LlmCompletionResult( - content, tokensUsed, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault(), + return RecordFailure(new LlmCompletionResult( + content, tokensUsed ?? 0, false, Provider: ProviderName, Model: GetConfiguredModelOrDefault(), IsDegraded: true, - DegradedReason: finishDegradation ?? "Response was truncated."); + DegradedReason: "Response was truncated.") + { + HasAuthoritativeTokenUsage = tokensUsed.HasValue, + ProviderFailureKind = LlmProviderFailureKind.Protocol + }, "OpenAI-compatible completion returned truncated structured content."); } if (useInstructionExtraction && LlmInstructionExtractionPrompt.TryParseStructuredResponse( content, out var reply, out var actionable, out var instructions)) { - return new LlmCompletionResult( - reply, tokensUsed, actionable, actionable ? "llm.extracted" : null, - ProviderName, GetConfiguredModelOrDefault(), Instructions: instructions.Count > 0 ? instructions : null); + return RecordSuccess(new LlmCompletionResult( + reply, tokensUsed ?? 0, actionable, actionable ? "llm.extracted" : null, + ProviderName, GetConfiguredModelOrDefault(), Instructions: instructions.Count > 0 ? instructions : null) + { + HasAuthoritativeTokenUsage = tokensUsed.HasValue + }); } var (isActionable, actionIntent) = LlmIntentClassifier.Classify(userMessage); var fallbackInstructions = isActionable ? NaturalLanguageInstructionExtractor.Extract(userMessage, actionIntent) : []; - return new LlmCompletionResult( - content, tokensUsed, isActionable, actionIntent, ProviderName, GetConfiguredModelOrDefault(), - Instructions: fallbackInstructions.Count > 0 ? fallbackInstructions : null); + return RecordSuccess(new LlmCompletionResult( + content, tokensUsed ?? 0, isActionable, actionIntent, ProviderName, GetConfiguredModelOrDefault(), + Instructions: fallbackInstructions.Count > 0 ? fallbackInstructions : null) + { + HasAuthoritativeTokenUsage = tokensUsed.HasValue + }); } } catch (OperationCanceledException) when (ct.IsCancellationRequested) @@ -125,19 +171,72 @@ public async Task CompleteAsync(ChatCompletionRequest reque catch (OperationCanceledException) { _logger.LogWarning("OpenAI-compatible completion exceeded its configured response deadline."); - return BuildFallbackResult(userMessage, "Live provider request timed out."); + return RecordFailureAndBuildFallback( + "OpenAI-compatible completion timed out.", + "Live provider request timed out.", + LlmProviderFailureKind.Timeout); } catch (LlmProviderResponseLimitException ex) { _logger.LogWarning("OpenAI-compatible completion exceeded a response budget: {Limit}", ex.Message); - return BuildFallbackResult(userMessage, "Live provider response exceeded a safety limit."); + return RecordFailureAndBuildFallback( + "OpenAI-compatible completion exceeded a response budget.", + "Live provider response exceeded a safety limit.", + LlmProviderFailureKind.ResponseLimit); + } + catch (IOException ex) + { + _logger.LogWarning( + "OpenAI-compatible completion response body failed. {ExceptionSummary}", + SensitiveDataRedactor.SummarizeException(ex)); + return RecordFailureAndBuildFallback( + "OpenAI-compatible completion response body failed.", + "Live provider response body failed.", + LlmProviderFailureKind.ResponseBody); } catch (Exception ex) { _logger.LogError( "OpenAI-compatible completion request failed with unexpected error. {ExceptionSummary}", SensitiveDataRedactor.SummarizeException(ex)); - return BuildFallbackResult(userMessage, "Live provider request errored."); + return RecordFailureAndBuildFallback( + "OpenAI-compatible completion transport failed.", + "Live provider request errored.", + LlmProviderFailureKind.Transport); + } + finally + { + if (!circuitSettled) + AbandonProviderRequest(lease); + } + + LlmCompletionResult RecordSuccess(LlmCompletionResult result) + { + if (participateInCircuit) + RecordProviderSuccess(lease); + circuitSettled = true; + return result; + } + + LlmCompletionResult RecordFailure(LlmCompletionResult result, string reason) + { + if (participateInCircuit) + RecordProviderFailure(reason, lease); + circuitSettled = true; + return result; + } + + LlmCompletionResult RecordFailureAndBuildFallback( + string circuitReason, + string userReason, + LlmProviderFailureKind failureKind) + { + var result = BuildFallbackResult( + userMessage, + userReason, + shouldSettleQuotaReservation: true, + failureKind: failureKind); + return RecordFailure(result, circuitReason); } } @@ -147,79 +246,108 @@ public async IAsyncEnumerable StreamAsync( { if (!TryValidateSettings(out var validationError)) { - yield return TerminalError($"Live provider configuration is invalid: {validationError}"); + yield return TerminalError( + $"Live provider configuration is invalid: {validationError}", + failureKind: LlmProviderFailureKind.None); yield break; } - if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null && - !_circuitBreakerTracker.TryEnterStreamingRequest(CircuitName, _circuitBreakerSettings, out var circuitError)) + if (!TryEnterProviderRequest(out var lease, out var circuitError)) { - yield return TerminalError(circuitError ?? "OpenAI-compatible streaming circuit is open."); + yield return TerminalError( + circuitError ?? "OpenAI-compatible provider circuit is open.", + failureKind: LlmProviderFailureKind.None); yield break; } using var timeout = CreateProviderTimeoutTokenSource(ct); await using var enumerator = StreamCoreAsync(request, timeout.Token).GetAsyncEnumerator(timeout.Token); var emittedTerminal = false; + var circuitSettled = false; - while (true) + try { - LlmTokenEvent? failure = null; - bool moved; - try + while (true) { - moved = await enumerator.MoveNextAsync(); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - throw; - } - catch (OperationCanceledException) - { - moved = false; - failure = TerminalError("OpenAI-compatible SSE response timed out after headers were received."); - } - catch (Exception ex) - { - _logger.LogWarning( - "OpenAI-compatible SSE response failed after request dispatch. {ExceptionSummary}", - SensitiveDataRedactor.SummarizeException(ex)); - moved = false; - failure = TerminalError(ex is LlmProviderResponseLimitException - ? "OpenAI-compatible SSE response exceeded a safety limit." - : "OpenAI-compatible SSE transport failed before completion."); - } + LlmTokenEvent? failure = null; + bool moved; + try + { + moved = await enumerator.MoveNextAsync(); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) + { + moved = false; + failure = TerminalError( + "OpenAI-compatible SSE response timed out after headers were received.", + failureKind: LlmProviderFailureKind.Timeout); + } + catch (Exception ex) + { + _logger.LogWarning( + "OpenAI-compatible SSE response failed after request dispatch. {ExceptionSummary}", + SensitiveDataRedactor.SummarizeException(ex)); + moved = false; + var failureKind = ex switch + { + LlmProviderResponseLimitException => LlmProviderFailureKind.ResponseLimit, + IOException => LlmProviderFailureKind.ResponseBody, + _ => LlmProviderFailureKind.Transport + }; + failure = TerminalError( + failureKind == LlmProviderFailureKind.ResponseLimit + ? "OpenAI-compatible SSE response exceeded a safety limit." + : failureKind == LlmProviderFailureKind.ResponseBody + ? "OpenAI-compatible SSE response body failed before completion." + : "OpenAI-compatible SSE transport failed before completion.", + failureKind: failureKind); + } - if (failure is not null) - { - RecordStreamingFailure(failure.Error!); - yield return failure; - yield break; - } + if (failure is not null) + { + RecordProviderFailure(failure.Error!, lease); + circuitSettled = true; + yield return failure; + yield break; + } - if (!moved) - break; + if (!moved) + break; - var current = enumerator.Current; - if (current.IsComplete) - { - emittedTerminal = true; - if (IsStreamingFailure(current)) - RecordStreamingFailure(current.Error ?? current.DegradedReason ?? "Degraded SSE completion."); - else - RecordStreamingSuccess(); + var current = enumerator.Current; + if (current.IsComplete) + { + emittedTerminal = true; + if (IsProviderFailure(current)) + RecordProviderFailure( + current.Error ?? current.DegradedReason ?? "OpenAI-compatible provider completion failed.", + lease); + else + RecordProviderSuccess(lease); + circuitSettled = true; + } + + yield return current; + if (current.IsComplete) + yield break; } - yield return current; - if (current.IsComplete) - yield break; + if (!emittedTerminal) + { + var failure = TerminalError("OpenAI-compatible SSE stream ended before a completion marker."); + RecordProviderFailure(failure.Error!, lease); + circuitSettled = true; + yield return failure; + } } - - if (!emittedTerminal) + finally { - var failure = TerminalError("OpenAI-compatible SSE stream ended before a completion marker."); - RecordStreamingFailure(failure.Error!); - yield return failure; + if (!circuitSettled) + AbandonProviderRequest(lease); } } @@ -283,6 +411,7 @@ private async IAsyncEnumerable StreamCoreAsync( detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: false); + var lineReader = new BoundedSseLineReader(reader); var eventData = new StringBuilder(); var eventBytes = 0; var responseBytes = 0; @@ -291,11 +420,11 @@ private async IAsyncEnumerable StreamCoreAsync( while (true) { - var line = await ReadBoundedLineAsync(reader, compatible.MaxSseLineBytes, ct); + var line = await lineReader.ReadLineAsync(compatible.MaxSseLineBytes, ct); if (line.Text is null) break; - responseBytes = checked(responseBytes + line.Utf8Bytes + 1); + responseBytes = checked(responseBytes + line.Utf8Bytes + line.DelimiterUtf8Bytes); if (responseBytes > compatible.MaxResponseBytes) throw new LlmProviderResponseLimitException("aggregate SSE response byte budget exceeded"); @@ -407,18 +536,19 @@ private async IAsyncEnumerable EmitBufferedFallbackAsync( ChatCompletionRequest request, [EnumeratorCancellation] CancellationToken ct) { - var result = await CompleteAsync(request, ct); + var result = await CompleteCoreAsync(request, ct, participateInCircuit: false); yield return new LlmTokenEvent( result.Content, true, - TokensUsed: result.TokensUsed, + TokensUsed: result.HasAuthoritativeTokenUsage ? result.TokensUsed : null, Provider: ProviderName, Model: result.Model) { IsDegraded = true, DegradedReason = result.IsDegraded ? $"{BufferedStreamingFallbackReason} {result.DegradedReason}" - : BufferedStreamingFallbackReason + : BufferedStreamingFallbackReason, + ProviderFailureKind = result.ProviderFailureKind }; } @@ -559,10 +689,10 @@ private static SseEventParseResult TryParseSseEvent(string data) } } - private static bool TryParseResponse(string body, out string content, out int tokensUsed, out string? finishReason) + private static bool TryParseResponse(string body, out string content, out int? tokensUsed, out string? finishReason) { content = string.Empty; - tokensUsed = 0; + tokensUsed = null; finishReason = null; try { @@ -597,12 +727,9 @@ private static bool TryParseResponse(string body, out string content, out int to if (usage.ValueKind != JsonValueKind.Object || !usage.TryGetProperty("total_tokens", out var total) || total.ValueKind != JsonValueKind.Number || - !total.TryGetInt32(out tokensUsed) || tokensUsed < 0) + !total.TryGetInt32(out var parsedTokens) || parsedTokens < 0) return false; - } - else - { - tokensUsed = EstimateTokens(content); + tokensUsed = parsedTokens; } return true; } @@ -636,29 +763,6 @@ private static async Task ReadBoundedContentAsync( return new UTF8Encoding(false, true).GetString(buffer.GetBuffer(), 0, checked((int)buffer.Length)); } - private static async Task ReadBoundedLineAsync( - StreamReader reader, - int maxBytes, - CancellationToken ct) - { - var line = new StringBuilder(); - var one = new char[1]; - var bytes = 0; - while (true) - { - var read = await reader.ReadAsync(one.AsMemory(), ct); - if (read == 0) - return line.Length == 0 ? new BoundedLine(null, 0) : new BoundedLine(line.ToString().TrimEnd('\r'), bytes); - if (one[0] == '\n') - return new BoundedLine(line.ToString().TrimEnd('\r'), bytes); - - bytes = checked(bytes + Encoding.UTF8.GetByteCount(one)); - if (bytes > maxBytes) - throw new LlmProviderResponseLimitException("single SSE line byte budget exceeded"); - line.Append(one[0]); - } - } - private LlmTokenEvent BuildTerminalCompletion(string? finishReason, int? tokensUsed) { if (string.IsNullOrWhiteSpace(finishReason)) @@ -677,13 +781,19 @@ private LlmTokenEvent BuildTerminalCompletion(string? finishReason, int? tokensU }; } - private LlmTokenEvent TerminalError(string error, int? tokensUsed = null) => new( - string.Empty, - true, - Error: error, - TokensUsed: tokensUsed, - Provider: ProviderName, - Model: GetConfiguredModelOrDefault()); + private LlmTokenEvent TerminalError( + string error, + int? tokensUsed = null, + LlmProviderFailureKind failureKind = LlmProviderFailureKind.Protocol) => new( + string.Empty, + true, + Error: error, + TokensUsed: tokensUsed, + Provider: ProviderName, + Model: GetConfiguredModelOrDefault()) + { + ProviderFailureKind = failureKind + }; private CancellationTokenSource CreateProviderTimeoutTokenSource(CancellationToken ct) { @@ -692,23 +802,36 @@ private CancellationTokenSource CreateProviderTimeoutTokenSource(CancellationTok return timeout; } - private void RecordStreamingFailure(string reason) + private bool TryEnterProviderRequest(out CircuitRequestLease lease, out string? error) { - if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null) - _circuitBreakerTracker.RecordStreamingFailure(CircuitName, _circuitBreakerSettings, reason); + if (_circuitBreakerTracker is null || _circuitBreakerSettings is null) + { + lease = default; + error = null; + return true; + } + + return _circuitBreakerTracker.TryEnterProviderRequest( + CircuitName, + _circuitBreakerSettings, + out lease, + out error); } - private void RecordStreamingSuccess() + private void RecordProviderFailure(string reason, CircuitRequestLease lease) { - _circuitBreakerTracker?.RecordStreamingSuccess(CircuitName); + if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null) + _circuitBreakerTracker.RecordProviderFailure(CircuitName, _circuitBreakerSettings, reason, lease); } - private static bool IsStreamingFailure(LlmTokenEvent terminal) => - terminal.Error is not null || - (terminal.IsDegraded && !string.Equals( - terminal.DegradedReason, - BufferedStreamingFallbackReason, - StringComparison.Ordinal)); + private void RecordProviderSuccess(CircuitRequestLease lease) => + _circuitBreakerTracker?.RecordProviderSuccess(CircuitName, lease); + + private void AbandonProviderRequest(CircuitRequestLease lease) => + _circuitBreakerTracker?.AbandonProviderRequest(CircuitName, lease); + + private static bool IsProviderFailure(LlmTokenEvent terminal) => + terminal.Error is not null || terminal.CountsAsProviderFailure; private static string? GetFinishReasonDegradation(string? finishReason) { @@ -756,9 +879,11 @@ private string GetConfiguredModelOrDefault() => string.IsNullOrWhiteSpace(_setti private static string GetLastUserMessage(ChatCompletionRequest request) => request.Messages .LastOrDefault(message => string.Equals(message.Role, "User", StringComparison.OrdinalIgnoreCase))?.Content ?? string.Empty; - private static int EstimateTokens(string text) => Math.Max(1, string.IsNullOrWhiteSpace(text) ? 1 : text.Length / 4); - - private LlmCompletionResult BuildFallbackResult(string userMessage, string reason) + private LlmCompletionResult BuildFallbackResult( + string userMessage, + string reason, + bool shouldSettleQuotaReservation, + LlmProviderFailureKind failureKind) { var (isActionable, actionIntent) = LlmIntentClassifier.Classify(userMessage); var instructions = isActionable ? NaturalLanguageInstructionExtractor.Extract(userMessage, actionIntent) : []; @@ -767,14 +892,19 @@ private LlmCompletionResult BuildFallbackResult(string userMessage, string reaso : $"I can help with that request. ({reason})"; return new LlmCompletionResult( content, - EstimateTokens(userMessage) + EstimateTokens(content), + 0, isActionable, actionIntent, ProviderName, GetConfiguredModelOrDefault(), true, reason, - instructions.Count > 0 ? instructions : null); + instructions.Count > 0 ? instructions : null) + { + HasAuthoritativeTokenUsage = false, + ShouldSettleQuotaReservation = shouldSettleQuotaReservation, + ProviderFailureKind = failureKind + }; } private sealed record SseEventParseResult( @@ -784,7 +914,63 @@ private sealed record SseEventParseResult( int? TokensUsed = null, string? Error = null); - private sealed record BoundedLine(string? Text, int Utf8Bytes); + private sealed record BoundedLine(string? Text, int Utf8Bytes, int DelimiterUtf8Bytes); + + private sealed class BoundedSseLineReader(StreamReader reader) + { + private readonly char[] _one = new char[1]; + private char? _pending; + + public async Task ReadLineAsync(int maxBytes, CancellationToken ct) + { + var line = new StringBuilder(); + var bytes = 0; + while (true) + { + var next = await ReadCharacterAsync(ct); + if (next is null) + return line.Length == 0 + ? new BoundedLine(null, 0, 0) + : new BoundedLine(line.ToString(), bytes, 0); + + if (next == '\n') + return new BoundedLine(line.ToString(), bytes, 1); + + if (next == '\r') + { + var afterCarriageReturn = await ReadCharacterAsync(ct); + var delimiterBytes = 1; + if (afterCarriageReturn == '\n') + { + delimiterBytes = 2; + } + else if (afterCarriageReturn is not null) + { + _pending = afterCarriageReturn; + } + + return new BoundedLine(line.ToString(), bytes, delimiterBytes); + } + + bytes = checked(bytes + Encoding.UTF8.GetByteCount([next.Value])); + if (bytes > maxBytes) + throw new LlmProviderResponseLimitException("single SSE line byte budget exceeded"); + line.Append(next.Value); + } + } + + private async ValueTask ReadCharacterAsync(CancellationToken ct) + { + if (_pending is { } pending) + { + _pending = null; + return pending; + } + + var read = await reader.ReadAsync(_one.AsMemory(), ct); + return read == 0 ? null : _one[0]; + } + } private sealed class LlmProviderResponseLimitException(string message) : IOException(message); } diff --git a/backend/src/Taskdeck.Application/Taskdeck.Application.csproj b/backend/src/Taskdeck.Application/Taskdeck.Application.csproj index 4a966ba41..933a3d6b9 100644 --- a/backend/src/Taskdeck.Application/Taskdeck.Application.csproj +++ b/backend/src/Taskdeck.Application/Taskdeck.Application.csproj @@ -8,6 +8,7 @@ + diff --git a/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs b/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs index 8ab60b6f9..f65c5a365 100644 --- a/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs +++ b/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs @@ -66,5 +66,8 @@ public enum EgressViolationType RedirectToUnknownHost = 1, /// The host resolved to a private/internal IP range. - PrivateNetworkAttempt = 2 + PrivateNetworkAttempt = 2, + + /// The client policy refuses redirects even when the target host is allowed. + RedirectNotAllowed = 3 } diff --git a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs index cfdd05d75..ff5a5e3b7 100644 --- a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Net; using FluentAssertions; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; @@ -35,14 +36,17 @@ public void AddLlmProviders_ResolvesOpenAiCompatibleProvider_WhenSelectionIsVali services.AddLlmProviders(configuration); using var provider = services.BuildServiceProvider(); - provider.GetRequiredService() - .Should().BeOfType(); - provider.GetService().Should().BeNull( + provider.GetRequiredService().GetType().FullName + .Should().Be("Taskdeck.Application.Services.OpenAiCompatibleLlmProvider"); + var compatibleType = typeof(ILlmProvider).Assembly.GetType( + "Taskdeck.Application.Services.OpenAiCompatibleLlmProvider", + throwOnError: true)!; + provider.GetService(compatibleType).Should().BeNull( "the selector, not a directly resolvable concrete transport, owns the live-provider decision"); provider.GetRequiredService().GetAllEntries() .Should().ContainSingle(entry => entry.Host == "api.groq.com" && - entry.ToolOrAgentName == nameof(OpenAiCompatibleLlmProvider)); + entry.ToolOrAgentName == "OpenAiCompatibleLlmProvider"); var handler = provider.GetRequiredService() .CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); EnumeratePipeline(handler).Should().Contain(item => item is EgressEnvelopeHandler, @@ -50,17 +54,79 @@ public void AddLlmProviders_ResolvesOpenAiCompatibleProvider_WhenSelectionIsVali } [Fact] - public void ProtectedSocketHandlers_DisableSystemProxyBypass() + public void ProtectedSocketHandlers_ScopeDirectConnectionsToCompatibleClient() { - using var llmHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler(false); + using var existingProviderHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler(false); + using var compatibleHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler( + false, + disableSystemProxy: true); using var webhookHandler = WorkerRegistration.CreateProtectedWebhookHandler(false); - llmHandler.UseProxy.Should().BeFalse(); - llmHandler.AllowAutoRedirect.Should().BeFalse(); - webhookHandler.UseProxy.Should().BeFalse(); + existingProviderHandler.UseProxy.Should().BeTrue( + "OpenAI, Gemini, and Ollama retain their established system-proxy behavior"); + compatibleHandler.UseProxy.Should().BeFalse( + "the arbitrary compatible origin must be validated directly"); + existingProviderHandler.AllowAutoRedirect.Should().BeFalse(); + compatibleHandler.AllowAutoRedirect.Should().BeFalse(); + webhookHandler.UseProxy.Should().BeTrue( + "webhook proxy behavior is outside the compatible-provider change"); webhookHandler.AllowAutoRedirect.Should().BeFalse(); } + [Fact] + public void RegisteredPipelines_DisableProxyOnlyForCompatibleProvider() + { + var services = BuildCompatibleServices(); + using var provider = services.BuildServiceProvider(); + var factory = provider.GetRequiredService(); + + using var openAi = factory.CreateHandler(nameof(OpenAiLlmProvider)); + using var compatible = factory.CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); + using var gemini = factory.CreateHandler(nameof(GeminiLlmProvider)); + using var ollama = factory.CreateHandler(nameof(OllamaLlmProvider)); + + EnumeratePipeline(openAi).OfType().Single().UseProxy.Should().BeTrue(); + EnumeratePipeline(compatible).OfType().Single().UseProxy.Should().BeFalse(); + EnumeratePipeline(gemini).OfType().Single().UseProxy.Should().BeTrue(); + EnumeratePipeline(ollama).OfType().Single().UseProxy.Should().BeTrue(); + + var workerServices = new ServiceCollection(); + workerServices.AddLogging(); + workerServices.AddTaskdeckWorkers( + new ConfigurationBuilder().Build(), + new TestWebHostEnvironment("Production")); + using var workerProvider = workerServices.BuildServiceProvider(); + using var webhook = workerProvider.GetRequiredService() + .CreateHandler("OutboundWebhookDelivery"); + EnumeratePipeline(webhook).OfType().Single().UseProxy.Should().BeTrue(); + } + + [Theory] + [InlineData(HttpStatusCode.MovedPermanently)] + [InlineData(HttpStatusCode.Found)] + [InlineData(HttpStatusCode.SeeOther)] + [InlineData(HttpStatusCode.TemporaryRedirect)] + [InlineData(HttpStatusCode.PermanentRedirect)] + public async Task CompatibleClientPipeline_RefusesEveryRedirectWithoutFollowing(HttpStatusCode statusCode) + { + var services = BuildCompatibleServices(); + using var provider = services.BuildServiceProvider(); + using var handler = provider.GetRequiredService() + .CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); + var egressHandler = EnumeratePipeline(handler).OfType().Single(); + var redirectHandler = new RedirectStubHandler(statusCode, "https://api.groq.com/second-hop"); + egressHandler.InnerHandler = redirectHandler; + using var invoker = new HttpMessageInvoker(handler); + + var act = () => invoker.SendAsync( + new HttpRequestMessage(HttpMethod.Get, "https://api.groq.com/openai/v1/chat/completions"), + CancellationToken.None); + + var exception = await act.Should().ThrowAsync(); + exception.Which.Violation.ViolationType.Should().Be(Taskdeck.Domain.Agents.EgressViolationType.RedirectNotAllowed); + redirectHandler.InvocationCount.Should().Be(1, "the compatible pipeline must never dispatch a redirected request"); + } + [Theory] [InlineData(nameof(OpenAiLlmProvider))] [InlineData(nameof(GeminiLlmProvider))] @@ -387,6 +453,40 @@ private sealed class TestWebHostEnvironment(string environmentName) : IWebHostEn public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider(); } + private static ServiceCollection BuildCompatibleServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(new TestWebHostEnvironment("Production")); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Llm:EnableLiveProviders"] = "true", + ["Llm:Provider"] = "OpenAICompatible", + ["Llm:OpenAiCompatible:ApiKey"] = "test-compatible-key", + ["Llm:OpenAiCompatible:BaseUrl"] = "https://api.groq.com/openai/v1", + ["Llm:OpenAiCompatible:Model"] = "llama-3.1-8b-instant" + }) + .Build(); + services.AddLlmProviders(configuration); + return services; + } + + private sealed class RedirectStubHandler(HttpStatusCode statusCode, string location) : HttpMessageHandler + { + public int InvocationCount { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + InvocationCount++; + var response = new HttpResponseMessage(statusCode); + response.Headers.Location = new Uri(location); + return Task.FromResult(response); + } + } + private static IEnumerable EnumeratePipeline(HttpMessageHandler root) { for (var current = root; current is not null; current = (current as DelegatingHandler)?.InnerHandler) diff --git a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs index 2b5b725f0..d0b6c42e1 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs @@ -1987,6 +1987,61 @@ public async Task StreamResponseAsync_ProviderYieldsOnlyErrorEvent_ShouldRelease Times.Never); } + [Fact] + public async Task SendMessageAsync_UnknownCompatibleUsage_CommitsReservationEstimateForLargePrompt() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Unknown usage quota test"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, default)) + .ReturnsAsync(session); + _llmProviderMock + .Setup(p => p.CompleteAsync(It.IsAny(), default)) + .ReturnsAsync(new LlmCompletionResult( + "short reply", + 0, + false, + Provider: "OpenAICompatible", + Model: "vendor/model") + { + HasAuthoritativeTokenUsage = false + }); + + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, default)) + .ReturnsAsync(new DTOs.QuotaReservationDto( + true, null, reservationId, 10000, 100, EstimatedTokens: 4000)); + var serviceWithQuota = new ChatService( + _unitOfWorkMock.Object, + _llmProviderMock.Object, + _plannerMock.Object, + _proposalServiceMock.Object, + _policyEngineMock.Object, + _notificationServiceMock.Object, + _authorizationServiceMock.Object, + quotaService: quotaMock.Object); + + var result = await serviceWithQuota.SendMessageAsync( + session.Id, + userId, + new SendChatMessageDto(new string('x', 4000)), + default); + + result.IsSuccess.Should().BeTrue(); + result.Value.TokenUsage.Should().BeNull("upstream did not report authoritative usage"); + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 4000, + 0, + CancellationToken.None), Times.Once); + quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); + } + [Theory] [InlineData(false, "Response was stopped by the upstream content filter.")] [InlineData(true, "OpenAI-compatible SSE transport failed before completion.")] @@ -2019,6 +2074,57 @@ public async Task StreamResponseAsync_TerminalDegradedOrError_PersistsPartialHis persisted.DegradedReason.Should().Be(expectedReason); } + [Fact] + public async Task StreamResponseAsync_EmptyDegradedTerminal_PersistsSanitizedReloadableHistoryAndUsage() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Empty degraded terminal persistence"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, default)) + .ReturnsAsync(session); + _chatMessageRepoMock + .Setup(r => r.AddAsync(It.IsAny(), default)) + .ReturnsAsync((ChatMessage message, CancellationToken _) => message); + _llmProviderMock + .Setup(p => p.StreamAsync(It.IsAny(), default)) + .Returns(EmptyDegradedTerminalStream()); + + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, default)) + .ReturnsAsync(new DTOs.QuotaReservationDto( + true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var serviceWithQuota = new ChatService( + _unitOfWorkMock.Object, + _llmProviderMock.Object, + _plannerMock.Object, + _proposalServiceMock.Object, + _policyEngineMock.Object, + _notificationServiceMock.Object, + _authorizationServiceMock.Object, + quotaService: quotaMock.Object); + + await foreach (var _ in serviceWithQuota.StreamResponseAsync(session.Id, userId, default)) { } + var reloaded = await serviceWithQuota.GetSessionAsync(session.Id, userId, default); + + reloaded.IsSuccess.Should().BeTrue(); + reloaded.Value.RecentMessages.Should().ContainSingle(); + var persisted = reloaded.Value.RecentMessages.Single(); + persisted.Content.Should().Be("The provider ended the response without returning text."); + persisted.MessageType.Should().Be("degraded"); + persisted.DegradedReason.Should().Be("Response was stopped by the upstream content filter."); + persisted.TokenUsage.Should().Be(7); + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 7, + 0, + CancellationToken.None), Times.Once); + } + [Fact] public async Task StreamResponseAsync_UsageAbsent_CommitsReservationEstimate() { @@ -2118,6 +2224,21 @@ private static async IAsyncEnumerable StreamWithoutUsage() await Task.CompletedTask; } + private static async IAsyncEnumerable EmptyDegradedTerminalStream() + { + yield return new LlmTokenEvent( + string.Empty, + true, + TokensUsed: 7, + Provider: "OpenAICompatible", + Model: "vendor/model") + { + IsDegraded = true, + DegradedReason = "Response was stopped by the upstream content filter." + }; + await Task.CompletedTask; + } + private static async IAsyncEnumerable StreamEvents() { yield return new LlmTokenEvent("token", true, TokensUsed: 10, Provider: "Mock", Model: "mock-default"); diff --git a/backend/tests/Taskdeck.Application.Tests/Services/EgressEnvelopeHandlerTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/EgressEnvelopeHandlerTests.cs index 27db05d33..b82580a4d 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/EgressEnvelopeHandlerTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/EgressEnvelopeHandlerTests.cs @@ -1,5 +1,6 @@ using System.Net; using FluentAssertions; +using Microsoft.Extensions.Logging; using Taskdeck.Application.Services; using Taskdeck.Domain.Agents; using Xunit; @@ -91,6 +92,33 @@ public async Task SendAsync_RedirectToUnknownHost_ThrowsEgressViolation() ex.Which.Violation.AttemptedHost.Should().Be("evil.example.com"); } + [Fact] + public async Task SendAsync_BlockedRedirect_SanitizesViolationAndLogToOrigins() + { + var logger = new CapturingLogger(); + var registry = CreateRegistry("trusted.example.com"); + const string secret = "attacker-secret-token"; + const string userContent = "private-user-content"; + var inner = new SingleRedirectHandler( + $"https://user:{secret}@evil.example.com/{userContent}?token={secret}#{userContent}"); + var handler = new EgressEnvelopeHandler(registry, logger, sourceComponent: "test") + { + InnerHandler = inner + }; + using var invoker = new HttpMessageInvoker(handler); + + var act = () => invoker.SendAsync( + new HttpRequestMessage(HttpMethod.Get, "https://trusted.example.com/start?user=content"), + CancellationToken.None); + + var exception = await act.Should().ThrowAsync(); + exception.Which.Violation.RequestUri.Should().Be("https://evil.example.com"); + exception.Which.Violation.Reason.Should().NotContain(secret).And.NotContain(userContent); + logger.Messages.Should().ContainSingle(); + logger.Messages[0].Should().NotContain(secret).And.NotContain(userContent).And.NotContain("user=content"); + logger.Messages[0].Should().Contain("https://trusted.example.com").And.Contain("https://evil.example.com"); + } + [Fact] public async Task SendAsync_RedirectToAllowedHost_FollowsRedirect() { @@ -678,4 +706,21 @@ protected override bool TryComputeLength(out long length) return true; } } + + private sealed class CapturingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + } } diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs index e0cae4bcb..c939139a6 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs @@ -221,6 +221,44 @@ public async Task ExtractAsync_ShouldReturnProviderDegraded_AndStillRecordUsage_ Times.Never); } + [Fact] + public async Task ExtractAsync_UnknownCompatibleUsage_CommitsReservationEstimateForLargeTranscript() + { + _quotaMock + .Setup(q => q.ReserveAsync(_userId, LlmSurface.CaptureTriage, It.IsAny())) + .ReturnsAsync(new QuotaReservationDto( + true, null, _reservationId, 100_000, 60, EstimatedTokens: 4000)); + _providerMock + .Setup(p => p.CompleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new LlmCompletionResult( + """{"tasks":[{"title":"Send report","evidence":"send report"}]}""", + 0, + IsActionable: false, + Provider: "OpenAICompatible", + Model: "vendor/model") + { + HasAuthoritativeTokenUsage = false + }); + var extractor = BuildExtractor(); + + var result = await extractor.ExtractAsync( + _userId, + _boardId, + TranscriptPayload(new string('x', 4000) + " send report")); + + result.Outcome.Should().Be(LlmCaptureTriageOutcome.Succeeded); + _quotaMock.Verify(q => q.CommitReservationAsync( + _reservationId, + _userId, + LlmSurface.CaptureTriage, + "OpenAICompatible", + "vendor/model", + 4000, + 0, + CancellationToken.None), Times.Once); + _quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); + } + [Fact] public async Task ExtractAsync_ShouldReleaseReservation_WhenNoTokensWereConsumed() { diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs index 7e4ce5495..887ac7eb2 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs @@ -102,6 +102,27 @@ public async Task StreamAsync_WhenEndpointRejectsStreaming_EmitsBufferedFallback .Should().OnlyContain(item => item.GetProperty("role").GetString() != "system"); } + [Fact] + public async Task StreamAsync_WhenEndpointRejectsStreamingAndUsageIsAbsent_LeavesUsageUnknown() + { + var dispatches = 0; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return dispatches == 1 + ? new HttpResponseMessage(HttpStatusCode.BadRequest) + : JsonResponse("""{"choices":[{"message":{"content":"short reply"},"finish_reason":"stop"}]}"""); + }), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Token.Should().Be("short reply"); + events[0].TokensUsed.Should().BeNull(); + events[0].IsDegraded.Should().BeTrue(); + dispatches.Should().Be(2); + } + [Fact] public async Task CompleteAsync_WhenResponseFormatIsRejected_RetriesWithPromptOnlyJson() { @@ -126,6 +147,34 @@ public async Task CompleteAsync_WhenResponseFormatIsRejected_RetriesWithPromptOn fallbackPayload.RootElement.TryGetProperty("response_format", out _).Should().BeFalse(); } + [Fact] + public async Task CompleteAsync_UsageAbsent_PreservesUnknownUsage() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => + JsonResponse("""{"choices":[{"message":{"content":"short reply"},"finish_reason":"stop"}]}""")), BuildSettings()); + + var result = await provider.CompleteAsync(Request()); + + result.Content.Should().Be("short reply"); + result.TokensUsed.Should().Be(0); + result.HasAuthoritativeTokenUsage.Should().BeFalse(); + result.ShouldSettleQuotaReservation.Should().BeTrue(); + } + + [Fact] + public async Task StreamAsync_NonSseResponseWithoutUsage_LeavesUsageUnknown() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => + JsonResponse("""{"choices":[{"message":{"content":"short reply"},"finish_reason":"stop"}]}""")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Token.Should().Be("short reply"); + events[0].TokensUsed.Should().BeNull(); + events[0].IsDegraded.Should().BeTrue(); + } + [Fact] public async Task StreamAsync_PropagatesCancellation() { @@ -173,6 +222,22 @@ public async Task StreamAsync_UsageAbsent_LeavesTerminalUsageUnknown() "the quota layer must settle the reservation estimate when upstream usage is absent"); } + [Fact] + public async Task StreamAsync_ParsesCarriageReturnOnlySseFraming() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"},\"finish_reason\":\"stop\"}]}\r\r" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":11}}\r\r" + + "data: [DONE]\r\r")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Select(item => item.Token).Should().Equal("hello", string.Empty); + events[^1].IsComplete.Should().BeTrue(); + events[^1].TokensUsed.Should().Be(11); + events[^1].Error.Should().BeNull(); + } + [Theory] [InlineData("length", "token limit")] [InlineData("content_filter", "content filter")] @@ -181,17 +246,49 @@ public async Task StreamAsync_UsageAbsent_LeavesTerminalUsageUnknown() [InlineData("vendor_reason", "non-standard")] public async Task StreamAsync_NonStopFinishReason_IsTerminalDegraded(string finishReason, string expectedReason) { - var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( - $"data: {{\"choices\":[{{\"delta\":{{\"content\":\"partial\"}},\"finish_reason\":\"{finishReason}\"}}]}}\n\n" + - "data: {\"choices\":[],\"usage\":{\"total_tokens\":5}}\n\n" + - "data: [DONE]\n\n")), BuildSettings()); + var dispatches = 0; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 60 }; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return SseResponse( + $"data: {{\"choices\":[{{\"delta\":{{\"content\":\"partial\"}},\"finish_reason\":\"{finishReason}\"}}]}}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":5}}\n\n" + + "data: [DONE]\n\n"); + }), BuildSettings(), tracker, circuitSettings); var events = await CollectAsync(provider.StreamAsync(Request())); + var second = await CollectAsync(provider.StreamAsync(Request())); events[^1].IsComplete.Should().BeTrue(); events[^1].IsDegraded.Should().BeTrue(); events[^1].DegradedReason.Should().Contain(expectedReason); events[^1].TokensUsed.Should().Be(5); + second[^1].Error.Should().BeNull(); + dispatches.Should().Be(2, "normal upstream finish reasons must not open the companion circuit"); + tracker.Get("OpenAICompatible")?.State.Should().NotBe(CircuitState.Open); + } + + [Fact] + public async Task CompleteAsync_ContentFilterFinish_DoesNotCountAsCircuitFailure() + { + var dispatches = 0; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 60 }; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return JsonResponse("""{"choices":[{"message":{"content":"partial"},"finish_reason":"content_filter"}],"usage":{"total_tokens":4}}"""); + }), BuildSettings(), tracker, circuitSettings); + + var first = await provider.CompleteAsync(Request()); + var second = await provider.CompleteAsync(Request()); + + first.IsDegraded.Should().BeTrue(); + second.IsDegraded.Should().BeTrue(); + dispatches.Should().Be(2); + tracker.Get("OpenAICompatible")?.State.Should().NotBe(CircuitState.Open); } [Theory] @@ -225,6 +322,7 @@ public async Task StreamAsync_HttpTransportFailure_EmitsExplicitTerminalError() events.Should().ContainSingle(); events[0].IsComplete.Should().BeTrue(); events[0].Error.Should().Contain("transport failed"); + events[0].ProviderFailureKind.Should().Be(LlmProviderFailureKind.Transport); } [Fact] @@ -237,7 +335,8 @@ public async Task StreamAsync_ResponseBodyIoFailure_EmitsExplicitTerminalError() var events = await CollectAsync(provider.StreamAsync(Request())); events.Should().ContainSingle(); - events[0].Error.Should().Contain("transport failed"); + events[0].Error.Should().Contain("response body failed"); + events[0].ProviderFailureKind.Should().Be(LlmProviderFailureKind.ResponseBody); } [Fact] @@ -254,6 +353,7 @@ public async Task StreamAsync_StalledBodyAfterHeaders_EmitsTimeoutTerminalEvent( events.Should().ContainSingle(); events[0].IsComplete.Should().BeTrue(); events[0].Error.Should().Contain("timed out"); + events[0].ProviderFailureKind.Should().Be(LlmProviderFailureKind.Timeout); } [Fact] @@ -367,6 +467,141 @@ public async Task StreamAsync_BodyFailuresOpenCompanionCircuit() tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Open); } + [Theory] + [InlineData("malformed", (int)LlmProviderFailureKind.Protocol)] + [InlineData("body-io", (int)LlmProviderFailureKind.ResponseBody)] + [InlineData("oversized", (int)LlmProviderFailureKind.ResponseLimit)] + [InlineData("stalled", (int)LlmProviderFailureKind.Timeout)] + public async Task CompleteAsync_PostHeaderFailuresOpenCompanionCircuit( + string failureMode, + int expectedFailureKind) + { + var settings = BuildSettings(); + settings.OpenAiCompatible.TimeoutSeconds = 1; + settings.OpenAiCompatible.MaxResponseBytes = 1024; + settings.OpenAiCompatible.MaxSseEventBytes = 512; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 60 }; + var dispatches = 0; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return failureMode switch + { + "malformed" => JsonResponse("{}"), + "body-io" => JsonStreamResponse(new ThrowingReadStream()), + "oversized" => JsonResponse(new string('x', 1025)), + "stalled" => JsonStreamResponse(new BlockingReadStream()), + _ => throw new InvalidOperationException("Unknown test mode") + }; + }), settings, tracker, circuitSettings); + + var first = await provider.CompleteAsync(Request()); + var second = await provider.CompleteAsync(Request()); + + first.IsDegraded.Should().BeTrue(); + first.ProviderFailureKind.Should().Be((LlmProviderFailureKind)expectedFailureKind); + first.CountsAsProviderFailure.Should().BeTrue(); + second.DegradedReason.Should().Contain("circuit is open"); + dispatches.Should().Be(1); + tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Open); + } + + [Fact] + public async Task CompleteAsync_SuccessResetsConsecutiveBodyFailures() + { + var responses = new Queue( + [ + JsonResponse("{}"), + JsonResponse("""{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"total_tokens":2}}"""), + JsonResponse("{}"), + JsonResponse("""{"choices":[{"message":{"content":"ok again"},"finish_reason":"stop"}],"usage":{"total_tokens":3}}""") + ]); + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 2, BreakDurationSeconds = 60 }; + var dispatches = 0; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return responses.Dequeue(); + }), BuildSettings(), tracker, circuitSettings); + + var results = new List(); + for (var i = 0; i < 4; i++) + results.Add(await provider.CompleteAsync(Request())); + + results[0].IsDegraded.Should().BeTrue(); + results[1].IsDegraded.Should().BeFalse(); + results[2].IsDegraded.Should().BeTrue(); + results[3].IsDegraded.Should().BeFalse(); + dispatches.Should().Be(4, "each success resets the prior consecutive body failure"); + tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Closed); + } + + [Fact] + public async Task StreamAsync_DisposingHalfOpenProbe_ReleasesLeaseForNextProbe() + { + var dispatches = 0; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 1 }; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return dispatches switch + { + 1 => SseResponse("data: not-json\n\n"), + 2 => SseResponse("data: {\"choices\":[{\"delta\":{\"content\":\"probe\"},\"finish_reason\":null}]}\n\n"), + _ => SuccessfulSseResponse("recovered") + }; + }), BuildSettings(), tracker, circuitSettings); + + await CollectAsync(provider.StreamAsync(Request())); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + + var enumerator = provider.StreamAsync(Request()).GetAsyncEnumerator(); + (await enumerator.MoveNextAsync()).Should().BeTrue(); + enumerator.Current.Token.Should().Be("probe"); + await enumerator.DisposeAsync(); + + var recovered = await CollectAsync(provider.StreamAsync(Request())); + + recovered[^1].Error.Should().BeNull(); + dispatches.Should().Be(3); + tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Closed); + } + + [Fact] + public async Task StreamAsync_CancellingHalfOpenProbe_ReleasesLeaseForNextProbe() + { + var dispatches = 0; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 1 }; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return dispatches switch + { + 1 => SseResponse("data: not-json\n\n"), + 2 => SseStreamResponse(new BlockingReadStream()), + _ => SuccessfulSseResponse("recovered") + }; + }), BuildSettings(), tracker, circuitSettings); + + await CollectAsync(provider.StreamAsync(Request())); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + + Func>> cancelled = () => + CollectAsync(provider.StreamAsync(Request(), cancellation.Token)); + await cancelled.Should().ThrowAsync(); + + var recovered = await CollectAsync(provider.StreamAsync(Request())); + + recovered[^1].Error.Should().BeNull(); + dispatches.Should().Be(3); + tracker.Get("OpenAICompatible")!.State.Should().Be(CircuitState.Closed); + } + [Fact] public async Task ProbeAsync_ProviderTimeout_ReturnsUnavailableInsteadOfThrowing() { @@ -433,19 +668,21 @@ public async Task StreamAsync_EmitsEachServerDerivedAttributionHeaderExactlyOnce } [Fact] - public void LlmTokenEvent_PreservesSixParameterConstructorAndPascalCaseWireShape() + public void LlmTokenEvent_PreservesSixParameterConstructorAndCompatibleWireShape() { typeof(LlmTokenEvent).GetConstructors() .Should().ContainSingle(constructor => constructor.GetParameters().Length == 6); - var json = JsonSerializer.Serialize(new LlmTokenEvent("", true) + var normalJson = JsonSerializer.Serialize(new LlmTokenEvent("token", false, Provider: "OpenAI", Model: "model")); + var degradedJson = JsonSerializer.Serialize(new LlmTokenEvent("", true, TokensUsed: 4, Provider: "OpenAICompatible", Model: "vendor/model") { IsDegraded = true, DegradedReason = "reason" }); - json.Should().Contain("\"IsDegraded\":true"); - json.Should().Contain("\"DegradedReason\":\"reason\""); - json.Should().NotContain("\"isDegraded\""); + normalJson.Should().Be( + "{\"Token\":\"token\",\"IsComplete\":false,\"Error\":null,\"TokensUsed\":null,\"Provider\":\"OpenAI\",\"Model\":\"model\"}"); + degradedJson.Should().Be( + "{\"Token\":\"\",\"IsComplete\":true,\"Error\":null,\"TokensUsed\":4,\"Provider\":\"OpenAICompatible\",\"Model\":\"vendor/model\",\"IsDegraded\":true,\"DegradedReason\":\"reason\"}"); } private static OpenAiCompatibleLlmProvider CreateProvider( @@ -488,6 +725,21 @@ private static HttpResponseMessage SseStreamResponse(Stream stream) return response; } + private static HttpResponseMessage JsonStreamResponse(Stream stream) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); + return response; + } + + private static HttpResponseMessage SuccessfulSseResponse(string content) => SseResponse( + $"data: {{\"choices\":[{{\"delta\":{{\"content\":\"{content}\"}},\"finish_reason\":\"stop\"}}]}}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":3}}\n\n" + + "data: [DONE]\n\n"); + private static HttpResponseMessage JsonResponse(string body) => new(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") diff --git a/backend/tests/Taskdeck.Architecture.Tests/OpenAiCompatibleProviderArchitectureTests.cs b/backend/tests/Taskdeck.Architecture.Tests/OpenAiCompatibleProviderArchitectureTests.cs new file mode 100644 index 000000000..ec76de711 --- /dev/null +++ b/backend/tests/Taskdeck.Architecture.Tests/OpenAiCompatibleProviderArchitectureTests.cs @@ -0,0 +1,88 @@ +using System.Reflection; +using Taskdeck.Application.Services; +using Xunit; + +namespace Taskdeck.Architecture.Tests; + +public class OpenAiCompatibleProviderArchitectureTests +{ + [Fact] + public void ProviderTransport_IsNotPubliclyConstructible() + { + var providerType = typeof(ILlmProvider).Assembly.GetType( + "Taskdeck.Application.Services.OpenAiCompatibleLlmProvider", + throwOnError: true)!; + + Assert.False(providerType.IsPublic); + Assert.Empty(providerType.GetConstructors(BindingFlags.Public | BindingFlags.Instance)); + Assert.NotEmpty(providerType.GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance)); + + var apiRoot = ArchitectureTestPaths.GetBackendPath("src/Taskdeck.Api"); + var apiConstructionSites = Directory.GetFiles(apiRoot, "*.cs", SearchOption.AllDirectories) + .Where(path => File.ReadAllText(path).Contains( + "new OpenAiCompatibleLlmProvider(", + StringComparison.Ordinal)) + .Select(Path.GetFileName) + .ToArray(); + Assert.Equal(new[] { "LlmProviderRegistration.cs" }, apiConstructionSites); + } + + [Fact] + public void DeploymentSurfaces_ExposeTheSameCompatibleProviderSettingsWithoutASecretValue() + { + var repoRoot = Directory.GetParent(ArchitectureTestPaths.BackendRoot)!.FullName; + var compose = ReadRepoFile(repoRoot, "deploy/docker-compose.yml"); + var envExample = ReadRepoFile(repoRoot, "deploy/.env.example"); + var render = ReadRepoFile(repoRoot, "deploy/render.yaml"); + var productionTemplate = ReadRepoFile(repoRoot, "deploy/.env.production.template"); + + var composeMappings = new[] + { + "Llm__OpenAiCompatible__ApiKey: ${TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY:-}", + "Llm__OpenAiCompatible__BaseUrl: ${TASKDECK_LLM_OPENAI_COMPATIBLE_BASE_URL:-}", + "Llm__OpenAiCompatible__Model: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MODEL:-}", + "Llm__OpenAiCompatible__TimeoutSeconds: ${TASKDECK_LLM_OPENAI_COMPATIBLE_TIMEOUT_SECONDS:-30}", + "Llm__OpenAiCompatible__MaxResponseBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_RESPONSE_BYTES:-1048576}", + "Llm__OpenAiCompatible__MaxSseLineBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_LINE_BYTES:-65536}", + "Llm__OpenAiCompatible__MaxSseEventBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_EVENT_BYTES:-131072}", + "Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer: ${TASKDECK_LLM_OPENAI_COMPATIBLE_HTTP_REFERER:-}", + "Llm__OpenAiCompatible__ExtraHeaders__X-Title: ${TASKDECK_LLM_OPENAI_COMPATIBLE_X_TITLE:-}" + }; + Assert.All(composeMappings, mapping => Assert.Contains(mapping, compose, StringComparison.Ordinal)); + + var externalNames = new[] + { + "TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY=", + "TASKDECK_LLM_OPENAI_COMPATIBLE_BASE_URL=", + "TASKDECK_LLM_OPENAI_COMPATIBLE_MODEL=", + "TASKDECK_LLM_OPENAI_COMPATIBLE_TIMEOUT_SECONDS=30", + "TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_RESPONSE_BYTES=1048576", + "TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_LINE_BYTES=65536", + "TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_EVENT_BYTES=131072", + "TASKDECK_LLM_OPENAI_COMPATIBLE_HTTP_REFERER=", + "TASKDECK_LLM_OPENAI_COMPATIBLE_X_TITLE=" + }; + Assert.All(externalNames, name => Assert.Contains(name, envExample, StringComparison.Ordinal)); + + var appSettingNames = new[] + { + "Llm__OpenAiCompatible__ApiKey", + "Llm__OpenAiCompatible__BaseUrl", + "Llm__OpenAiCompatible__Model", + "Llm__OpenAiCompatible__TimeoutSeconds", + "Llm__OpenAiCompatible__MaxResponseBytes", + "Llm__OpenAiCompatible__MaxSseLineBytes", + "Llm__OpenAiCompatible__MaxSseEventBytes", + "Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer", + "Llm__OpenAiCompatible__ExtraHeaders__X-Title" + }; + Assert.All(appSettingNames, name => Assert.Contains(name, render, StringComparison.Ordinal)); + Assert.All(appSettingNames, name => Assert.Contains(name, productionTemplate, StringComparison.Ordinal)); + + Assert.DoesNotContain("TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY=sk-", envExample, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Llm__OpenAiCompatible__ApiKey=sk-", productionTemplate, StringComparison.OrdinalIgnoreCase); + } + + private static string ReadRepoFile(string repoRoot, string relativePath) => + File.ReadAllText(Path.Combine(repoRoot, relativePath.Replace('/', Path.DirectorySeparatorChar))); +} diff --git a/deploy/.env.example b/deploy/.env.example index a19317cd3..9f177a505 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -28,4 +28,16 @@ TASKDECK_CONNECTORS_ENCRYPTION_KEY= # TASKDECK_LLM_ENABLE_LIVE_PROVIDERS=false # TASKDECK_LLM_PROVIDER=Mock # TASKDECK_LLM_OPENAI_API_KEY= +# OpenAICompatible production endpoints must use public HTTPS URLs. HTTP is +# accepted only for explicit localhost development configurations outside this +# Production compose profile. +# TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY= +# TASKDECK_LLM_OPENAI_COMPATIBLE_BASE_URL=https://api.groq.com/openai/v1 +# TASKDECK_LLM_OPENAI_COMPATIBLE_MODEL=llama-3.1-8b-instant +# TASKDECK_LLM_OPENAI_COMPATIBLE_TIMEOUT_SECONDS=30 +# TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_RESPONSE_BYTES=1048576 +# TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_LINE_BYTES=65536 +# TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_EVENT_BYTES=131072 +# TASKDECK_LLM_OPENAI_COMPATIBLE_HTTP_REFERER= +# TASKDECK_LLM_OPENAI_COMPATIBLE_X_TITLE= # TASKDECK_LLM_GEMINI_API_KEY= diff --git a/deploy/.env.production.template b/deploy/.env.production.template index 4af24aad3..9218e9eb5 100644 --- a/deploy/.env.production.template +++ b/deploy/.env.production.template @@ -92,10 +92,21 @@ Connectors__EncryptionKey=CHANGE_ME_GENERATE_WITH_openssl_rand_base64_32 # --------------------------------------------------------------------------- # Mock provider is the default (no external API calls). -# Set EnableLiveProviders=true and Provider to OpenAI or Gemini for live AI. +# Set EnableLiveProviders=true and select OpenAI, OpenAICompatible, or Gemini. Llm__EnableLiveProviders=false Llm__Provider=Mock # Llm__OpenAi__ApiKey=sk-... +# OpenAICompatible production endpoints must use public HTTPS URLs. Keep the +# API key in the deployment platform's secret store, not in a committed file. +# Llm__OpenAiCompatible__ApiKey= +# Llm__OpenAiCompatible__BaseUrl=https://api.groq.com/openai/v1 +# Llm__OpenAiCompatible__Model=llama-3.1-8b-instant +# Llm__OpenAiCompatible__TimeoutSeconds=30 +# Llm__OpenAiCompatible__MaxResponseBytes=1048576 +# Llm__OpenAiCompatible__MaxSseLineBytes=65536 +# Llm__OpenAiCompatible__MaxSseEventBytes=131072 +# Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer=https://your-taskdeck-url.example.com +# Llm__OpenAiCompatible__ExtraHeaders__X-Title=Taskdeck # Llm__Gemini__ApiKey=AI... # --------------------------------------------------------------------------- diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1aad008a4..3bbf7b7bf 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -38,6 +38,15 @@ services: Llm__EnableLiveProviders: ${TASKDECK_LLM_ENABLE_LIVE_PROVIDERS:-false} Llm__Provider: ${TASKDECK_LLM_PROVIDER:-Mock} Llm__OpenAi__ApiKey: ${TASKDECK_LLM_OPENAI_API_KEY:-} + Llm__OpenAiCompatible__ApiKey: ${TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY:-} + Llm__OpenAiCompatible__BaseUrl: ${TASKDECK_LLM_OPENAI_COMPATIBLE_BASE_URL:-} + Llm__OpenAiCompatible__Model: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MODEL:-} + Llm__OpenAiCompatible__TimeoutSeconds: ${TASKDECK_LLM_OPENAI_COMPATIBLE_TIMEOUT_SECONDS:-30} + Llm__OpenAiCompatible__MaxResponseBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_RESPONSE_BYTES:-1048576} + Llm__OpenAiCompatible__MaxSseLineBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_LINE_BYTES:-65536} + Llm__OpenAiCompatible__MaxSseEventBytes: ${TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_EVENT_BYTES:-131072} + Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer: ${TASKDECK_LLM_OPENAI_COMPATIBLE_HTTP_REFERER:-} + Llm__OpenAiCompatible__ExtraHeaders__X-Title: ${TASKDECK_LLM_OPENAI_COMPATIBLE_X_TITLE:-} Llm__Gemini__ApiKey: ${TASKDECK_LLM_GEMINI_API_KEY:-} Connectors__EncryptionKey: ${TASKDECK_CONNECTORS_ENCRYPTION_KEY:?TASKDECK_CONNECTORS_ENCRYPTION_KEY must be set to a base64-encoded 256-bit key (openssl rand -base64 32)} DevelopmentSandbox__Enabled: "false" diff --git a/deploy/render.yaml b/deploy/render.yaml index 455922523..de3d3926a 100644 --- a/deploy/render.yaml +++ b/deploy/render.yaml @@ -62,6 +62,25 @@ services: value: Mock # - key: Llm__OpenAi__ApiKey # sync: false + # OpenAICompatible requires a public HTTPS base URL in Production. + # - key: Llm__OpenAiCompatible__ApiKey + # sync: false + # - key: Llm__OpenAiCompatible__BaseUrl + # value: https://api.groq.com/openai/v1 + # - key: Llm__OpenAiCompatible__Model + # value: llama-3.1-8b-instant + # - key: Llm__OpenAiCompatible__TimeoutSeconds + # value: "30" + # - key: Llm__OpenAiCompatible__MaxResponseBytes + # value: "1048576" + # - key: Llm__OpenAiCompatible__MaxSseLineBytes + # value: "65536" + # - key: Llm__OpenAiCompatible__MaxSseEventBytes + # value: "131072" + # - key: Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer + # value: https://your-taskdeck-url.example.com + # - key: Llm__OpenAiCompatible__ExtraHeaders__X-Title + # value: Taskdeck # - key: Llm__Gemini__ApiKey # sync: false diff --git a/docs/ops/CLOUD_REFERENCE_ARCHITECTURE.md b/docs/ops/CLOUD_REFERENCE_ARCHITECTURE.md index 9c3b01ae3..608a9299a 100644 --- a/docs/ops/CLOUD_REFERENCE_ARCHITECTURE.md +++ b/docs/ops/CLOUD_REFERENCE_ARCHITECTURE.md @@ -234,8 +234,16 @@ Not implemented in the initial topology. When multi-region becomes necessary: | `ConnectionStrings__DefaultConnection` | Secrets Manager | PostgreSQL connection string | | `Jwt__SecretKey` | Secrets Manager | JWT signing key | | `Redis__ConnectionString` | Secrets Manager | Redis connection string | -| `Llm__Provider` | Task definition env | `Mock`, `OpenAI`, or `Gemini` | +| `Llm__Provider` | Task definition env | `Mock`, `OpenAI`, `OpenAICompatible`, or `Gemini` | | `Llm__OpenAi__ApiKey` | Secrets Manager | OpenAI API key (if enabled) | +| `Llm__OpenAiCompatible__ApiKey` | Secrets Manager | Compatible-provider API key (if enabled) | +| `Llm__OpenAiCompatible__BaseUrl` | Task definition env | Required public HTTPS compatible API base URL | +| `Llm__OpenAiCompatible__Model` | Task definition env | Required compatible model identifier | +| `Llm__OpenAiCompatible__TimeoutSeconds` | Task definition env | Full response deadline; default `30` seconds | +| `Llm__OpenAiCompatible__MaxResponseBytes` | Task definition env | Buffered or aggregate SSE byte budget; default `1048576` | +| `Llm__OpenAiCompatible__MaxSseLineBytes` | Task definition env | Per-line SSE byte budget; default `65536` | +| `Llm__OpenAiCompatible__MaxSseEventBytes` | Task definition env | Per-event SSE byte budget; default `131072` | +| `Llm__OpenAiCompatible__ExtraHeaders__` | Task definition env | Optional non-secret gateway header, such as `HTTP-Referer` or `X-Title` | | `Llm__Gemini__ApiKey` | Secrets Manager | Gemini API key (if enabled) | | `ASPNETCORE_ENVIRONMENT` | Task definition env | `Production` | | `ASPNETCORE_FORWARDEDHEADERS_ENABLED` | Task definition env | `true` | diff --git a/docs/platform/CLOUD_DEPLOYMENT_GUIDE.md b/docs/platform/CLOUD_DEPLOYMENT_GUIDE.md index 2cb38c538..920020f7d 100644 --- a/docs/platform/CLOUD_DEPLOYMENT_GUIDE.md +++ b/docs/platform/CLOUD_DEPLOYMENT_GUIDE.md @@ -221,8 +221,16 @@ See `deploy/.env.production.template` for the authoritative list with descriptio | Variable | Default | Purpose | |----------|---------|---------| | `Llm__EnableLiveProviders` | `false` | Enable live LLM API calls | -| `Llm__Provider` | `Mock` | LLM provider: `Mock`, `OpenAI`, or `Gemini` | +| `Llm__Provider` | `Mock` | LLM provider: `Mock`, `OpenAI`, `OpenAICompatible`, or `Gemini` | | `Llm__OpenAi__ApiKey` | (empty) | OpenAI API key | +| `Llm__OpenAiCompatible__ApiKey` | (empty) | Compatible-provider API key; store as a platform secret | +| `Llm__OpenAiCompatible__BaseUrl` | (empty) | Required public HTTPS API base URL; cloud Production does not permit local HTTP | +| `Llm__OpenAiCompatible__Model` | (empty) | Required compatible model identifier | +| `Llm__OpenAiCompatible__TimeoutSeconds` | `30` | Full response deadline, including body/SSE reads | +| `Llm__OpenAiCompatible__MaxResponseBytes` | `1048576` | Buffered or aggregate SSE response byte budget | +| `Llm__OpenAiCompatible__MaxSseLineBytes` | `65536` | Per-line SSE byte budget | +| `Llm__OpenAiCompatible__MaxSseEventBytes` | `131072` | Per-event SSE byte budget | +| `Llm__OpenAiCompatible__ExtraHeaders__` | (empty) | Optional non-secret gateway header; `HTTP-Referer` and `X-Title` are common | | `Llm__Gemini__ApiKey` | (empty) | Gemini API key | | `GitHubOAuth__ClientId` | (empty) | GitHub OAuth app client ID | | `GitHubOAuth__ClientSecret` | (empty) | GitHub OAuth app secret | diff --git a/docs/platform/CONFIGURATION_REFERENCE.md b/docs/platform/CONFIGURATION_REFERENCE.md index 76e67cdca..683a08434 100644 --- a/docs/platform/CONFIGURATION_REFERENCE.md +++ b/docs/platform/CONFIGURATION_REFERENCE.md @@ -223,7 +223,7 @@ default and the only one that ships enabled. See | `Llm:OpenAi:Model` | `string` | `gpt-4o-mini` | Model identifier sent in chat requests. | No | | `Llm:OpenAi:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` applied to the OpenAI provider. Must be `> 0`: `LlmProviderSelectionPolicy.TryValidateOpenAiSettings` rejects values `<= 0` as invalid and the selection policy falls back to the Mock provider. (The `HttpClient` registration also substitutes `30` when the value is `<= 0`, but only as a safety net — the provider will still not be selected.) | No | | `Llm:OpenAiCompatible:ApiKey` | `string` | `""` | API key for the configured OpenAI-compatible endpoint. Store as a secret. | Only for `Llm:Provider = OpenAiCompatible` | -| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. It may contain a path but not user information, a query, or a fragment, and passes HTTP(S), private-network, metadata-host, egress-envelope, and DNS connection-time SSRF controls. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. Production/non-development endpoints must use public HTTPS; plain HTTP is accepted only for gated loopback development. The URL may contain a path but not user information, a query, or a fragment, and passes private-network, metadata-host, egress-envelope, and DNS connection-time SSRF controls. | Only for `Llm:Provider = OpenAiCompatible` | | `Llm:OpenAiCompatible:Model` | `string` | `""` | Required model identifier supplied by the compatible vendor. | Only for `Llm:Provider = OpenAiCompatible` | | `Llm:OpenAiCompatible:TimeoutSeconds` | `int` | `30` | Full compatible-provider response deadline, including response-body/SSE reads after headers. Valid range: 1--300; invalid values select Mock. | No | | `Llm:OpenAiCompatible:MaxResponseBytes` | `int` | `1048576` | Maximum UTF-8 bytes accepted from a buffered response or aggregate SSE response. Valid range: 1024--4194304. | No | @@ -764,6 +764,15 @@ Examples: | `Jwt:SecretKey` | `Jwt__SecretKey` | | `Auth:Registration:Mode` | `Auth__Registration__Mode` | | `Llm:OpenAi:ApiKey` | `Llm__OpenAi__ApiKey` | +| `Llm:OpenAiCompatible:ApiKey` | `Llm__OpenAiCompatible__ApiKey` | +| `Llm:OpenAiCompatible:BaseUrl` | `Llm__OpenAiCompatible__BaseUrl` | +| `Llm:OpenAiCompatible:Model` | `Llm__OpenAiCompatible__Model` | +| `Llm:OpenAiCompatible:TimeoutSeconds` | `Llm__OpenAiCompatible__TimeoutSeconds` | +| `Llm:OpenAiCompatible:MaxResponseBytes` | `Llm__OpenAiCompatible__MaxResponseBytes` | +| `Llm:OpenAiCompatible:MaxSseLineBytes` | `Llm__OpenAiCompatible__MaxSseLineBytes` | +| `Llm:OpenAiCompatible:MaxSseEventBytes` | `Llm__OpenAiCompatible__MaxSseEventBytes` | +| `Llm:OpenAiCompatible:ExtraHeaders:HTTP-Referer` | `Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer` | +| `Llm:OpenAiCompatible:ExtraHeaders:X-Title` | `Llm__OpenAiCompatible__ExtraHeaders__X-Title` | | `Workers:RetryBackoffSeconds:0` | `Workers__RetryBackoffSeconds__0` | | `RateLimiting:AuthPerIp:PermitLimit` | `RateLimiting__AuthPerIp__PermitLimit` | | `RateLimiting:McpAuthenticationPerIp:PermitLimit` | `RateLimiting__McpAuthenticationPerIp__PermitLimit` | @@ -787,6 +796,15 @@ container as standard ASP.NET Core environment variables. | `TASKDECK_LLM_ENABLE_LIVE_PROVIDERS` | `Llm__EnableLiveProviders` | `false` | No | | `TASKDECK_LLM_PROVIDER` | `Llm__Provider` | `Mock` | No | | `TASKDECK_LLM_OPENAI_API_KEY` | `Llm__OpenAi__ApiKey` | `""` | Only for `TASKDECK_LLM_PROVIDER=OpenAi` | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_API_KEY` | `Llm__OpenAiCompatible__ApiKey` | `""` | Only for `TASKDECK_LLM_PROVIDER=OpenAICompatible`; keep in a secret store | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_BASE_URL` | `Llm__OpenAiCompatible__BaseUrl` | `""` | Required public HTTPS base URL in the Production compose profile | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_MODEL` | `Llm__OpenAiCompatible__Model` | `""` | Required compatible model identifier | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_TIMEOUT_SECONDS` | `Llm__OpenAiCompatible__TimeoutSeconds` | `30` | Full response deadline, including body/SSE reads | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_RESPONSE_BYTES` | `Llm__OpenAiCompatible__MaxResponseBytes` | `1048576` | Buffered or aggregate SSE response budget | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_LINE_BYTES` | `Llm__OpenAiCompatible__MaxSseLineBytes` | `65536` | Per-line SSE budget | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_MAX_SSE_EVENT_BYTES` | `Llm__OpenAiCompatible__MaxSseEventBytes` | `131072` | Per-event SSE budget | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_HTTP_REFERER` | `Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer` | `""` | Optional non-secret OpenRouter attribution header | +| `TASKDECK_LLM_OPENAI_COMPATIBLE_X_TITLE` | `Llm__OpenAiCompatible__ExtraHeaders__X-Title` | `""` | Optional non-secret OpenRouter application title | | `TASKDECK_LLM_GEMINI_API_KEY` | `Llm__Gemini__ApiKey` | `""` | Only for `TASKDECK_LLM_PROVIDER=Gemini` | | `TASKDECK_PROXY_PORT` | Host port mapped to the nginx reverse proxy | `8080` | No | | `TASKDECK_VITE_API_BASE_URL` | Build-time `VITE_API_BASE_URL` for the web image | `/api` | No | diff --git a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md index cd15bb50e..547b184d0 100644 --- a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md +++ b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md @@ -34,6 +34,7 @@ Selection is deterministic through `LlmProviderSelectionPolicy`: - the OpenAI, OpenAICompatible, Gemini, and Ollama primary `HttpClient` handlers use `OutboundWebhookConnectCallback` for DNS-level SSRF protection and set both `AllowAutoRedirect = false` and `UseProxy = false`; ambient/system proxy settings are ignored so the configured provider origin remains the host validated by the connect callback These provider transports are direct-only. Taskdeck has no proxy-aware outbound LLM mode, so a deployment that can reach providers only through a corporate proxy fails closed. This is a dedicated connect-callback boundary, not `EgressEnvelopeHandler` enforcement, and it does not change the existing redirect or audit posture. Registered protected clients mask their configured URI immediately before send, then an inner handler restores it for transport; this keeps path/query data out of outer .NET HTTP EventSource payloads. Public caller-owned provider clients do not opt into masking. Protected registrations remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private metric scope that Taskdeck's configured OpenTelemetry pipeline drops alongside marked HTTP activities. Enabled Sentry removes its outbound handler from these protected client pipelines only, so it cannot add separate `sentry-trace`/baggage headers or capture protected URLs and 5xx responses; unrelated clients retain instrumentation and server-side Sentry tracking remains active. Independently installed process-global `ActivityListener`/`MeterListener` and transport-stage host/IP observation remain outside the guarantee. +OpenAICompatible additionally rejects every observed 3xx response in its egress handler rather than following an allowlisted redirect. If any live-provider condition fails, runtime degrades safely to `Mock`. @@ -159,14 +160,33 @@ Set the common safety gates and provider name: - `Llm__AllowLiveProvidersInDevelopment=true` (only for development-like environments) - `Llm__Provider=OpenAICompatible` -The endpoint must be public HTTP(S) and pass the same URL and DNS-level SSRF -checks as OpenAI. Keep keys in a secret store; never commit them. Compatible +Production and other non-development endpoints must be public HTTPS and pass +the same URL and DNS-level SSRF checks as OpenAI. Plain HTTP is accepted only +for loopback development endpoints when both the development environment and +`AllowLiveProvidersInDevelopment` gate permit it. Keep keys in a secret store; +never commit them. Compatible gateways may require optional non-secret headers such as `HTTP-Referer` or `X-Title`; use `Llm__OpenAiCompatible__ExtraHeaders__` for those. Authorization, proxy, hop-by-hop, cookie, host-routing, forwarding, and `x-taskdeck-*` headers are reserved and cannot be overridden. The base URL may contain a path but not user information, a query, or a fragment. +The complete environment-variable surface is: + +- required: `Llm__OpenAiCompatible__ApiKey`, + `Llm__OpenAiCompatible__BaseUrl`, and `Llm__OpenAiCompatible__Model` +- response controls: `Llm__OpenAiCompatible__TimeoutSeconds`, + `Llm__OpenAiCompatible__MaxResponseBytes`, + `Llm__OpenAiCompatible__MaxSseLineBytes`, and + `Llm__OpenAiCompatible__MaxSseEventBytes` +- optional gateway headers: + `Llm__OpenAiCompatible__ExtraHeaders__`; for example, + `Llm__OpenAiCompatible__ExtraHeaders__HTTP-Referer` and + `Llm__OpenAiCompatible__ExtraHeaders__X-Title` + +Compatible requests fail closed on every HTTP redirect, including redirects +back to the configured host. Configure the final API base URL directly. + ### OpenRouter ```powershell From e88e696db96815237ea76fc5e10943f3132c6816 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 14:58:57 +0100 Subject: [PATCH 5/7] Preserve protected provider egress registration Signed-off-by: Chris0Jeky --- .../Extensions/LlmProviderRegistration.cs | 19 ----------- .../LlmProviderRegistrationTests.cs | 32 ++++--------------- 2 files changed, 7 insertions(+), 44 deletions(-) diff --git a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs index ab31d223b..30de24b7a 100644 --- a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs @@ -235,25 +235,6 @@ public static IServiceCollection AddLlmProviders( return services; } - internal static SocketsHttpHandler CreateProtectedSocketsHttpHandler( - bool allowLocalhostEndpoints, - bool disableSystemProxy = false) - { - return new SocketsHttpHandler - { - AllowAutoRedirect = false, - // The compatible client opts out of system proxies because an arbitrary - // configured origin must be the endpoint validated by ConnectCallback. - // Existing fixed-origin providers retain their established proxy behavior. - UseProxy = !disableSystemProxy, - ConnectCallback = (context, cancellationToken) => - OutboundWebhookConnectCallback.ConnectAsync( - context, - allowLocalhostEndpoints, - cancellationToken) - }; - } - private static OpenAiCompatibleLlmProvider CreateOpenAiCompatibleProvider( IServiceProvider services, LlmProviderSettings settings, diff --git a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs index ff5a5e3b7..1b5af210c 100644 --- a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs @@ -54,27 +54,7 @@ public void AddLlmProviders_ResolvesOpenAiCompatibleProvider_WhenSelectionIsVali } [Fact] - public void ProtectedSocketHandlers_ScopeDirectConnectionsToCompatibleClient() - { - using var existingProviderHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler(false); - using var compatibleHandler = LlmProviderRegistration.CreateProtectedSocketsHttpHandler( - false, - disableSystemProxy: true); - using var webhookHandler = WorkerRegistration.CreateProtectedWebhookHandler(false); - - existingProviderHandler.UseProxy.Should().BeTrue( - "OpenAI, Gemini, and Ollama retain their established system-proxy behavior"); - compatibleHandler.UseProxy.Should().BeFalse( - "the arbitrary compatible origin must be validated directly"); - existingProviderHandler.AllowAutoRedirect.Should().BeFalse(); - compatibleHandler.AllowAutoRedirect.Should().BeFalse(); - webhookHandler.UseProxy.Should().BeTrue( - "webhook proxy behavior is outside the compatible-provider change"); - webhookHandler.AllowAutoRedirect.Should().BeFalse(); - } - - [Fact] - public void RegisteredPipelines_DisableProxyOnlyForCompatibleProvider() + public void RegisteredPipelines_KeepCompatibleProviderInsideProtectedDirectEgressBoundary() { var services = BuildCompatibleServices(); using var provider = services.BuildServiceProvider(); @@ -85,10 +65,12 @@ public void RegisteredPipelines_DisableProxyOnlyForCompatibleProvider() using var gemini = factory.CreateHandler(nameof(GeminiLlmProvider)); using var ollama = factory.CreateHandler(nameof(OllamaLlmProvider)); - EnumeratePipeline(openAi).OfType().Single().UseProxy.Should().BeTrue(); + EnumeratePipeline(openAi).OfType().Single().UseProxy.Should().BeFalse(); EnumeratePipeline(compatible).OfType().Single().UseProxy.Should().BeFalse(); - EnumeratePipeline(gemini).OfType().Single().UseProxy.Should().BeTrue(); - EnumeratePipeline(ollama).OfType().Single().UseProxy.Should().BeTrue(); + EnumeratePipeline(gemini).OfType().Single().UseProxy.Should().BeFalse(); + EnumeratePipeline(ollama).OfType().Single().UseProxy.Should().BeFalse(); + ProxySafeHttpHandlerTestHarness.AssertProxySafeOriginHandler(compatible); + EnumeratePipeline(compatible).Should().Contain(item => item is EgressEnvelopeHandler); var workerServices = new ServiceCollection(); workerServices.AddLogging(); @@ -98,7 +80,7 @@ public void RegisteredPipelines_DisableProxyOnlyForCompatibleProvider() using var workerProvider = workerServices.BuildServiceProvider(); using var webhook = workerProvider.GetRequiredService() .CreateHandler("OutboundWebhookDelivery"); - EnumeratePipeline(webhook).OfType().Single().UseProxy.Should().BeTrue(); + EnumeratePipeline(webhook).OfType().Single().UseProxy.Should().BeFalse(); } [Theory] From 7390a3fcf24b487b6ca079d7ce484d9734dc02bc Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 15:22:55 +0100 Subject: [PATCH 6/7] Harden compatible provider dispatch and resilience Signed-off-by: Chris0Jeky --- .../Extensions/LlmProviderRegistration.cs | 46 ++- ...dOutboundSentryHttpMessageHandlerFilter.cs | 1 + .../Services/ChatService.cs | 107 +++++-- .../Services/CircuitBreakerStateTracker.cs | 272 +++++++++++------- .../Services/ILlmProvider.cs | 65 ++++- .../Services/LlmCaptureTriageExtractor.cs | 40 ++- .../Services/LlmDispatchTrackingHandler.cs | 29 ++ .../Services/OpenAiCompatibleLlmProvider.cs | 242 ++++++++++++---- .../Taskdeck.Domain/Agents/EgressViolation.cs | 2 +- .../LlmProviderRegistrationTests.cs | 189 ++++++++++-- .../OutboundWebhookConnectCallbackTests.cs | 39 ++- .../ProtectedOutboundTelemetryHandlerTests.cs | 4 + .../Services/ChatServiceTests.cs | 200 ++++++++++++- .../CircuitBreakerStateTrackerTests.cs | 102 +++++++ .../LlmCaptureTriageExtractorTests.cs | 65 +++++ .../LlmDispatchTrackingHandlerTests.cs | 62 ++++ .../OpenAiCompatibleLlmProviderTests.cs | 182 +++++++++++- 17 files changed, 1417 insertions(+), 230 deletions(-) create mode 100644 backend/src/Taskdeck.Application/Services/LlmDispatchTrackingHandler.cs create mode 100644 backend/tests/Taskdeck.Application.Tests/Services/CircuitBreakerStateTrackerTests.cs create mode 100644 backend/tests/Taskdeck.Application.Tests/Services/LlmDispatchTrackingHandlerTests.cs diff --git a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs index 30de24b7a..3e1c76ce5 100644 --- a/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs +++ b/backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Net.Sockets; using Microsoft.Extensions.DependencyInjection.Extensions; using Polly; @@ -80,7 +81,7 @@ public static IServiceCollection AddLlmProviders( circuitBreakerTracker, "Gemini", circuitBreakerSettings); var ollamaCircuitBreakerPolicy = BuildCircuitBreakerPolicy( circuitBreakerTracker, "Ollama", circuitBreakerSettings); - var openAiCompatibleCircuitBreakerPolicy = BuildCircuitBreakerPolicy( + var openAiCompatibleCircuitBreakerPolicy = BuildOpenAiCompatibleCircuitBreakerPolicy( circuitBreakerTracker, "OpenAICompatible", circuitBreakerSettings); // Determine once at startup whether localhost LLM endpoints are permitted. @@ -144,14 +145,15 @@ public static IServiceCollection AddLlmProviders( cancellationToken) }; }) + .AddPolicyHandler(openAiCompatibleCircuitBreakerPolicy) + .RemoveAllLoggers() + .AddHttpMessageHandler() .AddHttpMessageHandler(sp => new EgressEnvelopeHandler( sp.GetRequiredService(), sp.GetRequiredService>(), nameof(OpenAiCompatibleLlmProvider), followRedirects: false)) - .AddPolicyHandler(openAiCompatibleCircuitBreakerPolicy) - .RemoveAllLoggers() - .AddHttpMessageHandler(); + .AddHttpMessageHandler(_ => new LlmDispatchTrackingHandler()); services.AddHttpClient((sp, client) => { var settings = sp.GetRequiredService(); @@ -225,7 +227,7 @@ public static IServiceCollection AddLlmProviders( settings, circuitBreakerTracker, circuitBreakerSettings, - localhostPolicy.AllowGeneralProviderLocalhost), + localhostPolicy), LlmProviderKind.Gemini => sp.GetRequiredService(), LlmProviderKind.Ollama => sp.GetRequiredService(), _ => sp.GetRequiredService() @@ -240,19 +242,19 @@ private static OpenAiCompatibleLlmProvider CreateOpenAiCompatibleProvider( LlmProviderSettings settings, CircuitBreakerStateTracker circuitBreakerTracker, CircuitBreakerSettings circuitBreakerSettings, - bool allowLocalhostEndpoints) + LlmProviderRuntimePolicy runtimePolicy) { RegisterOpenAiCompatibleEgress( services.GetRequiredService(), settings, - allowLocalhostEndpoints); + runtimePolicy.AllowGeneralProviderLocalhost); return new OpenAiCompatibleLlmProvider( services.GetRequiredService().CreateClient(OpenAiCompatibleHttpClientName), settings, services.GetRequiredService>(), circuitBreakerTracker, circuitBreakerSettings, - allowLocalhostEndpoints); + runtimePolicy); } private static IEgressRegistry GetOrCreateEgressRegistry(IServiceCollection services) @@ -329,6 +331,34 @@ internal static IAsyncPolicy BuildCircuitBreakerPolicy( }); } + /// + /// Builds the compatible-provider policy. HTTP 501 is intentionally excluded: + /// the provider recognizes it as an SSE capability rejection and performs a + /// buffered retry, which must remain reachable even at a threshold of one. + /// + internal static IAsyncPolicy BuildOpenAiCompatibleCircuitBreakerPolicy( + CircuitBreakerStateTracker tracker, + string circuitName, + CircuitBreakerSettings settings) + { + return Policy + .Handle() + .OrResult(response => + response.StatusCode == HttpStatusCode.RequestTimeout || + ((int)response.StatusCode >= 500 && + response.StatusCode != HttpStatusCode.NotImplemented)) + .CircuitBreakerAsync( + handledEventsAllowedBeforeBreaking: settings.FailureThreshold, + durationOfBreak: TimeSpan.FromSeconds(settings.BreakDurationSeconds), + onBreak: (outcome, breakDuration) => + { + var reason = outcome.Exception?.Message ?? $"HTTP {(int)(outcome.Result?.StatusCode ?? 0)}"; + tracker.RecordState(circuitName, CircuitState.Open, reason); + }, + onReset: () => tracker.RecordState(circuitName, CircuitState.Closed), + onHalfOpen: () => tracker.RecordState(circuitName, CircuitState.HalfOpen)); + } + /// /// Determines whether localhost LLM endpoints should be permitted based on the /// hosting environment and LLM provider configuration. Returns true only in diff --git a/backend/src/Taskdeck.Api/Extensions/ProtectedOutboundSentryHttpMessageHandlerFilter.cs b/backend/src/Taskdeck.Api/Extensions/ProtectedOutboundSentryHttpMessageHandlerFilter.cs index 130d24995..913fd902e 100644 --- a/backend/src/Taskdeck.Api/Extensions/ProtectedOutboundSentryHttpMessageHandlerFilter.cs +++ b/backend/src/Taskdeck.Api/Extensions/ProtectedOutboundSentryHttpMessageHandlerFilter.cs @@ -14,6 +14,7 @@ internal sealed class ProtectedOutboundSentryHttpMessageHandlerFilter : IHttpMes private static readonly HashSet ProtectedClientNames = new(StringComparer.Ordinal) { nameof(OpenAiLlmProvider), + LlmProviderRegistration.OpenAiCompatibleHttpClientName, nameof(GeminiLlmProvider), nameof(OllamaLlmProvider), "OutboundWebhookDelivery" diff --git a/backend/src/Taskdeck.Application/Services/ChatService.cs b/backend/src/Taskdeck.Application/Services/ChatService.cs index 438db904b..ea32b423d 100644 --- a/backend/src/Taskdeck.Application/Services/ChatService.cs +++ b/backend/src/Taskdeck.Application/Services/ChatService.cs @@ -16,6 +16,8 @@ public class ChatService : IChatService private const int MaxPromptLength = 4000; private const int MaxStreamedAssistantBytes = 1_048_576; private const int MaxChecklistItemCount = 30; + private const string StreamErrorPlaceholder = "The provider could not complete the response."; + private const string StreamErrorDegradedReason = "The upstream provider could not complete the response."; private static readonly Regex MentionRegex = new(@"(?[A-Za-z0-9_.-]{3,50})", RegexOptions.Compiled); private static readonly string[] PromptInjectionDenylist = { @@ -140,6 +142,7 @@ public async Task> SendMessageAsync(Guid sessionId, Guid string? quotaBilledModel = null; var quotaBilledTokens = 0; var quotaEstimatedTokens = 0; + LlmDispatchContext? quotaDispatchContext = null; try { if (string.IsNullOrWhiteSpace(dto.Content)) @@ -242,6 +245,7 @@ public async Task> SendMessageAsync(Guid sessionId, Guid toolChatMessages, Attribution: BuildAttribution(session, userId), SystemPrompt: ToolCallingSystemPrompt.Prompt); + quotaDispatchContext = toolCompletionRequest.DispatchContext; var toolResult = await _toolCallingOrchestrator.ExecuteAsync( toolCompletionRequest, session.BoardId.Value, userId, ct); @@ -379,17 +383,17 @@ await _quotaService.CommitReservationAsync( Attribution: BuildAttribution(session, userId), BoardContext: boardContext, SystemPrompt: clarificationPrompt); + quotaDispatchContext = completionRequest.DispatchContext; llmResult = await _llmProvider.CompleteAsync(completionRequest, ct); // Finalize with authoritative combined usage when present. If the provider // omitted usage, commit the reservation estimate so a short output cannot // undercharge a large input. Record the chosen total as input tokens and 0 // for output until providers surface a reliable split. - var quotaTokens = llmResult.HasAuthoritativeTokenUsage - ? llmResult.TokensUsed - : llmResult.ShouldSettleQuotaReservation - ? quotaEstimatedTokens - : 0; + var quotaTokens = ResolveQuotaTokens( + llmResult, + quotaEstimatedTokens, + completionRequest.DispatchContext); if (_quotaService != null && quotaReservationId is Guid singleResId && quotaTokens > 0) { // CancellationToken.None (M1, #1427 review): see the tool-result commit above. @@ -558,7 +562,21 @@ await _quotaService.CommitReservationAsync( } else { - await _quotaService.ReleaseReservationAsync(rid, CancellationToken.None); + var dispatch = quotaDispatchContext?.ReadSnapshot(); + if (dispatch is { Phase: LlmDispatchPhase.Dispatched } && quotaEstimatedTokens > 0) + { + await _quotaService.CommitReservationAsync( + rid, + userId, Domain.Enums.LlmSurface.Chat, + dispatch.Value.Provider ?? string.Empty, + dispatch.Value.Model ?? string.Empty, + quotaEstimatedTokens, 0, + CancellationToken.None); + } + else + { + await _quotaService.ReleaseReservationAsync(rid, CancellationToken.None); + } } } catch (Exception settleEx) @@ -615,10 +633,12 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, string? model = null; var streamIsDegraded = false; string? streamDegradedReason = null; - var persistEmptyDegradedTerminal = false; + var persistEmptyTerminalOutcome = false; + var terminalHadError = false; // True once the provider has delivered at least one non-error event: the LLM call was made and // tokens flowed, so the reservation is billable even if the final usage event never arrives. var providerStreamed = false; + LlmDispatchContext? quotaDispatchContext = null; // try/finally (no catch — legal around `yield` in an iterator) guarantees the reservation is // settled if the stream throws or yields no usable tokens. The scope opens immediately after @@ -638,6 +658,7 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, chatMessages, Attribution: BuildAttribution(session, userId), BoardContext: boardContext); + quotaDispatchContext = request.DispatchContext; await foreach (var token in _llmProvider.StreamAsync(request, ct)) { @@ -674,13 +695,16 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, { streamIsDegraded = true; streamDegradedReason = token.DegradedReason ?? "Provider returned a degraded stream."; - if (token.IsComplete && token.Error is null) - persistEmptyDegradedTerminal = true; + if (token.IsComplete) + persistEmptyTerminalOutcome = true; } if (!string.IsNullOrWhiteSpace(token.Error)) { streamIsDegraded = true; - streamDegradedReason = token.Error; + streamDegradedReason = StreamErrorDegradedReason; + terminalHadError = true; + if (token.IsComplete) + persistEmptyTerminalOutcome = true; } yield return token; @@ -689,8 +713,10 @@ public async IAsyncEnumerable StreamResponseAsync(Guid sessionId, // Persist the streamed assistant message with token usage so the streaming // path is consistent with the non-streaming SendMessageAsync path. var streamedContent = contentBuilder.ToString(); - if (streamedContent.Length == 0 && persistEmptyDegradedTerminal) - streamedContent = "The provider ended the response without returning text."; + if (streamedContent.Length == 0 && persistEmptyTerminalOutcome) + streamedContent = terminalHadError + ? StreamErrorPlaceholder + : "The provider ended the response without returning text."; if (!string.IsNullOrEmpty(streamedContent)) { var assistantMessage = new ChatMessage( @@ -742,22 +768,33 @@ await _quotaService.CommitReservationAsync( tokensUsed.Value, 0, CancellationToken.None); } - else if (providerStreamed) + else { - // No final count exists, so the reserved estimate is committed as input tokens + var dispatch = quotaDispatchContext?.ReadSnapshot(); + var shouldCommitEstimate = dispatch switch + { + { Phase: LlmDispatchPhase.Dispatched } => true, + { Phase: LlmDispatchPhase.ObservedPreDispatch } => false, + _ => providerStreamed + } && quotaEstimatedTokens > 0; + if (!shouldCommitEstimate) + { + await _quotaService.ReleaseReservationAsync(rid, CancellationToken.None); + } + else + { + // No final count exists, so the reserved estimate is committed as input tokens // (output 0) — the deliberate over-count-not-bypass posture: better to charge // the estimate than to let abandonment erase real usage. Unknown provider/model - // fall back to the repository's reservation placeholder. - await _quotaService.CommitReservationAsync( - rid, - userId, Domain.Enums.LlmSurface.Chat, - provider ?? string.Empty, model ?? string.Empty, - quotaEstimatedTokens, 0, - CancellationToken.None); - } - else - { - await _quotaService.ReleaseReservationAsync(rid, CancellationToken.None); + // fall back to the repository's reservation placeholder. + await _quotaService.CommitReservationAsync( + rid, + userId, Domain.Enums.LlmSurface.Chat, + provider ?? dispatch?.Provider ?? string.Empty, + model ?? dispatch?.Model ?? string.Empty, + quotaEstimatedTokens, 0, + CancellationToken.None); + } } } catch (Exception settleEx) @@ -773,6 +810,26 @@ await _quotaService.CommitReservationAsync( } } + private static int ResolveQuotaTokens( + LlmCompletionResult result, + int reservationEstimate, + LlmDispatchContext dispatchContext) + { + var dispatch = dispatchContext.ReadSnapshot(); + if (dispatch.Phase == LlmDispatchPhase.ObservedPreDispatch) + return 0; + if (dispatch.Phase == LlmDispatchPhase.Dispatched) + return result.HasAuthoritativeTokenUsage && result.TokensUsed > 0 + ? result.TokensUsed + : reservationEstimate; + + return result.HasAuthoritativeTokenUsage + ? result.TokensUsed + : result.ShouldSettleQuotaReservation + ? reservationEstimate + : 0; + } + private static bool ContainsBlockedPromptPattern(string content) { var normalized = content.ToLowerInvariant(); diff --git a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs index ea7c0f283..1a49d60cd 100644 --- a/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs +++ b/backend/src/Taskdeck.Application/Services/CircuitBreakerStateTracker.cs @@ -3,50 +3,69 @@ namespace Taskdeck.Application.Services; /// -/// Thread-safe singleton that tracks the state of Polly circuit breakers -/// for external service HTTP clients. The health endpoint reads this to -/// report whether circuits are closed, open, or half-open. +/// Thread-safe singleton that tracks Polly circuit state and the companion +/// circuit used for failures that occur after response headers are received. /// public sealed class CircuitBreakerStateTracker { - private readonly ConcurrentDictionary _states = new(); + private readonly ConcurrentDictionary _pollyStates = new(); private readonly ConcurrentDictionary _providerFailures = new(); + private readonly TimeProvider _timeProvider; + + public CircuitBreakerStateTracker() + : this(TimeProvider.System) + { + } + + internal CircuitBreakerStateTracker(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } /// - /// Records a circuit state transition. Called by the Polly onBreak, - /// onReset, and onHalfOpen delegates. + /// Records a Polly circuit transition. Companion-provider transitions are + /// deliberately tracked in a separate lane so neither circuit can mask the other. /// public void RecordState(string circuitName, CircuitState state, string? lastFailureReason = null) { - _states[circuitName] = new CircuitBreakerSnapshot( - circuitName, - state, - DateTimeOffset.UtcNow, - lastFailureReason); + _pollyStates[circuitName] = CreateSnapshot(circuitName, state, lastFailureReason); } /// - /// Returns the current snapshot for every tracked circuit. + /// Returns the most restrictive current snapshot for every tracked circuit. /// public IReadOnlyDictionary GetAll() { - return _states; + var names = new HashSet(_pollyStates.Keys, StringComparer.Ordinal); + names.UnionWith(_providerFailures.Keys); + + var result = new Dictionary(StringComparer.Ordinal); + foreach (var name in names) + { + var snapshot = Get(name); + if (snapshot is not null) + result[name] = snapshot; + } + + return result; } /// - /// Returns the snapshot for a single circuit, or null if the circuit has - /// never transitioned (i.e., it has been closed since startup). + /// Returns the most restrictive Polly or companion snapshot for one circuit. /// public CircuitBreakerSnapshot? Get(string circuitName) { - return _states.TryGetValue(circuitName, out var snapshot) ? snapshot : null; + _pollyStates.TryGetValue(circuitName, out var polly); + var companion = _providerFailures.TryGetValue(circuitName, out var state) + ? state.PublicSnapshot + : null; + return SelectMostRestrictive(polly, companion); } /// /// Applies the configured circuit posture to failures that occur after HTTP - /// response headers. Polly cannot observe body-read, SSE-parse, or idle-timeout - /// failures when callers use ResponseHeadersRead, so providers report those - /// failures explicitly through this companion gate. + /// response headers. Every admitted request receives a generation-bearing + /// lease so outcomes from an older generation cannot mutate newer state. /// internal bool TryEnterProviderRequest( string circuitName, @@ -56,12 +75,7 @@ internal bool TryEnterProviderRequest( { while (true) { - if (!_providerFailures.TryGetValue(circuitName, out var state)) - { - lease = default; - error = null; - return true; - } + var state = _providerFailures.GetOrAdd(circuitName, static _ => ProviderFailureState.Initial); if (state.HalfOpenProbeId is not null) { @@ -70,30 +84,32 @@ internal bool TryEnterProviderRequest( return false; } - if (state.OpenUntilUtc is null) + var now = _timeProvider.GetUtcNow(); + if (state.OpenUntilUtc is not null && state.OpenUntilUtc > now) { lease = default; - error = null; - return true; + error = $"{circuitName} provider circuit is open after repeated transport, body, or protocol failures."; + return false; } - var now = DateTimeOffset.UtcNow; - if (state.OpenUntilUtc > now) + if (state.OpenUntilUtc is null) { - lease = default; - error = $"{circuitName} provider circuit is open after repeated transport, body, or protocol failures."; - return false; + lease = new CircuitRequestLease(state.Generation, null, IsTracked: true); + error = null; + return true; } var probeId = Guid.NewGuid(); - var halfOpen = new ProviderFailureState( - Math.Max(0, settings.FailureThreshold - 1), - null, - probeId); + var halfOpen = state with + { + Generation = state.Generation + 1, + OpenUntilUtc = null, + HalfOpenProbeId = probeId, + PublicSnapshot = CreateSnapshot(circuitName, CircuitState.HalfOpen) + }; if (_providerFailures.TryUpdate(circuitName, halfOpen, state)) { - RecordState(circuitName, CircuitState.HalfOpen); - lease = new CircuitRequestLease(probeId); + lease = new CircuitRequestLease(halfOpen.Generation, probeId, IsTracked: true); error = null; return true; } @@ -106,106 +122,162 @@ internal void RecordProviderFailure( string reason, CircuitRequestLease lease) { - var now = DateTimeOffset.UtcNow; - while (true) + if (!lease.IsTracked) + return; + + while (_providerFailures.TryGetValue(circuitName, out var existing)) { - if (!_providerFailures.TryGetValue(circuitName, out var existing)) - { - if (lease.IsHalfOpenProbe) - return; + if (!LeaseMatches(existing, lease)) + return; - var initial = new ProviderFailureState(1, null, null); - if (!_providerFailures.TryAdd(circuitName, initial)) - continue; - existing = initial; + var now = _timeProvider.GetUtcNow(); + ProviderFailureState next; + if (lease.IsHalfOpenProbe) + { + next = new ProviderFailureState( + existing.Generation + 1, + settings.FailureThreshold, + now.AddSeconds(settings.BreakDurationSeconds), + null, + CreateSnapshot(circuitName, CircuitState.Open, reason)); } else { - // Ignore stale completions from requests that pre-date a half-open probe, - // and ignore outcomes from a superseded probe lease. - if (existing.HalfOpenProbeId is not null && - existing.HalfOpenProbeId != lease.HalfOpenProbeId) - return; - if (lease.HalfOpenProbeId is not null && - existing.HalfOpenProbeId != lease.HalfOpenProbeId) - return; - if (existing.OpenUntilUtc is not null && existing.OpenUntilUtc > now && - lease.HalfOpenProbeId is null) + if (existing.OpenUntilUtc is not null || existing.HalfOpenProbeId is not null) return; - var failureCount = lease.IsHalfOpenProbe - ? settings.FailureThreshold - : existing.ConsecutiveFailures + 1; - var next = new ProviderFailureState(failureCount, null, null); - if (!_providerFailures.TryUpdate(circuitName, next, existing)) - continue; - existing = next; + var failureCount = existing.ConsecutiveFailures + 1; + next = failureCount >= settings.FailureThreshold + ? new ProviderFailureState( + existing.Generation + 1, + failureCount, + now.AddSeconds(settings.BreakDurationSeconds), + null, + CreateSnapshot(circuitName, CircuitState.Open, reason)) + : existing with { ConsecutiveFailures = failureCount }; } - if (existing.ConsecutiveFailures < settings.FailureThreshold) - return; - - var opened = new ProviderFailureState( - existing.ConsecutiveFailures, - now.AddSeconds(settings.BreakDurationSeconds), - null); - if (_providerFailures.TryUpdate(circuitName, opened, existing)) - { - RecordState(circuitName, CircuitState.Open, reason); + if (_providerFailures.TryUpdate(circuitName, next, existing)) return; - } } } internal void RecordProviderSuccess(string circuitName, CircuitRequestLease lease) { + if (!lease.IsTracked) + return; + while (_providerFailures.TryGetValue(circuitName, out var existing)) { - if (lease.HalfOpenProbeId is not null && existing.HalfOpenProbeId != lease.HalfOpenProbeId) + if (!LeaseMatches(existing, lease)) return; - if (lease.HalfOpenProbeId is null && existing.HalfOpenProbeId is not null) - return; - if (((ICollection>)_providerFailures).Remove( - new KeyValuePair(circuitName, existing))) + + ProviderFailureState next; + if (lease.IsHalfOpenProbe) { - RecordState(circuitName, CircuitState.Closed); - return; + next = new ProviderFailureState( + existing.Generation + 1, + 0, + null, + null, + CreateSnapshot(circuitName, CircuitState.Closed)); } + else + { + if (existing.OpenUntilUtc is not null || existing.HalfOpenProbeId is not null) + return; + + next = existing with + { + ConsecutiveFailures = 0, + PublicSnapshot = existing.ConsecutiveFailures > 0 + ? CreateSnapshot(circuitName, CircuitState.Closed) + : existing.PublicSnapshot + }; + } + + if (_providerFailures.TryUpdate(circuitName, next, existing)) + return; } } - internal void AbandonProviderRequest(string circuitName, CircuitRequestLease lease) + internal void AbandonProviderRequest( + string circuitName, + CircuitBreakerSettings settings, + CircuitRequestLease lease) { - if (!lease.IsHalfOpenProbe) + if (!lease.IsTracked || !lease.IsHalfOpenProbe) return; while (_providerFailures.TryGetValue(circuitName, out var existing)) { - if (existing.HalfOpenProbeId != lease.HalfOpenProbeId) + if (!LeaseMatches(existing, lease)) return; - // Release the exclusive probe immediately while retaining the open posture. - // The next caller can acquire a fresh half-open lease without waiting through - // another break duration after cancellation or iterator disposal. - var released = new ProviderFailureState( - existing.ConsecutiveFailures, - DateTimeOffset.UtcNow, - null); - if (_providerFailures.TryUpdate(circuitName, released, existing)) - { - RecordState(circuitName, CircuitState.Open, "Half-open provider probe was abandoned."); + var now = _timeProvider.GetUtcNow(); + var reopened = new ProviderFailureState( + existing.Generation + 1, + Math.Max(existing.ConsecutiveFailures, settings.FailureThreshold), + now.AddSeconds(settings.BreakDurationSeconds), + null, + CreateSnapshot( + circuitName, + CircuitState.Open, + "Half-open provider probe was abandoned.")); + if (_providerFailures.TryUpdate(circuitName, reopened, existing)) return; - } } } + private CircuitBreakerSnapshot CreateSnapshot( + string circuitName, + CircuitState state, + string? lastFailureReason = null) => + new(circuitName, state, _timeProvider.GetUtcNow(), lastFailureReason); + + private static bool LeaseMatches(ProviderFailureState state, CircuitRequestLease lease) => + state.Generation == lease.Generation && + state.HalfOpenProbeId == lease.HalfOpenProbeId; + + private static CircuitBreakerSnapshot? SelectMostRestrictive( + CircuitBreakerSnapshot? first, + CircuitBreakerSnapshot? second) + { + if (first is null) + return second; + if (second is null) + return first; + + var firstRank = GetRestrictiveness(first.State); + var secondRank = GetRestrictiveness(second.State); + if (firstRank != secondRank) + return firstRank > secondRank ? first : second; + + return first.LastTransitionUtc >= second.LastTransitionUtc ? first : second; + } + + private static int GetRestrictiveness(CircuitState state) => state switch + { + CircuitState.Open => 2, + CircuitState.HalfOpen => 1, + _ => 0 + }; + private sealed record ProviderFailureState( + long Generation, int ConsecutiveFailures, DateTimeOffset? OpenUntilUtc, - Guid? HalfOpenProbeId); + Guid? HalfOpenProbeId, + CircuitBreakerSnapshot? PublicSnapshot) + { + public static ProviderFailureState Initial { get; } = new(0, 0, null, null, null); + } } -internal readonly record struct CircuitRequestLease(Guid? HalfOpenProbeId) +internal readonly record struct CircuitRequestLease( + long Generation, + Guid? HalfOpenProbeId, + bool IsTracked) { public bool IsHalfOpenProbe => HalfOpenProbeId is not null; } diff --git a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs index 925eb6ab3..767210672 100644 --- a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs @@ -31,7 +31,16 @@ public record ChatCompletionRequest( LlmRequestAttribution? Attribution = null, string? SystemPrompt = null, string? BoardContext = null -); +) +{ + /// + /// Per-request transport participation state. Providers that support the + /// dispatch boundary observe this context before validation and mark it only + /// when the request reaches the innermost HTTP transport handler. + /// + [JsonIgnore] + internal LlmDispatchContext DispatchContext { get; init; } = new(); +} public record ChatCompletionMessage(string Role, string Content); @@ -121,6 +130,60 @@ internal enum LlmProviderFailureKind ResponseLimit = 5 } +internal enum LlmDispatchPhase +{ + Unobserved = 0, + ObservedPreDispatch = 1, + Dispatched = 2 +} + +internal readonly record struct LlmDispatchSnapshot( + LlmDispatchPhase Phase, + string? Provider, + string? Model); + +/// +/// Monotonic request-scoped state shared by the quota owner and HTTP pipeline. +/// A lock keeps provider/model identity and the phase visible as one snapshot. +/// +internal sealed class LlmDispatchContext +{ + private readonly object _gate = new(); + private LlmDispatchPhase _phase; + private string? _provider; + private string? _model; + + public void Observe(string provider, string model) + { + lock (_gate) + { + if (_phase != LlmDispatchPhase.Unobserved) + return; + + _provider = provider; + _model = model; + _phase = LlmDispatchPhase.ObservedPreDispatch; + } + } + + public void MarkDispatched() + { + lock (_gate) + { + if (_phase == LlmDispatchPhase.ObservedPreDispatch) + _phase = LlmDispatchPhase.Dispatched; + } + } + + public LlmDispatchSnapshot ReadSnapshot() + { + lock (_gate) + { + return new LlmDispatchSnapshot(_phase, _provider, _model); + } + } +} + public record LlmHealthStatus( bool IsAvailable, string ProviderName, diff --git a/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs b/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs index 7935fa09f..54b666f54 100644 --- a/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs +++ b/backend/src/Taskdeck.Application/Services/LlmCaptureTriageExtractor.cs @@ -113,7 +113,7 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell // for a dispatched call so short output cannot undercharge a large transcript. if (quotaReservationId is Guid reservationId) { - var quotaTokens = ResolveQuotaTokens(result, quotaEstimatedTokens); + var quotaTokens = ResolveQuotaTokens(result, quotaEstimatedTokens, request.DispatchContext); if (quotaTokens > 0) { // CancellationToken.None (M1, #1427 review): tokens are already billed at this @@ -147,17 +147,22 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell { try { + var dispatch = request.DispatchContext.ReadSnapshot(); var quotaTokens = completed is null - ? 0 - : ResolveQuotaTokens(completed, quotaEstimatedTokens); - if (completed is { } billed && quotaTokens > 0) + ? dispatch.Phase == LlmDispatchPhase.Dispatched + ? quotaEstimatedTokens + : 0 + : ResolveQuotaTokens(completed, quotaEstimatedTokens, request.DispatchContext); + var billedProvider = completed?.Provider ?? dispatch.Provider; + var billedModel = completed?.Model ?? dispatch.Model; + if (quotaTokens > 0 && billedProvider is not null && billedModel is not null) { await _quotaService!.CommitReservationAsync( unsettledId, userId, LlmSurface.CaptureTriage, - billed.Provider, - billed.Model, + billedProvider, + billedModel, quotaTokens, 0, CancellationToken.None); @@ -174,7 +179,11 @@ await _killSwitchService.IsKilledAsync(LlmSurface.CaptureTriage, userId, cancell "Quota reservation {ReservationId} settle failed in transcript triage (billed tokens: {Tokens}); " + "the row stays Reserved until the TTL sweep.", unsettledId, - completed is null ? 0 : ResolveQuotaTokens(completed, quotaEstimatedTokens)); + completed is null + ? request.DispatchContext.ReadSnapshot().Phase == LlmDispatchPhase.Dispatched + ? quotaEstimatedTokens + : 0 + : ResolveQuotaTokens(completed, quotaEstimatedTokens, request.DispatchContext)); } } } @@ -280,10 +289,23 @@ private static List SanitizeTasks(IReadOnlyList - result.HasAuthoritativeTokenUsage + private static int ResolveQuotaTokens( + LlmCompletionResult result, + int reservationEstimate, + LlmDispatchContext dispatchContext) + { + var dispatch = dispatchContext.ReadSnapshot(); + if (dispatch.Phase == LlmDispatchPhase.ObservedPreDispatch) + return 0; + if (dispatch.Phase == LlmDispatchPhase.Dispatched) + return result.HasAuthoritativeTokenUsage && result.TokensUsed > 0 + ? result.TokensUsed + : reservationEstimate; + + return result.HasAuthoritativeTokenUsage ? result.TokensUsed : result.ShouldSettleQuotaReservation ? reservationEstimate : 0; + } } diff --git a/backend/src/Taskdeck.Application/Services/LlmDispatchTrackingHandler.cs b/backend/src/Taskdeck.Application/Services/LlmDispatchTrackingHandler.cs new file mode 100644 index 000000000..830334943 --- /dev/null +++ b/backend/src/Taskdeck.Application/Services/LlmDispatchTrackingHandler.cs @@ -0,0 +1,29 @@ +namespace Taskdeck.Application.Services; + +/// +/// Marks a compatible-provider request as dispatched only after every outer +/// validation, egress, telemetry, and circuit handler has admitted it. +/// +internal sealed class LlmDispatchTrackingHandler : DelegatingHandler +{ + private static readonly HttpRequestOptionsKey DispatchContextKey = + new("Taskdeck.LlmDispatchContext"); + + internal static void Attach(HttpRequestMessage message, LlmDispatchContext dispatchContext) + { + ArgumentNullException.ThrowIfNull(message); + ArgumentNullException.ThrowIfNull(dispatchContext); + message.Options.Set(DispatchContextKey, dispatchContext); + } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (request.Options.TryGetValue(DispatchContextKey, out var dispatchContext)) + dispatchContext.MarkDispatched(); + + return base.SendAsync(request, cancellationToken); + } +} diff --git a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs index 673daf943..56a453acb 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiCompatibleLlmProvider.cs @@ -26,12 +26,13 @@ internal sealed class OpenAiCompatibleLlmProvider : ILlmProvider private readonly CircuitBreakerStateTracker? _circuitBreakerTracker; private readonly CircuitBreakerSettings? _circuitBreakerSettings; private readonly bool _allowLocalhostEndpoints; + private readonly bool _protectOutboundTelemetry; internal OpenAiCompatibleLlmProvider( HttpClient httpClient, LlmProviderSettings settings, ILogger logger) - : this(httpClient, settings, logger, null, null, false) + : this(httpClient, settings, logger, null, null, null) { } @@ -41,14 +42,15 @@ internal OpenAiCompatibleLlmProvider( ILogger logger, CircuitBreakerStateTracker? circuitBreakerTracker, CircuitBreakerSettings? circuitBreakerSettings, - bool allowLocalhostEndpoints) + LlmProviderRuntimePolicy? runtimePolicy) { _httpClient = httpClient; _settings = settings; _logger = logger; _circuitBreakerTracker = circuitBreakerTracker; _circuitBreakerSettings = circuitBreakerSettings; - _allowLocalhostEndpoints = allowLocalhostEndpoints; + _allowLocalhostEndpoints = runtimePolicy?.AllowGeneralProviderLocalhost ?? false; + _protectOutboundTelemetry = runtimePolicy?.ProtectOutboundTelemetry ?? false; } public Task CompleteAsync(ChatCompletionRequest request, CancellationToken ct = default) => @@ -59,6 +61,7 @@ private async Task CompleteCoreAsync( CancellationToken ct, bool participateInCircuit) { + request.DispatchContext.Observe(ProviderName, GetConfiguredModelOrDefault()); var userMessage = GetLastUserMessage(request); if (!TryValidateSettings(out var validationError)) { @@ -168,6 +171,10 @@ private async Task CompleteCoreAsync( { throw; } + catch (EgressViolationException) + { + throw; + } catch (OperationCanceledException) { _logger.LogWarning("OpenAI-compatible completion exceeded its configured response deadline."); @@ -244,6 +251,7 @@ public async IAsyncEnumerable StreamAsync( ChatCompletionRequest request, [EnumeratorCancellation] CancellationToken ct = default) { + request.DispatchContext.Observe(ProviderName, GetConfiguredModelOrDefault()); if (!TryValidateSettings(out var validationError)) { yield return TerminalError( @@ -279,6 +287,10 @@ public async IAsyncEnumerable StreamAsync( { throw; } + catch (EgressViolationException) + { + throw; + } catch (OperationCanceledException) { moved = false; @@ -359,12 +371,14 @@ private async IAsyncEnumerable StreamCoreAsync( // contract. Use this same effective request for a rejected-stream fallback. var streamingRequest = request.SystemPrompt is null ? request with { SystemPrompt = string.Empty } : request; using var message = CreateRequestMessage(streamingRequest, stream: true, includeResponseFormat: false); + PrepareForDispatch(message, streamingRequest.DispatchContext); using var response = await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); if (!response.IsSuccessStatusCode) { if (IsStreamingRejection(response.StatusCode)) { + response.Dispose(); await foreach (var fallback in EmitBufferedFallbackAsync(streamingRequest, ct)) yield return fallback; yield break; @@ -405,13 +419,14 @@ private async IAsyncEnumerable StreamCoreAsync( } await using var stream = await response.Content.ReadAsStreamAsync(ct); - using var reader = new StreamReader( - stream, - new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true), - detectEncodingFromByteOrderMarks: true, - bufferSize: 4096, - leaveOpen: false); - var lineReader = new BoundedSseLineReader(reader); + if (response.Content.Headers.ContentLength is long contentLength && + contentLength > compatible.MaxResponseBytes) + { + throw new LlmProviderResponseLimitException( + "Content-Length exceeded the aggregate SSE response byte budget"); + } + + var lineReader = new BoundedUtf8SseLineReader(stream); var eventData = new StringBuilder(); var eventBytes = 0; var responseBytes = 0; @@ -424,7 +439,7 @@ private async IAsyncEnumerable StreamCoreAsync( if (line.Text is null) break; - responseBytes = checked(responseBytes + line.Utf8Bytes + line.DelimiterUtf8Bytes); + responseBytes = checked(responseBytes + line.WireBytes); if (responseBytes > compatible.MaxResponseBytes) throw new LlmProviderResponseLimitException("aggregate SSE response byte budget exceeded"); @@ -558,9 +573,17 @@ private async Task SendCompletionAsync( CancellationToken ct) { using var message = CreateRequestMessage(request, stream: false, includeResponseFormat); + PrepareForDispatch(message, request.DispatchContext); return await _httpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, ct); } + private void PrepareForDispatch(HttpRequestMessage message, LlmDispatchContext dispatchContext) + { + LlmDispatchTrackingHandler.Attach(message, dispatchContext); + if (_protectOutboundTelemetry) + ProtectedOutboundTelemetryHandler.PrepareForSend(message); + } + private bool TryValidateSettings(out string error) { if (!_settings.EnableLiveProviders) @@ -633,24 +656,31 @@ private static SseEventParseResult TryParseSseEvent(string data) return new SseEventParseResult(Error: "OpenAI-compatible SSE stream reported an upstream error."); int? tokensUsed = null; + var hasUsageMetadata = false; if (root.TryGetProperty("usage", out var usage)) { - if (usage.ValueKind != JsonValueKind.Null && - (usage.ValueKind != JsonValueKind.Object || - !usage.TryGetProperty("total_tokens", out var total) || - total.ValueKind != JsonValueKind.Number || - !total.TryGetInt32(out var parsedTokens) || parsedTokens < 0)) + if (usage.ValueKind == JsonValueKind.Object) + { + if (!usage.TryGetProperty("total_tokens", out var total) || + total.ValueKind != JsonValueKind.Number || + !total.TryGetInt32(out var parsedTokens) || parsedTokens < 0) + { + return new SseEventParseResult(Error: "OpenAI-compatible SSE usage metadata was malformed."); + } + + hasUsageMetadata = true; + tokensUsed = parsedTokens > 0 ? parsedTokens : null; + } + else if (usage.ValueKind != JsonValueKind.Null) { return new SseEventParseResult(Error: "OpenAI-compatible SSE usage metadata was malformed."); } - if (usage.ValueKind == JsonValueKind.Object) - tokensUsed = usage.GetProperty("total_tokens").GetInt32(); } if (!root.TryGetProperty("choices", out var choices) || choices.ValueKind != JsonValueKind.Array) return new SseEventParseResult(Error: "OpenAI-compatible SSE event did not contain a choices array."); if (choices.GetArrayLength() == 0) - return tokensUsed is not null + return hasUsageMetadata ? new SseEventParseResult(TokensUsed: tokensUsed) : new SseEventParseResult(Error: "OpenAI-compatible SSE event contained no choices or usage."); @@ -706,12 +736,7 @@ private static bool TryParseResponse(string body, out string content, out int? t return false; var choice = choices[0]; if (!choice.TryGetProperty("message", out var message) || - message.ValueKind != JsonValueKind.Object || - !message.TryGetProperty("content", out var contentElement) || - contentElement.ValueKind != JsonValueKind.String) - return false; - content = contentElement.GetString() ?? string.Empty; - if (string.IsNullOrWhiteSpace(content)) + message.ValueKind != JsonValueKind.Object) return false; if (choice.TryGetProperty("finish_reason", out var finish)) @@ -722,6 +747,34 @@ private static bool TryParseResponse(string body, out string content, out int? t return false; } + var hasSanitizedRefusal = false; + if (message.TryGetProperty("refusal", out var refusal)) + { + if (refusal.ValueKind == JsonValueKind.String) + hasSanitizedRefusal = !string.IsNullOrWhiteSpace(refusal.GetString()); + else if (refusal.ValueKind != JsonValueKind.Null) + return false; + } + + if (!message.TryGetProperty("content", out var contentElement)) + return false; + + if (contentElement.ValueKind == JsonValueKind.String) + content = contentElement.GetString() ?? string.Empty; + else if (contentElement.ValueKind != JsonValueKind.Null) + return false; + + var isContentFilter = string.Equals( + finishReason, + "content_filter", + StringComparison.OrdinalIgnoreCase); + if (string.IsNullOrWhiteSpace(content) && !isContentFilter && !hasSanitizedRefusal) + return false; + if (hasSanitizedRefusal && !isContentFilter) + finishReason = "refusal"; + if (isContentFilter || hasSanitizedRefusal) + content = string.Empty; + if (root.TryGetProperty("usage", out var usage)) { if (usage.ValueKind != JsonValueKind.Object || @@ -729,7 +782,7 @@ private static bool TryParseResponse(string body, out string content, out int? t total.ValueKind != JsonValueKind.Number || !total.TryGetInt32(out var parsedTokens) || parsedTokens < 0) return false; - tokensUsed = parsedTokens; + tokensUsed = parsedTokens > 0 ? parsedTokens : null; } return true; } @@ -827,8 +880,11 @@ private void RecordProviderFailure(string reason, CircuitRequestLease lease) private void RecordProviderSuccess(CircuitRequestLease lease) => _circuitBreakerTracker?.RecordProviderSuccess(CircuitName, lease); - private void AbandonProviderRequest(CircuitRequestLease lease) => - _circuitBreakerTracker?.AbandonProviderRequest(CircuitName, lease); + private void AbandonProviderRequest(CircuitRequestLease lease) + { + if (_circuitBreakerTracker is not null && _circuitBreakerSettings is not null) + _circuitBreakerTracker.AbandonProviderRequest(CircuitName, _circuitBreakerSettings, lease); + } private static bool IsProviderFailure(LlmTokenEvent terminal) => terminal.Error is not null || terminal.CountsAsProviderFailure; @@ -841,6 +897,7 @@ private static bool IsProviderFailure(LlmTokenEvent terminal) => { "length" => "Response was truncated because the upstream token limit was reached.", "content_filter" => "Response was stopped by the upstream content filter.", + "refusal" => "The upstream provider refused this request.", "tool_calls" => "Response ended with unsupported upstream tool calls.", "function_call" => "Response ended with an unsupported upstream function call.", null or "" => null, @@ -914,61 +971,140 @@ private sealed record SseEventParseResult( int? TokensUsed = null, string? Error = null); - private sealed record BoundedLine(string? Text, int Utf8Bytes, int DelimiterUtf8Bytes); + private sealed record BoundedLine(string? Text, int WireBytes); - private sealed class BoundedSseLineReader(StreamReader reader) + /// + /// Reads SSE lines from their original bytes so response and line limits are + /// enforced on the wire representation, not reconstructed UTF-16 characters. + /// Only strict UTF-8 is accepted; one initial UTF-8 BOM is tolerated and + /// counted toward the aggregate response budget. + /// + private sealed class BoundedUtf8SseLineReader(Stream stream) { - private readonly char[] _one = new char[1]; - private char? _pending; + private static readonly UTF8Encoding StrictUtf8 = new(false, true); + private readonly byte[] _buffer = new byte[4096]; + private readonly Queue _initialBytes = new(); + private int _bufferOffset; + private int _bufferCount; + private int? _pendingByte; + private bool _initialized; + private int _initialBomBytes; public async Task ReadLineAsync(int maxBytes, CancellationToken ct) { - var line = new StringBuilder(); - var bytes = 0; + await InitializeAsync(ct); + + using var line = new MemoryStream(Math.Min(maxBytes, 4096)); + var prefixBytes = _initialBomBytes; + _initialBomBytes = 0; + while (true) { - var next = await ReadCharacterAsync(ct); + var next = await ReadByteAsync(ct); if (next is null) - return line.Length == 0 - ? new BoundedLine(null, 0, 0) - : new BoundedLine(line.ToString(), bytes, 0); + { + if (line.Length == 0 && prefixBytes == 0) + return new BoundedLine(null, 0); + + return new BoundedLine(Decode(line), checked(prefixBytes + (int)line.Length)); + } - if (next == '\n') - return new BoundedLine(line.ToString(), bytes, 1); + if (next == (byte)'\n') + { + return new BoundedLine( + Decode(line), + checked(prefixBytes + (int)line.Length + 1)); + } - if (next == '\r') + if (next == (byte)'\r') { - var afterCarriageReturn = await ReadCharacterAsync(ct); var delimiterBytes = 1; - if (afterCarriageReturn == '\n') + var afterCarriageReturn = await ReadByteAsync(ct); + if (afterCarriageReturn == (byte)'\n') { delimiterBytes = 2; } else if (afterCarriageReturn is not null) { - _pending = afterCarriageReturn; + _pendingByte = afterCarriageReturn; } - return new BoundedLine(line.ToString(), bytes, delimiterBytes); + return new BoundedLine( + Decode(line), + checked(prefixBytes + (int)line.Length + delimiterBytes)); } - bytes = checked(bytes + Encoding.UTF8.GetByteCount([next.Value])); - if (bytes > maxBytes) + if (line.Length >= maxBytes) throw new LlmProviderResponseLimitException("single SSE line byte budget exceeded"); - line.Append(next.Value); + line.WriteByte((byte)next); } } - private async ValueTask ReadCharacterAsync(CancellationToken ct) + private async Task InitializeAsync(CancellationToken ct) { - if (_pending is { } pending) + if (_initialized) + return; + _initialized = true; + + var first = await ReadRawByteAsync(ct); + if (first is null) + return; + if (first != 0xEF) + { + _initialBytes.Enqueue((byte)first); + return; + } + + var second = await ReadRawByteAsync(ct); + var third = await ReadRawByteAsync(ct); + if (second == 0xBB && third == 0xBF) { - _pending = null; + _initialBomBytes = 3; + return; + } + + _initialBytes.Enqueue((byte)first); + if (second is not null) + _initialBytes.Enqueue((byte)second); + if (third is not null) + _initialBytes.Enqueue((byte)third); + } + + private async ValueTask ReadByteAsync(CancellationToken ct) + { + if (_pendingByte is { } pending) + { + _pendingByte = null; return pending; } + if (_initialBytes.Count > 0) + return _initialBytes.Dequeue(); + return await ReadRawByteAsync(ct); + } + + private async ValueTask ReadRawByteAsync(CancellationToken ct) + { + if (_bufferOffset >= _bufferCount) + { + _bufferCount = await stream.ReadAsync(_buffer.AsMemory(), ct); + _bufferOffset = 0; + if (_bufferCount == 0) + return null; + } - var read = await reader.ReadAsync(_one.AsMemory(), ct); - return read == 0 ? null : _one[0]; + return _buffer[_bufferOffset++]; + } + + private static string Decode(MemoryStream line) + { + try + { + return StrictUtf8.GetString(line.GetBuffer(), 0, checked((int)line.Length)); + } + catch (DecoderFallbackException ex) + { + throw new IOException("OpenAI-compatible SSE response was not valid UTF-8.", ex); + } } } diff --git a/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs b/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs index f65c5a365..6de4c2ac3 100644 --- a/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs +++ b/backend/src/Taskdeck.Domain/Agents/EgressViolation.cs @@ -11,7 +11,7 @@ public sealed class EgressViolation /// The host that was attempted but is not in the egress envelope. public string AttemptedHost { get; } - /// The original request URI that triggered the violation. + /// The sanitized audit origin for the request that triggered the violation. public string RequestUri { get; } /// Category of the violation. diff --git a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs index 1b5af210c..0624bef5f 100644 --- a/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/LlmProviderRegistrationTests.cs @@ -71,6 +71,12 @@ public void RegisteredPipelines_KeepCompatibleProviderInsideProtectedDirectEgres EnumeratePipeline(ollama).OfType().Single().UseProxy.Should().BeFalse(); ProxySafeHttpHandlerTestHarness.AssertProxySafeOriginHandler(compatible); EnumeratePipeline(compatible).Should().Contain(item => item is EgressEnvelopeHandler); + EnumeratePipeline(compatible).Select(item => item.GetType().Name).Should().ContainInOrder( + "PolicyHttpMessageHandler", + nameof(ProtectedOutboundTelemetryHandler), + nameof(EgressEnvelopeHandler), + "LlmDispatchTrackingHandler", + nameof(SocketsHttpHandler)); var workerServices = new ServiceCollection(); workerServices.AddLogging(); @@ -96,27 +102,41 @@ public async Task CompatibleClientPipeline_RefusesEveryRedirectWithoutFollowing( using var handler = provider.GetRequiredService() .CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); var egressHandler = EnumeratePipeline(handler).OfType().Single(); - var redirectHandler = new RedirectStubHandler(statusCode, "https://api.groq.com/second-hop"); + const string sensitiveMarker = "must-not-appear-in-egress-audit"; + var redirectHandler = new RedirectStubHandler( + statusCode, + $"https://api.groq.com/second-hop?marker={sensitiveMarker}"); egressHandler.InnerHandler = redirectHandler; using var invoker = new HttpMessageInvoker(handler); - var act = () => invoker.SendAsync( - new HttpRequestMessage(HttpMethod.Get, "https://api.groq.com/openai/v1/chat/completions"), - CancellationToken.None); + using var request = new HttpRequestMessage( + HttpMethod.Get, + $"https://api.groq.com/openai/v1/chat/completions?marker={sensitiveMarker}"); + ProtectedOutboundTelemetryHandler.PrepareForSend(request); + var act = () => invoker.SendAsync(request, CancellationToken.None); var exception = await act.Should().ThrowAsync(); exception.Which.Violation.ViolationType.Should().Be(Taskdeck.Domain.Agents.EgressViolationType.RedirectNotAllowed); + exception.Which.Violation.RequestUri.Should().Be("https://api.groq.com"); + exception.Which.ToString().Should().NotContain(sensitiveMarker); + request.RequestUri!.Host.Should().Be("protected-outbound.invalid", + "the protected request must be remasked even when the egress boundary throws"); redirectHandler.InvocationCount.Should().Be(1, "the compatible pipeline must never dispatch a redirected request"); } [Theory] [InlineData(nameof(OpenAiLlmProvider))] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName)] [InlineData(nameof(GeminiLlmProvider))] [InlineData(nameof(OllamaLlmProvider))] public void AddLlmProviders_ShouldDisableProxyAndRetainOriginGuards_OnFactoryPipeline( string clientName) { - using var serviceProvider = BuildServiceProvider("Production"); + using var serviceProvider = BuildServiceProvider( + "Production", + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName + ? "OpenAiCompatible" + : "Mock"); var pipeline = serviceProvider .GetRequiredService() @@ -129,6 +149,9 @@ public void AddLlmProviders_ShouldDisableProxyAndRetainOriginGuards_OnFactoryPip [InlineData(nameof(OpenAiLlmProvider), "http://127.0.0.1/protected")] [InlineData(nameof(OpenAiLlmProvider), "http://10.0.0.1/protected")] [InlineData(nameof(OpenAiLlmProvider), "http://169.254.169.254/protected")] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName, "http://127.0.0.1/protected")] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName, "http://10.0.0.1/protected")] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName, "http://169.254.169.254/protected")] [InlineData(nameof(GeminiLlmProvider), "http://127.0.0.1/protected")] [InlineData(nameof(GeminiLlmProvider), "http://10.0.0.1/protected")] [InlineData(nameof(GeminiLlmProvider), "http://169.254.169.254/protected")] @@ -139,24 +162,35 @@ public async Task AddLlmProviders_ShouldRejectBlockedOriginWithoutConsultingHost string clientName, string blockedOrigin) { - using var serviceProvider = BuildServiceProvider("Production"); + using var serviceProvider = BuildServiceProvider( + "Production", + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName + ? "OpenAiCompatible" + : "Mock"); var pipeline = serviceProvider .GetRequiredService() .CreateHandler(clientName); await ProxySafeHttpHandlerTestHarness.AssertBlockedOriginIgnoresProxyAsync( pipeline, - blockedOrigin); + blockedOrigin, + expectStructuredEgressViolation: + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName); } [Theory] [InlineData(nameof(OpenAiLlmProvider))] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName)] [InlineData(nameof(GeminiLlmProvider))] [InlineData(nameof(OllamaLlmProvider))] public async Task AddLlmProviders_ShouldReachAllowedDirectOriginWithoutConsultingHostileProxy( string clientName) { - using var serviceProvider = BuildServiceProvider("Development"); + using var serviceProvider = BuildServiceProvider( + "Development", + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName + ? "OpenAiCompatible" + : "Mock"); var pipeline = serviceProvider .GetRequiredService() .CreateHandler(clientName); @@ -165,12 +199,13 @@ public async Task AddLlmProviders_ShouldReachAllowedDirectOriginWithoutConsultin } [Theory] - [InlineData("OpenAi", typeof(OpenAiLlmProvider))] - [InlineData("Gemini", typeof(GeminiLlmProvider))] - [InlineData("Ollama", typeof(OllamaLlmProvider))] + [InlineData("OpenAi", nameof(OpenAiLlmProvider))] + [InlineData("OpenAiCompatible", "OpenAiCompatibleLlmProvider")] + [InlineData("Gemini", nameof(GeminiLlmProvider))] + [InlineData("Ollama", nameof(OllamaLlmProvider))] public async Task AddLlmProviders_ShouldApplyResolvedLocalhostPolicyToConcreteProvider( string providerName, - Type expectedProviderType) + string expectedProviderType) { using var serviceProvider = BuildServiceProvider("Development", providerName); using var scope = serviceProvider.CreateScope(); @@ -183,7 +218,7 @@ public async Task AddLlmProviders_ShouldApplyResolvedLocalhostPolicyToConcretePr runtimePolicy.AllowOllamaLocalhost.Should().BeTrue(); runtimePolicy.ProtectOutboundTelemetry.Should().BeTrue( "registered provider clients must mask destinations before HttpClient diagnostics run"); - provider.Should().BeOfType(expectedProviderType); + provider.GetType().Name.Should().Be(expectedProviderType); health.IsAvailable.Should().BeTrue( "the selected concrete provider must reuse the same localhost policy as selection and connect-time validation"); } @@ -191,6 +226,8 @@ public async Task AddLlmProviders_ShouldApplyResolvedLocalhostPolicyToConcretePr [Theory] [InlineData("OpenAi", false, "/v1/chat/completions")] [InlineData("OpenAi", true, "/v1/chat/completions")] + [InlineData("OpenAiCompatible", false, "/openai/v1/chat/completions")] + [InlineData("OpenAiCompatible", true, "/openai/v1/chat/completions")] [InlineData("Gemini", false, "/v1beta/models/test-gemini-model:generateContent")] [InlineData("Gemini", true, "/v1beta/models/test-gemini-model:generateContent")] [InlineData("Ollama", false, "/api/chat")] @@ -206,6 +243,7 @@ public async Task AddLlmProviders_ShouldDispatchProbeAndCompletionThroughRegiste var providerBaseUrl = providerName switch { "OpenAi" => $"{origin}/v1", + "OpenAiCompatible" => $"{origin}/openai/v1", "Gemini" => $"{origin}/v1beta", _ => origin }; @@ -237,6 +275,40 @@ [new ChatCompletionMessage("User", "loopback dispatch")], AssertProviderRequestBody(requestBody, providerName, probe); } + [Fact] + public async Task AddLlmProviders_CompatibleStreamUsesRegisteredLoopbackPipelineIncrementally() + { + const string sse = + "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"},\"finish_reason\":null}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":7}}\n\n" + + "data: [DONE]\n\n"; + await using var server = new SingleRequestLoopbackServer( + responseBody: sse, + responseContentType: "text/event-stream"); + using var serviceProvider = BuildServiceProvider( + "Development", + "OpenAiCompatible", + providerBaseUrl: $"http://localhost:{server.Port}/openai/v1"); + using var scope = serviceProvider.CreateScope(); + var provider = scope.ServiceProvider.GetRequiredService(); + + var events = new List(); + await foreach (var item in provider.StreamAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "stream loopback")], + SystemPrompt: string.Empty))) + { + events.Add(item); + } + + events.Select(item => item.Token).Should().Equal("Hel", "lo", string.Empty); + events[^1].TokensUsed.Should().Be(7); + (await server.ReceivedRequest).Should().StartWith( + "POST /openai/v1/chat/completions HTTP/1.1"); + using var payload = JsonDocument.Parse(await server.ReceivedBody); + payload.RootElement.GetProperty("stream").GetBoolean().Should().BeTrue(); + } + [Fact] public async Task AddLlmProviders_ShouldInjectProductionPolicyIntoConcreteOllamaProvider() { @@ -269,25 +341,73 @@ [new ChatCompletionMessage("User", "must remain local")], [Theory] [InlineData(nameof(OpenAiLlmProvider))] + [InlineData(LlmProviderRegistration.OpenAiCompatibleHttpClientName)] [InlineData(nameof(GeminiLlmProvider))] [InlineData(nameof(OllamaLlmProvider))] public async Task AddLlmProviders_ShouldSuppressProtectedRequestLogging(string clientName) { var loggerProvider = new RecordingHttpLoggerProvider(); - using var serviceProvider = BuildServiceProvider("Production", loggerProvider: loggerProvider); + using var serviceProvider = BuildServiceProvider( + "Production", + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName + ? "OpenAiCompatible" + : "Mock", + loggerProvider: loggerProvider); var pipeline = serviceProvider .GetRequiredService() .CreateHandler(clientName); await ProxySafeHttpHandlerTestHarness.AssertBlockedOriginIgnoresProxyAsync( pipeline, - "http://127.0.0.1/protected"); + "http://127.0.0.1/protected", + expectStructuredEgressViolation: + clientName == LlmProviderRegistration.OpenAiCompatibleHttpClientName); loggerProvider.Messages.Should().NotContain( message => message.Contains(ProxySafeHttpHandlerTestHarness.SensitiveMarker, StringComparison.Ordinal), "protected query/header/body markers must not reach default IHttpClientFactory logs"); } + [Fact] + public async Task CompatibleRegisteredPolicy_Http501StillReachesBufferedFallbackAtThresholdOne() + { + var services = BuildCompatibleServices(failureThreshold: 1); + using var serviceProvider = services.BuildServiceProvider(); + var pipeline = serviceProvider.GetRequiredService() + .CreateHandler(LlmProviderRegistration.OpenAiCompatibleHttpClientName); + var dispatchHandler = EnumeratePipeline(pipeline) + .OfType() + .Single(handler => handler.GetType().Name == "LlmDispatchTrackingHandler"); + var transport = new SequentialResponseHandler( + new HttpResponseMessage(HttpStatusCode.NotImplemented), + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """{"choices":[{"message":{"content":"fallback reply"},"finish_reason":"stop"}],"usage":{"total_tokens":9}}""", + System.Text.Encoding.UTF8, + "application/json") + }); + dispatchHandler.InnerHandler = transport; + using var scope = serviceProvider.CreateScope(); + var compatibleProvider = scope.ServiceProvider.GetRequiredService(); + + var events = new List(); + await foreach (var item in compatibleProvider.StreamAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "fallback")], + SystemPrompt: string.Empty))) + { + events.Add(item); + } + + events.Should().ContainSingle(); + events[0].Token.Should().Be("fallback reply"); + events[0].IsDegraded.Should().BeTrue(); + events[0].Error.Should().BeNull(); + transport.InvocationCount.Should().Be(2); + serviceProvider.GetRequiredService() + .Get("OpenAICompatible")?.State.Should().NotBe(CircuitState.Open); + } + [Theory] [InlineData("Development", true, true, true)] [InlineData("Development", true, false, false)] @@ -349,6 +469,13 @@ private static ServiceProvider BuildServiceProvider( ? providerBaseUrl : "http://localhost:12345", ["Llm:OpenAi:Model"] = "test-openai-model", + ["Llm:OpenAiCompatible:ApiKey"] = "test-compatible-key", + ["Llm:OpenAiCompatible:BaseUrl"] = providerName == "OpenAiCompatible" && providerBaseUrl is not null + ? providerBaseUrl + : environmentName == "Production" + ? "https://api.groq.com/openai/v1" + : "http://localhost:12345/openai/v1", + ["Llm:OpenAiCompatible:Model"] = "test-compatible-model", ["Llm:Gemini:ApiKey"] = "test-gemini-key", ["Llm:Gemini:BaseUrl"] = providerName == "Gemini" && providerBaseUrl is not null ? providerBaseUrl @@ -371,6 +498,10 @@ private static ServiceProvider BuildServiceProvider( """ {"choices":[{"message":{"content":"OK"},"finish_reason":"stop"}],"usage":{"total_tokens":1}} """, + "OpenAiCompatible" => + """ + {"choices":[{"message":{"content":"OK"},"finish_reason":"stop"}],"usage":{"total_tokens":1}} + """, "Gemini" => """ {"candidates":[{"content":{"parts":[{"text":"OK"}]},"finishReason":"STOP"}],"usageMetadata":{"totalTokenCount":1}} @@ -401,6 +532,13 @@ private static void AssertProviderRequestBody(string requestBody, string provide root.GetProperty("temperature").GetDouble().Should().BeApproximately(expectedTemperature, 0.000001); root.GetProperty("messages")[0].GetProperty("content").GetString().Should().Be(expectedContent); break; + case "OpenAiCompatible": + root.GetProperty("model").GetString().Should().Be("test-compatible-model"); + root.GetProperty("stream").GetBoolean().Should().BeFalse(); + root.GetProperty("max_tokens").GetInt32().Should().Be(expectedMaxTokens); + root.GetProperty("temperature").GetDouble().Should().BeApproximately(expectedTemperature, 0.000001); + root.GetProperty("messages")[0].GetProperty("content").GetString().Should().Be(expectedContent); + break; case "Gemini": var generationConfig = root.GetProperty("generationConfig"); generationConfig.GetProperty("maxOutputTokens").GetInt32().Should().Be(expectedMaxTokens); @@ -435,7 +573,7 @@ private sealed class TestWebHostEnvironment(string environmentName) : IWebHostEn public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider(); } - private static ServiceCollection BuildCompatibleServices() + private static ServiceCollection BuildCompatibleServices(int failureThreshold = 5) { var services = new ServiceCollection(); services.AddLogging(); @@ -447,7 +585,9 @@ private static ServiceCollection BuildCompatibleServices() ["Llm:Provider"] = "OpenAICompatible", ["Llm:OpenAiCompatible:ApiKey"] = "test-compatible-key", ["Llm:OpenAiCompatible:BaseUrl"] = "https://api.groq.com/openai/v1", - ["Llm:OpenAiCompatible:Model"] = "llama-3.1-8b-instant" + ["Llm:OpenAiCompatible:Model"] = "llama-3.1-8b-instant", + ["CircuitBreaker:FailureThreshold"] = failureThreshold.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["CircuitBreaker:BreakDurationSeconds"] = "60" }) .Build(); services.AddLlmProviders(configuration); @@ -469,6 +609,21 @@ protected override Task SendAsync( } } + private sealed class SequentialResponseHandler(params HttpResponseMessage[] responses) : HttpMessageHandler + { + private readonly Queue _responses = new(responses); + + public int InvocationCount { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + InvocationCount++; + return Task.FromResult(_responses.Dequeue()); + } + } + private static IEnumerable EnumeratePipeline(HttpMessageHandler root) { for (var current = root; current is not null; current = (current as DelegatingHandler)?.InnerHandler) diff --git a/backend/tests/Taskdeck.Api.Tests/OutboundWebhookConnectCallbackTests.cs b/backend/tests/Taskdeck.Api.Tests/OutboundWebhookConnectCallbackTests.cs index 9966dd644..e247253d8 100644 --- a/backend/tests/Taskdeck.Api.Tests/OutboundWebhookConnectCallbackTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/OutboundWebhookConnectCallbackTests.cs @@ -299,7 +299,8 @@ internal static void AssertProxySafeOriginHandler(HttpMessageHandler pipeline) internal static async Task AssertBlockedOriginIgnoresProxyAsync( HttpMessageHandler pipeline, - string blockedOrigin) + string blockedOrigin, + bool expectStructuredEgressViolation = false) { AssertProxySafeOriginHandler(pipeline); var primaryHandler = GetPrimaryHandler(pipeline); @@ -315,12 +316,21 @@ internal static async Task AssertBlockedOriginIgnoresProxyAsync( request.Content = new StringContent(SensitiveMarker); ProtectedOutboundTelemetryHandler.PrepareForSend(request); - var exception = await Assert.ThrowsAsync( - () => invoker.SendAsync(request, cancellationSource.Token)); - - exception.Message.Should().Contain(new Uri(blockedOrigin).Host); - exception.Message.Should().Contain("is not allowed"); - exception.Message.Should().NotContain(SensitiveMarker); + if (expectStructuredEgressViolation) + { + var exception = await Assert.ThrowsAsync( + () => invoker.SendAsync(request, cancellationSource.Token)); + exception.Message.Should().Contain(new Uri(blockedOrigin).Host); + exception.Message.Should().NotContain(SensitiveMarker); + } + else + { + var exception = await Assert.ThrowsAsync( + () => invoker.SendAsync(request, cancellationSource.Token)); + exception.Message.Should().Contain(new Uri(blockedOrigin).Host); + exception.Message.Should().Contain("is not allowed"); + exception.Message.Should().NotContain(SensitiveMarker); + } request.RequestUri!.AbsoluteUri.Should().NotContain(SensitiveMarker, "blocked requests must be remasked after connect-time rejection"); hostileProxy.InvocationCount.Should().Be(0, @@ -398,13 +408,15 @@ internal sealed class SingleRequestLoopbackServer : IAsyncDisposable internal SingleRequestLoopbackServer( HttpStatusCode responseStatus = HttpStatusCode.OK, - string responseBody = "") + string responseBody = "", + string responseContentType = "application/json") { _listener.Start(); Port = ((IPEndPoint)_listener.LocalEndpoint).Port; _capturedRequest = ReceiveSingleRequestAsync( responseStatus, responseBody, + responseContentType, _cancellationSource.Token); ReceivedRequest = SelectRawRequestAsync(_capturedRequest); ReceivedBody = SelectBodyAsync(_capturedRequest); @@ -450,6 +462,7 @@ public async ValueTask DisposeAsync() private async Task ReceiveSingleRequestAsync( HttpStatusCode responseStatus, string responseBody, + string responseContentType, CancellationToken cancellationToken) { using var client = await _listener.AcceptTcpClientAsync(cancellationToken); @@ -494,7 +507,12 @@ private async Task ReceiveSingleRequestAsync( isChunked, out var capturedRequest)) { - await WriteResponseAsync(stream, responseStatus, responseBody, cancellationToken); + await WriteResponseAsync( + stream, + responseStatus, + responseBody, + responseContentType, + cancellationToken); return capturedRequest; } } @@ -691,6 +709,7 @@ private static async Task WriteResponseAsync( NetworkStream stream, HttpStatusCode status, string body, + string responseContentType, CancellationToken cancellationToken) { var bodyBytes = Encoding.UTF8.GetBytes(body); @@ -703,7 +722,7 @@ private static async Task WriteResponseAsync( var responseHeaders = Encoding.ASCII.GetBytes( $"HTTP/1.1 {(int)status} {reason}\r\n" + $"Content-Length: {bodyBytes.Length}\r\n" + - "Content-Type: application/json\r\n" + + $"Content-Type: {responseContentType}\r\n" + "Connection: close\r\n\r\n"); await stream.WriteAsync(responseHeaders, cancellationToken); diff --git a/backend/tests/Taskdeck.Api.Tests/ProtectedOutboundTelemetryHandlerTests.cs b/backend/tests/Taskdeck.Api.Tests/ProtectedOutboundTelemetryHandlerTests.cs index 559d0672d..3ac32f277 100644 --- a/backend/tests/Taskdeck.Api.Tests/ProtectedOutboundTelemetryHandlerTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ProtectedOutboundTelemetryHandlerTests.cs @@ -229,6 +229,9 @@ public async Task EnabledSentry_ShouldExcludeOnlyProtectedClients() ["Llm:OpenAi:ApiKey"] = "test-openai-key", ["Llm:OpenAi:BaseUrl"] = "http://localhost:12345", ["Llm:OpenAi:Model"] = "test-openai-model", + ["Llm:OpenAiCompatible:ApiKey"] = "test-compatible-key", + ["Llm:OpenAiCompatible:BaseUrl"] = "http://localhost:12345/openai/v1", + ["Llm:OpenAiCompatible:Model"] = "test-compatible-model", ["Llm:Ollama:AllowLocalhostEndpoints"] = "true", ["OutboundWebhooks:Security:AllowLocalhostEndpoints"] = "true" }); @@ -260,6 +263,7 @@ public async Task EnabledSentry_ShouldExcludeOnlyProtectedClients() var protectedClientNames = new[] { nameof(OpenAiLlmProvider), + LlmProviderRegistration.OpenAiCompatibleHttpClientName, nameof(GeminiLlmProvider), nameof(OllamaLlmProvider), "OutboundWebhookDelivery" diff --git a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs index d0b6c42e1..5dcdb6f10 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/ChatServiceTests.cs @@ -1947,6 +1947,92 @@ public async Task StreamResponseAsync_ClientDisconnectsMidStream_ShouldCommitEst quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); } + [Fact] + public async Task SendMessageAsync_CancelledAfterDispatch_CommitsReservationEstimate() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Buffered dispatch cancellation"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, It.IsAny())) + .ReturnsAsync(session); + _llmProviderMock + .Setup(p => p.CompleteAsync(It.IsAny(), It.IsAny())) + .Returns((ChatCompletionRequest request, CancellationToken _) => + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + request.DispatchContext.MarkDispatched(); + return Task.FromException(new OperationCanceledException()); + }); + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, It.IsAny())) + .ReturnsAsync(new DTOs.QuotaReservationDto(true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var service = CreateServiceWithQuota(quotaMock.Object); + + var act = () => service.SendMessageAsync( + session.Id, + userId, + new SendChatMessageDto("hello"), + default); + + await act.Should().ThrowAsync(); + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 2000, + 0, + CancellationToken.None), Times.Once); + quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task SendMessageAsync_ObservedPreDispatch_ReleasesReservation() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Buffered pre-dispatch rejection"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, It.IsAny())) + .ReturnsAsync(session); + _llmProviderMock + .Setup(p => p.CompleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((ChatCompletionRequest request, CancellationToken _) => + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + return new LlmCompletionResult( + "configuration rejected", + 0, + false, + Provider: "OpenAICompatible", + Model: "vendor/model", + IsDegraded: true) + { + HasAuthoritativeTokenUsage = false, + ShouldSettleQuotaReservation = true + }; + }); + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, It.IsAny())) + .ReturnsAsync(new DTOs.QuotaReservationDto(true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var service = CreateServiceWithQuota(quotaMock.Object); + + var result = await service.SendMessageAsync( + session.Id, + userId, + new SendChatMessageDto("hello"), + default); + + result.IsSuccess.Should().BeTrue(); + quotaMock.Verify(q => q.ReleaseReservationAsync(reservationId, CancellationToken.None), Times.Once); + quotaMock.Verify(q => q.CommitReservationAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny()), Times.Never); + } + [Fact] public async Task StreamResponseAsync_ProviderYieldsOnlyErrorEvent_ShouldReleaseNotCommit() { @@ -1987,6 +2073,77 @@ public async Task StreamResponseAsync_ProviderYieldsOnlyErrorEvent_ShouldRelease Times.Never); } + [Fact] + public async Task StreamResponseAsync_PostDispatchErrorOnly_CommitsEstimateAndPersistsSanitizedPlaceholder() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Dispatched error-only stream"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, It.IsAny())) + .ReturnsAsync(session); + _llmProviderMock + .Setup(p => p.StreamAsync(It.IsAny(), It.IsAny())) + .Returns((ChatCompletionRequest request, CancellationToken _) => DispatchedErrorOnlyStream(request)); + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, It.IsAny())) + .ReturnsAsync(new DTOs.QuotaReservationDto(true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var service = CreateServiceWithQuota(quotaMock.Object); + + await foreach (var _ in service.StreamResponseAsync(session.Id, userId, default)) { } + + var persisted = session.Messages.Single(message => message.Role == ChatMessageRole.Assistant); + persisted.Content.Should().Be("The provider could not complete the response."); + persisted.DegradedReason.Should().Be("The upstream provider could not complete the response."); + persisted.Content.Should().NotContain("secret-upstream-detail"); + persisted.DegradedReason.Should().NotContain("secret-upstream-detail"); + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 2000, + 0, + CancellationToken.None), Times.Once); + } + + [Fact] + public async Task StreamResponseAsync_CancelledAfterDispatchBeforeFirstEvent_CommitsEstimate() + { + var userId = Guid.NewGuid(); + var session = new ChatSession(userId, "Cancelled dispatched stream"); + _chatSessionRepoMock + .Setup(r => r.GetByIdWithMessagesAsync(session.Id, It.IsAny())) + .ReturnsAsync(session); + _llmProviderMock + .Setup(p => p.StreamAsync(It.IsAny(), It.IsAny())) + .Returns((ChatCompletionRequest request, CancellationToken token) => + DispatchedBlockingStream(request, token)); + var reservationId = Guid.NewGuid(); + var quotaMock = new Mock(); + quotaMock.Setup(q => q.ReserveAsync(userId, Domain.Enums.LlmSurface.Chat, It.IsAny())) + .ReturnsAsync(new DTOs.QuotaReservationDto(true, null, reservationId, 10000, 100, EstimatedTokens: 2000)); + var service = CreateServiceWithQuota(quotaMock.Object); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + + var act = async () => + { + await foreach (var _ in service.StreamResponseAsync(session.Id, userId, cancellation.Token)) { } + }; + + await act.Should().ThrowAsync(); + quotaMock.Verify(q => q.CommitReservationAsync( + reservationId, + userId, + Domain.Enums.LlmSurface.Chat, + "OpenAICompatible", + "vendor/model", + 2000, + 0, + CancellationToken.None), Times.Once); + } + [Fact] public async Task SendMessageAsync_UnknownCompatibleUsage_CommitsReservationEstimateForLargePrompt() { @@ -2043,10 +2200,11 @@ public async Task SendMessageAsync_UnknownCompatibleUsage_CommitsReservationEsti } [Theory] - [InlineData(false, "Response was stopped by the upstream content filter.")] - [InlineData(true, "OpenAI-compatible SSE transport failed before completion.")] + [InlineData(false, "Response was stopped by the upstream content filter.", "Response was stopped by the upstream content filter.")] + [InlineData(true, "secret-upstream-detail", "The upstream provider could not complete the response.")] public async Task StreamResponseAsync_TerminalDegradedOrError_PersistsPartialHistoryAsDegraded( bool terminalError, + string terminalReason, string expectedReason) { var userId = Guid.NewGuid(); @@ -2064,7 +2222,7 @@ public async Task StreamResponseAsync_TerminalDegradedOrError_PersistsPartialHis }); _llmProviderMock .Setup(p => p.StreamAsync(It.IsAny(), default)) - .Returns(TerminalStateStream(terminalError, expectedReason)); + .Returns(TerminalStateStream(terminalError, terminalReason)); await foreach (var _ in _service.StreamResponseAsync(session.Id, userId, default)) { } @@ -2072,6 +2230,8 @@ public async Task StreamResponseAsync_TerminalDegradedOrError_PersistsPartialHis persisted!.Content.Should().Be("partial response"); persisted.MessageType.Should().Be("degraded"); persisted.DegradedReason.Should().Be(expectedReason); + if (terminalError) + persisted.DegradedReason.Should().NotContain(terminalReason); } [Fact] @@ -2175,6 +2335,40 @@ private static async IAsyncEnumerable ErrorOnlyStream() await Task.CompletedTask; } + private static async IAsyncEnumerable DispatchedErrorOnlyStream( + ChatCompletionRequest request) + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + request.DispatchContext.MarkDispatched(); + yield return new LlmTokenEvent( + string.Empty, + true, + Error: "secret-upstream-detail", + Provider: "OpenAICompatible", + Model: "vendor/model"); + await Task.CompletedTask; + } + + private static async IAsyncEnumerable DispatchedBlockingStream( + ChatCompletionRequest request, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + request.DispatchContext.MarkDispatched(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + yield break; + } + + private ChatService CreateServiceWithQuota(ILlmQuotaService quotaService) => new( + _unitOfWorkMock.Object, + _llmProviderMock.Object, + _plannerMock.Object, + _proposalServiceMock.Object, + _policyEngineMock.Object, + _notificationServiceMock.Object, + _authorizationServiceMock.Object, + quotaService: quotaService); + private static async IAsyncEnumerable TerminalStateStream( bool terminalError, string reason) diff --git a/backend/tests/Taskdeck.Application.Tests/Services/CircuitBreakerStateTrackerTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/CircuitBreakerStateTrackerTests.cs new file mode 100644 index 000000000..e7d391f07 --- /dev/null +++ b/backend/tests/Taskdeck.Application.Tests/Services/CircuitBreakerStateTrackerTests.cs @@ -0,0 +1,102 @@ +using FluentAssertions; +using Taskdeck.Application.Services; +using Xunit; + +namespace Taskdeck.Application.Tests.Services; + +public class CircuitBreakerStateTrackerTests +{ + private static readonly CircuitBreakerSettings Settings = new() + { + FailureThreshold = 1, + BreakDurationSeconds = 60 + }; + + [Fact] + public void AbandonedHalfOpenProbe_RemainsOpenForFullConfiguredBreak() + { + var clock = new ManualTimeProvider(); + var tracker = new CircuitBreakerStateTracker(clock); + tracker.TryEnterProviderRequest("provider", Settings, out var initial, out _).Should().BeTrue(); + tracker.RecordProviderFailure("provider", Settings, "failed", initial); + clock.Advance(TimeSpan.FromSeconds(60)); + tracker.TryEnterProviderRequest("provider", Settings, out var probe, out _).Should().BeTrue(); + + tracker.AbandonProviderRequest("provider", Settings, probe); + + tracker.TryEnterProviderRequest("provider", Settings, out _, out _).Should().BeFalse(); + clock.Advance(TimeSpan.FromSeconds(59)); + tracker.TryEnterProviderRequest("provider", Settings, out _, out _).Should().BeFalse(); + clock.Advance(TimeSpan.FromSeconds(1)); + tracker.TryEnterProviderRequest("provider", Settings, out var nextProbe, out _).Should().BeTrue(); + nextProbe.IsHalfOpenProbe.Should().BeTrue(); + } + + [Fact] + public void StalePreOpenSuccess_DoesNotCloseNewOpenGeneration() + { + var tracker = new CircuitBreakerStateTracker(new ManualTimeProvider()); + tracker.TryEnterProviderRequest("provider", Settings, out var failing, out _).Should().BeTrue(); + tracker.TryEnterProviderRequest("provider", Settings, out var staleSuccess, out _).Should().BeTrue(); + + tracker.RecordProviderFailure("provider", Settings, "failed", failing); + tracker.RecordProviderSuccess("provider", staleSuccess); + + tracker.Get("provider")!.State.Should().Be(CircuitState.Open); + tracker.TryEnterProviderRequest("provider", Settings, out _, out _).Should().BeFalse(); + } + + [Fact] + public void StalePreOpenFailure_DoesNotReopenAfterSuccessfulHalfOpenProbe() + { + var clock = new ManualTimeProvider(); + var tracker = new CircuitBreakerStateTracker(clock); + tracker.TryEnterProviderRequest("provider", Settings, out var openingFailure, out _).Should().BeTrue(); + tracker.TryEnterProviderRequest("provider", Settings, out var staleFailure, out _).Should().BeTrue(); + tracker.RecordProviderFailure("provider", Settings, "open", openingFailure); + clock.Advance(TimeSpan.FromSeconds(60)); + tracker.TryEnterProviderRequest("provider", Settings, out var probe, out _).Should().BeTrue(); + + tracker.RecordProviderSuccess("provider", probe); + tracker.RecordProviderFailure("provider", Settings, "stale", staleFailure); + + tracker.Get("provider")!.State.Should().Be(CircuitState.Closed); + tracker.TryEnterProviderRequest("provider", Settings, out var admitted, out _).Should().BeTrue(); + admitted.IsHalfOpenProbe.Should().BeFalse(); + } + + [Fact] + public void CompanionSuccess_DoesNotMaskPollyOpen() + { + var tracker = new CircuitBreakerStateTracker(new ManualTimeProvider()); + tracker.RecordState("provider", CircuitState.Open, "polly open"); + tracker.TryEnterProviderRequest("provider", Settings, out var companion, out _).Should().BeTrue(); + + tracker.RecordProviderSuccess("provider", companion); + + tracker.Get("provider")!.State.Should().Be(CircuitState.Open); + tracker.Get("provider")!.LastFailureReason.Should().Be("polly open"); + } + + [Fact] + public void PollyReset_DoesNotMaskCompanionOpen() + { + var tracker = new CircuitBreakerStateTracker(new ManualTimeProvider()); + tracker.TryEnterProviderRequest("provider", Settings, out var companion, out _).Should().BeTrue(); + tracker.RecordProviderFailure("provider", Settings, "companion open", companion); + + tracker.RecordState("provider", CircuitState.Closed); + + tracker.Get("provider")!.State.Should().Be(CircuitState.Open); + tracker.GetAll()["provider"].LastFailureReason.Should().Be("companion open"); + } + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow = new(2026, 7, 27, 12, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan duration) => _utcNow = _utcNow.Add(duration); + } +} diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs index c939139a6..eccb07876 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmCaptureTriageExtractorTests.cs @@ -298,6 +298,71 @@ await FluentActions Times.Never); } + [Fact] + public async Task ExtractAsync_CancelledAfterDispatch_CommitsReservationEstimate() + { + _quotaMock + .Setup(q => q.ReserveAsync(_userId, LlmSurface.CaptureTriage, It.IsAny())) + .ReturnsAsync(new QuotaReservationDto( + true, null, _reservationId, 100_000, 60, EstimatedTokens: 2000)); + _providerMock + .Setup(p => p.CompleteAsync(It.IsAny(), It.IsAny())) + .Returns((ChatCompletionRequest request, CancellationToken _) => + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + request.DispatchContext.MarkDispatched(); + return Task.FromException(new OperationCanceledException()); + }); + var extractor = BuildExtractor(); + + await FluentActions + .Awaiting(() => extractor.ExtractAsync(_userId, _boardId, TranscriptPayload())) + .Should().ThrowAsync(); + + _quotaMock.Verify(q => q.CommitReservationAsync( + _reservationId, + _userId, + LlmSurface.CaptureTriage, + "OpenAICompatible", + "vendor/model", + 2000, + 0, + CancellationToken.None), Times.Once); + _quotaMock.Verify(q => q.ReleaseReservationAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task ExtractAsync_ObservedPreDispatchResult_ReleasesReservation() + { + _providerMock + .Setup(p => p.CompleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((ChatCompletionRequest request, CancellationToken _) => + { + request.DispatchContext.Observe("OpenAICompatible", "vendor/model"); + return new LlmCompletionResult( + "configuration rejected", + 0, + IsActionable: false, + Provider: "OpenAICompatible", + Model: "vendor/model", + IsDegraded: true) + { + HasAuthoritativeTokenUsage = false, + ShouldSettleQuotaReservation = true + }; + }); + var extractor = BuildExtractor(); + + var result = await extractor.ExtractAsync(_userId, _boardId, TranscriptPayload()); + + result.Outcome.Should().Be(LlmCaptureTriageOutcome.ProviderDegraded); + _quotaMock.Verify(q => q.ReleaseReservationAsync(_reservationId, CancellationToken.None), Times.Once); + _quotaMock.Verify(q => q.CommitReservationAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny()), Times.Never); + } + [Fact] public async Task ExtractAsync_ShouldReturnInvalidOutput_WhenContentIsUnparseable() { diff --git a/backend/tests/Taskdeck.Application.Tests/Services/LlmDispatchTrackingHandlerTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/LlmDispatchTrackingHandlerTests.cs new file mode 100644 index 000000000..19ba5cc37 --- /dev/null +++ b/backend/tests/Taskdeck.Application.Tests/Services/LlmDispatchTrackingHandlerTests.cs @@ -0,0 +1,62 @@ +using FluentAssertions; +using Taskdeck.Application.Services; +using Xunit; + +namespace Taskdeck.Application.Tests.Services; + +public class LlmDispatchTrackingHandlerTests +{ + [Fact] + public async Task SendAsync_DoesNotMarkAlreadyCancelledRequest() + { + var context = new LlmDispatchContext(); + context.Observe("provider", "model"); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/"); + LlmDispatchTrackingHandler.Attach(request, context); + using var handler = new LlmDispatchTrackingHandler + { + InnerHandler = new SnapshotHandler(context) + }; + using var invoker = new HttpMessageInvoker(handler); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var act = () => invoker.SendAsync(request, cancellation.Token); + + await act.Should().ThrowAsync(); + context.ReadSnapshot().Phase.Should().Be(LlmDispatchPhase.ObservedPreDispatch); + } + + [Fact] + public async Task SendAsync_MarksDispatchedBeforeAwaitingTransport() + { + var context = new LlmDispatchContext(); + context.Observe("provider", "model"); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/"); + LlmDispatchTrackingHandler.Attach(request, context); + var transport = new SnapshotHandler(context); + using var handler = new LlmDispatchTrackingHandler { InnerHandler = transport }; + using var invoker = new HttpMessageInvoker(handler); + + using var response = await invoker.SendAsync(request, CancellationToken.None); + + transport.PhaseAtDispatch.Should().Be(LlmDispatchPhase.Dispatched); + context.ReadSnapshot().Should().Be(new LlmDispatchSnapshot( + LlmDispatchPhase.Dispatched, + "provider", + "model")); + } + + private sealed class SnapshotHandler(LlmDispatchContext context) : HttpMessageHandler + { + public LlmDispatchPhase PhaseAtDispatch { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + PhaseAtDispatch = context.ReadSnapshot().Phase; + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NoContent)); + } + } +} diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs index 887ac7eb2..a7b5cbfda 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiCompatibleLlmProviderTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Taskdeck.Application.Services; using Taskdeck.Application.Tests.TestUtilities; +using Taskdeck.Domain.Agents; using Xunit; namespace Taskdeck.Application.Tests.Services; @@ -161,6 +162,35 @@ public async Task CompleteAsync_UsageAbsent_PreservesUnknownUsage() result.ShouldSettleQuotaReservation.Should().BeTrue(); } + [Fact] + public async Task CompleteAsync_NonEmptyResponseWithZeroUsage_TreatsUsageAsUnknown() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => + JsonResponse("""{"choices":[{"message":{"content":"short reply"},"finish_reason":"stop"}],"usage":{"total_tokens":0}}""")), BuildSettings()); + + var result = await provider.CompleteAsync(Request()); + + result.Content.Should().Be("short reply"); + result.TokensUsed.Should().Be(0); + result.HasAuthoritativeTokenUsage.Should().BeFalse(); + result.ShouldSettleQuotaReservation.Should().BeTrue(); + } + + [Fact] + public async Task StreamAsync_ZeroUsageChunk_IsValidAndLeavesTerminalUsageUnknown() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: {\"choices\":[],\"usage\":{\"total_tokens\":0}}\n\n" + + "data: [DONE]\n\n")), BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Select(item => item.Token).Should().Equal("hello", string.Empty); + events[^1].Error.Should().BeNull(); + events[^1].TokensUsed.Should().BeNull(); + } + [Fact] public async Task StreamAsync_NonSseResponseWithoutUsage_LeavesUsageUnknown() { @@ -291,6 +321,47 @@ public async Task CompleteAsync_ContentFilterFinish_DoesNotCountAsCircuitFailure tracker.Get("OpenAICompatible")?.State.Should().NotBe(CircuitState.Open); } + [Theory] + [InlineData("""{"choices":[{"message":{"content":null},"finish_reason":"content_filter"}],"usage":{"total_tokens":4}}""", "content filter")] + [InlineData("""{"choices":[{"message":{"content":null,"refusal":"sensitive vendor refusal detail"},"finish_reason":"stop"}],"usage":{"total_tokens":4}}""", "refused")] + public async Task CompleteAsync_NullContentFilterOrRefusal_IsSanitizedSuccess( + string responseBody, + string expectedReason) + { + var dispatches = 0; + var tracker = new CircuitBreakerStateTracker(); + var circuitSettings = new CircuitBreakerSettings { FailureThreshold = 1, BreakDurationSeconds = 60 }; + var provider = CreateProvider(new StubHttpMessageHandler(_ => + { + dispatches++; + return JsonResponse(responseBody); + }), BuildSettings(), tracker, circuitSettings); + + var first = await provider.CompleteAsync(Request()); + var second = await provider.CompleteAsync(Request()); + + first.Content.Should().BeEmpty(); + first.IsDegraded.Should().BeTrue(); + first.DegradedReason.Should().Contain(expectedReason); + first.DegradedReason.Should().NotContain("sensitive vendor refusal detail"); + first.CountsAsProviderFailure.Should().BeFalse(); + second.IsDegraded.Should().BeTrue(); + dispatches.Should().Be(2, "a sanitized refusal/filter outcome must not open the companion circuit"); + tracker.Get("OpenAICompatible")?.State.Should().NotBe(CircuitState.Open); + } + + [Fact] + public async Task CompleteAsync_NullContentWithOrdinaryStop_RemainsProtocolFailure() + { + var provider = CreateProvider(new StubHttpMessageHandler(_ => + JsonResponse("""{"choices":[{"message":{"content":null},"finish_reason":"stop"}],"usage":{"total_tokens":4}}""")), BuildSettings()); + + var result = await provider.CompleteAsync(Request()); + + result.IsDegraded.Should().BeTrue(); + result.ProviderFailureKind.Should().Be(LlmProviderFailureKind.Protocol); + } + [Theory] [InlineData("{\"choices\":[1]}")] [InlineData("{\"choices\":[{\"delta\":\"bad\",\"finish_reason\":null}]}")] @@ -408,6 +479,91 @@ public async Task StreamAsync_AggregateResponseBudget_EmitsBoundedTerminalError( events[^1].Error.Should().Contain("safety limit"); } + [Theory] + [InlineData("utf16")] + [InlineData("utf32")] + public async Task StreamAsync_NonUtf8BomPayload_IsRejected(string encodingName) + { + const string sse = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n"; + var encoding = encodingName == "utf16" + ? Encoding.Unicode + : new UTF32Encoding(bigEndian: false, byteOrderMark: true, throwOnInvalidCharacters: true); + var bytes = encoding.GetPreamble().Concat(encoding.GetBytes(sse)).ToArray(); + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseBytesResponse(bytes)), + BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Should().ContainSingle(); + events[0].Error.Should().Contain("response body failed"); + } + + [Fact] + public async Task StreamAsync_EmojiAtExactRawUtf8LineLimit_IsAccepted() + { + var content = new string('x', 220) + "😀"; + var line = $"data: {{\"choices\":[{{\"delta\":{{\"content\":\"{content}\"}},\"finish_reason\":\"stop\"}}]}}"; + var lineBytes = Encoding.UTF8.GetByteCount(line); + lineBytes.Should().BeGreaterThanOrEqualTo(256); + var settings = BuildSettings(); + settings.OpenAiCompatible.MaxSseLineBytes = lineBytes; + settings.OpenAiCompatible.MaxSseEventBytes = Math.Max(512, lineBytes); + var provider = CreateProvider(new StubHttpMessageHandler(_ => SseResponse( + line + "\n\n" + + "data: [DONE]\n\n")), settings); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events[0].Token.Should().Be(content); + events[^1].Error.Should().BeNull(); + } + + [Fact] + public async Task StreamAsync_InitialUtf8Bom_IsAcceptedAndCounted() + { + var sse = "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n"; + var bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes(sse)).ToArray(); + var provider = CreateProvider( + new StubHttpMessageHandler(_ => SseBytesResponse(bytes)), + BuildSettings()); + + var events = await CollectAsync(provider.StreamAsync(Request())); + + events.Select(item => item.Token).Should().Equal("hello", string.Empty); + events[^1].Error.Should().BeNull(); + } + + [Fact] + public async Task CompleteAsync_EgressViolation_IsRethrownUnchanged() + { + var expected = CreateEgressViolationException(); + var provider = CreateProvider( + new StubHttpMessageHandler((_, _) => Task.FromException(expected)), + BuildSettings()); + + var act = () => provider.CompleteAsync(Request()); + + var thrown = await act.Should().ThrowAsync(); + thrown.Which.Should().BeSameAs(expected); + } + + [Fact] + public async Task StreamAsync_EgressViolation_IsRethrownUnchanged() + { + var expected = CreateEgressViolationException(); + var provider = CreateProvider( + new StubHttpMessageHandler((_, _) => Task.FromException(expected)), + BuildSettings()); + + var act = () => CollectAsync(provider.StreamAsync(Request())); + + var thrown = await act.Should().ThrowAsync(); + thrown.Which.Should().BeSameAs(expected); + } + [Fact] public async Task StreamAsync_OversizedNonSseBody_EmitsBoundedTerminalError() { @@ -539,7 +695,7 @@ public async Task CompleteAsync_SuccessResetsConsecutiveBodyFailures() } [Fact] - public async Task StreamAsync_DisposingHalfOpenProbe_ReleasesLeaseForNextProbe() + public async Task StreamAsync_DisposingHalfOpenProbe_ReopensForConfiguredCooldown() { var dispatches = 0; var tracker = new CircuitBreakerStateTracker(); @@ -563,6 +719,10 @@ public async Task StreamAsync_DisposingHalfOpenProbe_ReleasesLeaseForNextProbe() enumerator.Current.Token.Should().Be("probe"); await enumerator.DisposeAsync(); + var rejectedDuringCooldown = await CollectAsync(provider.StreamAsync(Request())); + rejectedDuringCooldown[^1].Error.Should().Contain("circuit is open"); + dispatches.Should().Be(2); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); var recovered = await CollectAsync(provider.StreamAsync(Request())); recovered[^1].Error.Should().BeNull(); @@ -571,7 +731,7 @@ public async Task StreamAsync_DisposingHalfOpenProbe_ReleasesLeaseForNextProbe() } [Fact] - public async Task StreamAsync_CancellingHalfOpenProbe_ReleasesLeaseForNextProbe() + public async Task StreamAsync_CancellingHalfOpenProbe_ReopensForConfiguredCooldown() { var dispatches = 0; var tracker = new CircuitBreakerStateTracker(); @@ -595,6 +755,10 @@ public async Task StreamAsync_CancellingHalfOpenProbe_ReleasesLeaseForNextProbe( CollectAsync(provider.StreamAsync(Request(), cancellation.Token)); await cancelled.Should().ThrowAsync(); + var rejectedDuringCooldown = await CollectAsync(provider.StreamAsync(Request())); + rejectedDuringCooldown[^1].Error.Should().Contain("circuit is open"); + dispatches.Should().Be(2); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); var recovered = await CollectAsync(provider.StreamAsync(Request())); recovered[^1].Error.Should().BeNull(); @@ -697,7 +861,9 @@ private static OpenAiCompatibleLlmProvider CreateProvider( NullLogger.Instance, tracker, circuitSettings, - allowLocalhostEndpoints); + new LlmProviderRuntimePolicy( + AllowGeneralProviderLocalhost: allowLocalhostEndpoints, + AllowOllamaLocalhost: allowLocalhostEndpoints)); private static ChatCompletionRequest Request(LlmRequestAttribution? attribution = null) => new([new ChatCompletionMessage("user", "hello")], Attribution: attribution); @@ -725,6 +891,16 @@ private static HttpResponseMessage SseStreamResponse(Stream stream) return response; } + private static HttpResponseMessage SseBytesResponse(byte[] bytes) => + SseStreamResponse(new MemoryStream(bytes, writable: false)); + + private static EgressViolationException CreateEgressViolationException() => new( + new EgressViolation( + "blocked.example", + "https://blocked.example/", + EgressViolationType.UnknownHost, + "blocked")); + private static HttpResponseMessage JsonStreamResponse(Stream stream) { var response = new HttpResponseMessage(HttpStatusCode.OK) From 1f529604aabe80a422f38c930658f151e860d915 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 15:41:19 +0100 Subject: [PATCH 7/7] Document compatible provider runtime Signed-off-by: Chris0Jeky --- docs/IMPLEMENTATION_MASTERPLAN.md | 8 ++++- docs/STATUS.md | 15 ++++++--- docs/TESTING_GUIDE.md | 39 ++++++++++++++++++++++- docs/platform/CONFIGURATION_REFERENCE.md | 25 ++++++++------- docs/platform/LLM_PROVIDER_SETUP_GUIDE.md | 29 ++++++++++------- 5 files changed, 85 insertions(+), 31 deletions(-) diff --git a/docs/IMPLEMENTATION_MASTERPLAN.md b/docs/IMPLEMENTATION_MASTERPLAN.md index ab7bf1cc7..c3bec73d7 100644 --- a/docs/IMPLEMENTATION_MASTERPLAN.md +++ b/docs/IMPLEMENTATION_MASTERPLAN.md @@ -10,9 +10,15 @@ Companion Active Docs: - `docs/MANUAL_TEST_CHECKLIST.md` - `docs/GOLDEN_PRINCIPLES.md` +## Active delivery checkpoint (2026-07-27, OpenAI-compatible provider replacement) + +- **REVIVAL-10 / `#1306` replacement branch:** add a distinct, config-gated `OpenAICompatible` adapter for public OpenAI Chat Completions endpoints while preserving `Mock` as the safe default. The provider supports buffered chat, structured instruction extraction, bounded board context, real upstream SSE, provider-specific non-secret headers, and an explicit buffered degradation path when a gateway rejects streaming or `response_format`. +- **Security and accounting boundary:** validate the configured public origin and connect-time DNS, allow plain HTTP only for exact `localhost` under the explicit `Development`/`Test`/`Testing` live-provider gate, disable proxies and redirects, and run the registered client through protected telemetry → fixed-origin egress → dispatch tracking → direct sockets. Quota settlement distinguishes proven pre-dispatch failures from dispatched requests; compatible Polly state and companion provider circuit state remain independent and generation-safe; response and SSE limits are enforced over strict UTF-8 bytes; provider error detail is not persisted. +- **Evidence and remaining gate:** focused Application 136/136 and API 80/80 tests are green; the full serialized backend passed 7,660 tests with five intentional skips and no failures (Domain 1,636; Application 3,678; API 2,189 + 4 skips; CLI 100; Architecture 22 + 1 skip; Integration 35); docs, golden-principles, GitHub-operations, and whitespace governance passed. Independent exact-head review, hosted CI, and signed publication are pending. Before issue closure or a claim of real-provider UX readiness, the maintainer must supply a compatible-provider key and verify a visibly incremental stream in the real UI; deterministic loopback coverage does not satisfy that human gate. + ## Active delivery checkpoint (2026-07-27, proxy-safe direct egress) -- **`#1513` local review-fix implementation:** the registered OpenAI, Gemini, Ollama, and outbound-webhook handlers set `UseProxy = false`, keeping the configured origin as the connect-time validation target. Just before send, protected registrations mask the URI seen by outer .NET HTTP diagnostics; the inner handler restores the real URI for transport and re-masks it for retry/completion. Public caller-owned provider clients keep normal `HttpClient` behavior. The four protected clients remove default request loggers, disable distributed-trace header propagation, and use a private HTTP metric scope; Taskdeck's configured OpenTelemetry provider drops their marked activities and scope. Enabled Sentry removes its outbound factory handler only from these four client pipelines, so unrelated clients retain instrumentation and server-side exception tracking remains active. The proven boundary includes pre-handler `System.Net.Http` EventSource path/query suppression, but excludes independently installed process-global activity/meter listeners and transport-stage host/IP observers. Existing localhost opt-ins and automatic-redirect blocking remain unchanged. No proxy-aware outbound mode exists; corporate proxy-only deployments fail closed. +- **`#1513` local review-fix implementation:** the registered OpenAI, OpenAICompatible, Gemini, Ollama, and outbound-webhook handlers set `UseProxy = false`, keeping the configured origin as the connect-time validation target. Just before send, protected registrations mask the URI seen by outer .NET HTTP diagnostics; the inner handler restores the real URI for transport and re-masks it for retry/completion. Public caller-owned provider clients keep normal `HttpClient` behavior. The five protected clients remove default request loggers, disable distributed-trace header propagation, and use a private HTTP metric scope; Taskdeck's configured OpenTelemetry provider drops their marked activities and scope. Enabled Sentry removes its outbound factory handler only from these five client pipelines, so unrelated clients retain instrumentation and server-side exception tracking remains active. The proven boundary includes pre-handler `System.Net.Http` EventSource path/query suppression, but excludes independently installed process-global activity/meter listeners and transport-stage host/IP observers. Existing localhost opt-ins and automatic-redirect blocking remain unchanged. No proxy-aware outbound mode exists; corporate proxy-only deployments fail closed. - **Current evidence boundary:** the exact documented filters pass 66/66 API and 158/158 Application tests (151 provider/policy compatibility plus seven constructor/remasking controls); registered EventSource and scoped-Sentry controls pass three fresh-process repetitions. At pre-documentation head `dad8d22a`, the full serialized backend passed 7,539 tests with five intentional skips and no failures, including Architecture 20/20 with one intentional skip and Integration 35/35. The independent security review was clean; the correctness review found only the now-corrected seven-test evidence omission. The merged Dockerless Testcontainers repair is absorbed, and the only later issue-scope change is this verification-document correction; the current-main merge touched only the independently reviewed `#1522` frontend/docs slice. Signed publication, exact hosted Required CI, CodeQL, and merge remain outstanding, so this is not yet a shipped delivery entry. ## Delivery update (2026-07-27, test substrate, Windows API diagnostics, and realtime reconciliation) diff --git a/docs/STATUS.md b/docs/STATUS.md index f260c543f..dc9b09c85 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2,8 +2,13 @@ Last Updated: 2026-07-27 +OpenAI-compatible provider replacement checkpoint (2026-07-27, `#1306`; issue branch, not yet shipped): +- **A separately configured `OpenAICompatible` provider now supports public OpenAI Chat Completions endpoints without weakening the safe `Mock` default.** Selection validates the explicit base URL, model, API key, timeouts, response/SSE byte ceilings, and optional non-secret gateway headers. Buffered completion, structured instruction extraction, board context, real upstream SSE, and explicit buffered-stream degradation share sanitized error behavior and preserve token/provider/model provenance. +- **The registered client extends the direct-only telemetry boundary and adds fixed-origin egress enforcement.** The effective chain is circuit policy → protected telemetry → `EgressEnvelopeHandler` → dispatch tracking → direct sockets; it disables proxies and redirects, validates DNS at connect time, preserves the configured origin for transport, and records whether a request crossed the transport boundary so quota reservations are released only for proven pre-dispatch failures. Compatible-provider Polly state and the companion provider circuit state are tracked independently with generation-safe half-open leases. Exact `localhost` plain HTTP is limited to `Development`, `Test`, or `Testing` with the explicit live-provider gate; numeric loopback and other private/link-local origins remain blocked. +- **Local evidence is green:** 136/136 focused Application tests and 80/80 focused API tests cover provider parsing/streaming, raw-byte UTF-8 limits, registered transport/egress/telemetry/Sentry behavior, dispatch-aware quota settlement, and circuit races/cooldowns. The full serialized backend passed **7,660 tests with five intentional skips and no failures** (Domain 1,636; Application 3,678; API 2,189 + 4 skips; CLI 100; Architecture 22 + 1 skip; Integration 35), and docs, golden-principles, GitHub-operations, and whitespace governance passed. Independent exact-head review, hosted CI, and DCO publication are still pending. A maintainer-supplied compatible-provider key plus a visibly incremental real-UI stream remain human gates and are not claimed by deterministic loopback tests. + Proxy-safe direct egress implementation checkpoint (2026-07-27, `#1513`): -- **The registered OpenAI, Gemini, Ollama, and outbound-webhook primary clients preserve the validated direct origin without exporting protected path/query data through Taskdeck's configured telemetry.** Each primary handler sets `UseProxy = false`, so ambient/system proxy settings cannot replace the `OutboundWebhookConnectCallback` target. Immediately before `SendAsync`, registered providers and webhook delivery mask the request URI from outer .NET HTTP diagnostics; the protected inner handler restores the configured URI for transport and re-masks it for retry/completion. Public caller-owned provider clients do not opt into this mechanism and retain ordinary `HttpClient` URI/body/auth behavior. The four protected clients remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private HTTP metric scope; Taskdeck's OpenTelemetry configuration drops marked activities and that scope. When Sentry is enabled, its outbound handler is removed only from these protected clients; unrelated clients such as `GitHubConnectorProvider` retain Sentry instrumentation, while server-side exception tracking remains enabled. The proven boundary covers pre-handler `System.Net.Http` EventSource path/query data and Taskdeck's configured exporters, not independently installed process-global activity/meter listeners or transport-stage host/IP observation. Existing localhost opt-ins and automatic-redirect blocking are unchanged. Proxy-aware outbound support does not exist, so corporate proxy-only deployments fail closed. +- **The registered OpenAI, OpenAICompatible, Gemini, Ollama, and outbound-webhook primary clients preserve the validated direct origin without exporting protected path/query data through Taskdeck's configured telemetry.** Each primary handler sets `UseProxy = false`, so ambient/system proxy settings cannot replace the `OutboundWebhookConnectCallback` target. Immediately before `SendAsync`, registered providers and webhook delivery mask the request URI from outer .NET HTTP diagnostics; the protected inner handler restores the configured URI for transport and re-masks it for retry/completion. Public caller-owned provider clients do not opt into this mechanism and retain ordinary `HttpClient` URI/body/auth behavior. The five protected clients remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private HTTP metric scope; Taskdeck's OpenTelemetry configuration drops marked activities and that scope. When Sentry is enabled, its outbound handler is removed only from these protected clients; unrelated clients such as `GitHubConnectorProvider` retain Sentry instrumentation, while server-side exception tracking remains enabled. The proven boundary covers pre-handler `System.Net.Http` EventSource path/query data and Taskdeck's configured exporters, not independently installed process-global activity/meter listeners or transport-stage host/IP observation. Existing localhost opt-ins and automatic-redirect blocking are unchanged. Proxy-aware outbound support does not exist, so corporate proxy-only deployments fail closed. - **The current-main review-fix working tree is local and not yet shipped:** the documented exact filters pass 66/66 API and 158/158 Application tests (151 provider/policy compatibility plus seven constructor/remasking controls) across registered pipelines, proxy canaries, delivery, EventSource, OpenTelemetry, and scoped-Sentry boundaries. The registered EventSource and Sentry positive/negative controls passed three fresh-process repetitions. At pre-documentation head `dad8d22a`, the full serialized backend passed 7,539 tests with five intentional skips and no failures, including Architecture 20/20 with one intentional skip and Integration 35/35. The independent security review was clean; the correctness review found only the now-corrected seven-test evidence omission. The only later issue-scope change is this verification-document correction; the current-main merge touched only the independently reviewed `#1522` frontend/docs slice, so `#1513` source behavior is unchanged. Signed publication, exact hosted CI, and CodeQL remain outstanding. Realtime column reconciliation (2026-07-27, `#1522`): @@ -237,7 +242,7 @@ Current constraints are mostly hardening and consistency: - ~~**security bug discovered 2026-04-03**: `#722` (SEC-20) — `ChangePassword` endpoint does not verify caller identity~~ **RESOLVED** (`#722`/`#732`, 2026-04-04): `ChangePassword` now derives userId exclusively from JWT claims; `[Authorize]` enforced; `UserId` removed from request body; `AuthController` inherits `AuthenticatedControllerBase`; 5 integration tests proving the fix - security and identity behavior is converging but still not uniform across all controller families - some UX/operator surfaces are functional but not yet keyboard-first or discoverability-first -- LLM flow now supports config-gated `OpenAI` and `Gemini` providers with deterministic `Mock` fallback for safe local/test posture; degraded provider responses are now structurally distinct (`messageType: "degraded"` + `degradedReason`) and the health endpoint supports opt-in probe verification (`?probe=true`); chat-to-proposal pipeline improvements delivered: `LlmIntentClassifier` now uses compiled regex patterns with word-distance matching, stemming/plurals, broader verb coverage, and negative context filtering for negations and other-tool questions (`#571`); parse failures now return structured hint payloads with closest-match suggestions and a frontend hint card with "try this instead" pre-fill (`#572`); dedicated classifier and chat-to-proposal integration test coverage added (`#577`); LLM-assisted instruction extraction now delivered (`#573`): OpenAI and Gemini providers request structured JSON output with a system prompt describing supported instruction patterns, parse the response into `LlmCompletionResult.Instructions`, and fall back to the static `LlmIntentClassifier` when structured parsing fails; `ChatService` iterates LLM-extracted instructions (supporting multiple proposals from a single message) and falls back to raw user message parsing when no instructions are extracted; Mock provider unchanged for deterministic test behavior; multi-instruction batch parsing now delivered (`#574`): `ParseBatchInstructionAsync` splits multiple natural-language instructions into individual planner calls, `ChatService` routes multi-instruction messages through batch parsing to generate multiple proposals from a single chat message; board-context LLM prompting now delivered (`#575`, expanded in `#617`): `BoardContextBuilder` constructs bounded board context (columns, card IDs, titles, labels) grouped per column and appends it to system prompts across OpenAI and Gemini providers via `LlmSystemPromptBuilder`; card IDs are included as first-8 hex chars so the LLM can generate `move card ` instructions; context budget increased to 4000 chars with single-query card fetch; **remaining gap**: conversational refinement (`#576`) remains undelivered; analysis at `docs/analysis/2026-03-29_chat_nlp_proposal_gap.md` +- LLM flow now supports config-gated `OpenAI`, `OpenAICompatible`, `Gemini`, and `Ollama` providers with deterministic `Mock` fallback for safe local/test posture; degraded provider responses are now structurally distinct (`messageType: "degraded"` + `degradedReason`) and the health endpoint supports opt-in probe verification (`?probe=true`); chat-to-proposal pipeline improvements delivered: `LlmIntentClassifier` now uses compiled regex patterns with word-distance matching, stemming/plurals, broader verb coverage, and negative context filtering for negations and other-tool questions (`#571`); parse failures now return structured hint payloads with closest-match suggestions and a frontend hint card with "try this instead" pre-fill (`#572`); dedicated classifier and chat-to-proposal integration test coverage added (`#577`); LLM-assisted instruction extraction now delivered (`#573`): OpenAI, OpenAICompatible, Gemini, and Ollama providers request structured JSON output with a system prompt describing supported instruction patterns, parse the response into `LlmCompletionResult.Instructions`, and fall back to the static `LlmIntentClassifier` when structured parsing fails; `ChatService` iterates LLM-extracted instructions (supporting multiple proposals from a single message) and falls back to raw user message parsing when no instructions are extracted; Mock provider unchanged for deterministic test behavior; multi-instruction batch parsing now delivered (`#574`): `ParseBatchInstructionAsync` splits multiple natural-language instructions into individual planner calls, `ChatService` routes multi-instruction messages through batch parsing to generate multiple proposals from a single chat message; board-context LLM prompting now delivered (`#575`, expanded in `#617`): `BoardContextBuilder` constructs bounded board context (columns, card IDs, titles, labels) grouped per column and appends it to system prompts across OpenAI, OpenAICompatible, Gemini, and Ollama providers via `LlmSystemPromptBuilder`; card IDs are included as first-8 hex chars so the LLM can generate `move card ` instructions; context budget increased to 4000 chars with single-query card fetch; **remaining gap**: conversational refinement (`#576`) remains undelivered; analysis at `docs/analysis/2026-03-29_chat_nlp_proposal_gap.md` - managed-key shared-token abuse-control strategy is now explicitly seeded in `#235` to `#240` before broad external exposure - testing-harness guardrail expansion from `#254` to `#260` is shipped; remaining work is normal follow-up hardening rather than the original wave - rigorous test expansion wave seeded 2026-04-03 (`#721` tracker, 22 issues `#699`–`#726`): systematic codebase audit identified 25+ untested infrastructure repositories, zero tests on the central worker, 6 controllers with untested HTTP surfaces, and no golden-path integration test for the capture → proposal → board pipeline; execution is tracked in `docs/TESTING_GUIDE.md`; first delivery: infrastructure repository integration tests (`#699`/`#730` — 77 tests across 7 repo classes against real SQLite); **major wave delivery 2026-04-04** (PRs `#732`–`#739`, 8 issues, ~300 new tests): SEC-20 ChangePassword fix (`#722`/`#732`), golden-path capture→board integration test (`#703`/`#735` — 7 tests proving full pipeline), cross-user data isolation tests (`#704`/`#733` — 38 tests across all major API boundaries), LlmQueueToProposalWorker integration tests (`#700`/`#734` — 24 tests, previously zero coverage), controller HTTP integration tests (`#702`/`#738` — 67 tests covering 6 untested controllers, found 2 pre-existing bugs), proposal lifecycle edge cases (`#708`/`#736` — 74 tests for state machine/expiry/race conditions), OAuth/auth edge cases (`#707`/`#737` — 44 tests, found and fixed `Substring` overflow bug in `ExternalLoginAsync`), MCP full resource/tool inventory (`#653`/`#739` — 9 resources + 11 tools with 42 tests, GP-06 compliant, user-scoping gap fixed during review); **second wave delivery 2026-04-04** (PRs `#740`–`#755`, 8 issues, ~586 new tests with two rounds of adversarial review, 47 review-fix commits): domain entity state machine exhaustive tests (`#701`/`#740` — 174 tests across 7 entities: CommandRun, ArchiveItem, ChatSession, UserPreference, NotificationPreference, CardLabel, CardCommentMention), SignalR hub and realtime integration tests (`#706`/`#751` — 19 tests covering auth, presence lifecycle, multi-user, authorization, edge cases), LLM provider abstraction and tool-calling edge cases (`#709`/`#747` — 101 tests across orchestrator, provider, classifier, registry), data export/import round-trip integrity tests (`#713`/`#752` — 64 tests covering JSON, CSV, GDPR, database, cross-format validation), API error contract regression and boundary validation (`#714`/`#753` — 57 tests across 7 endpoint families with GP-03 contract enforcement), archive and restore lifecycle integration tests (`#715`/`#755` — 74 tests: 45 domain + 29 API covering state machine, cross-user isolation, conflict detection, audit trail), board metrics and analytics accuracy verification (`#718`/`#749` — 61 tests: 51 service + 10 controller covering throughput, cycle time, WIP, blocked cards, done-column heuristic), notification delivery, deduplication, and preference filtering (`#719`/`#746` — 36 tests covering all 5 notification types, deduplication, preference filtering, cross-user isolation, batch operations) @@ -444,7 +449,7 @@ Direction guardrails (explicit): - `StarterPackManifestValidator` decomposed into `StarterPackSchemaValidator`, `StarterPackSemanticValidator`, `StarterPackConflictDetector`, `StarterPackIdempotencyChecker` - `AbuseDetectionService` with `AbuseActor`/`AbuseEvent` domain entities and a 4-state containment model (Observe → Suspicious → Restricted → Blocked); operator kill-switch API groundwork for SEC-18 - agent tool registry substrate (AGT-02): `ITaskdeckTool`/`ITaskdeckToolRegistry` domain interfaces with `ToolScope`/`ToolRiskLevel` classification, `PolicyDecision` value object, `AgentPolicyEvaluator` (allowlist + risk-level gating, review-first default), `InboxTriageAssistant` bounded template (proposal-only, never direct board mutation), singleton registry with scoped evaluation - - `ChatService` + deterministic `ILlmProvider` selection policy (`Mock` default; `OpenAI`/`Gemini` behind explicit gates with config validation fallback); `ToolCallingChatOrchestrator` wraps `ChatService` for board-scoped sessions with multi-turn tool-calling loop (11 tools: 5 read + 6 write, max 5 rounds, 60s timeout, Mock pattern-based dispatch); write tools produce proposals via `propose_*` prefix (GP-06 compliant); `ChatService` reuses orchestrator text when no tools called to avoid double LLM invocation; streaming responses now persist assistant `ChatMessage` records with token usage and record quota via `ILlmQuotaService` (`#763`/`#768`); multi-turn replay preserves original tool arguments in provider-specific wire format (`#673`/`#770`); **conversational refinement loop** (`#576`/`#791`): `ClarificationDetector` with strong/weak signal pattern split detects ambiguous requests and asks clarifying questions (max 2 rounds, then best-effort); skip-phrase detection supports "just do your best"; Mock provider simulates clarification for deterministic testing + - `ChatService` + deterministic `ILlmProvider` selection policy (`Mock` default; `OpenAI`/`OpenAICompatible`/`Gemini`/`Ollama` behind explicit gates with config validation fallback); `ToolCallingChatOrchestrator` wraps `ChatService` for board-scoped sessions with multi-turn tool-calling loop (11 tools: 5 read + 6 write, max 5 rounds, 60s timeout, Mock pattern-based dispatch); write tools produce proposals via `propose_*` prefix (GP-06 compliant); `ChatService` reuses orchestrator text when no tools called to avoid double LLM invocation; streaming responses now persist assistant `ChatMessage` records with token usage and record quota via `ILlmQuotaService` (`#763`/`#768`); multi-turn replay preserves original tool arguments in provider-specific wire format (`#673`/`#770`); **conversational refinement loop** (`#576`/`#791`): `ClarificationDetector` with strong/weak signal pattern split detects ambiguous requests and asks clarifying questions (max 2 rounds, then best-effort); skip-phrase detection supports "just do your best"; Mock provider simulates clarification for deterministic testing - `DataExportService` (versioned JSON export of all user-scoped data; streaming export via new `GET /api/account/export/stream` endpoint using `Utf8JsonWriter` for memory-constant large-dataset exports — `#670`/`#774`; exception logging via `ILogger` with `OperationCanceledException` filter, `#759`/`#766`) + `AccountDeletionService` (password re-auth, confirmation phrase, PII anonymization, sole-owner guard, transactional rollback with `CancellationToken.None` for rollback reliability) + `DataPortabilityController` with audit logging - `BoardMetricsService` (throughput, cycle time, WIP, blocked — audit-log-based completion tracking, done column name heuristic, SQL-level filtering via dedicated repository methods) + `MetricsController` with date/board/label filters + `MetricsExportService` for schema-versioned CSV export with CSV injection protection (`#78`/`#787`) - `ForecastingService` (heuristic completion forecasting using rolling-average throughput from audit log card-move events, standard-deviation confidence bands, cycle time tracking) + `ForecastController` with `GET /api/forecast/board/{boardId}` endpoint (`#79`/`#790`) @@ -456,7 +461,7 @@ Direction guardrails (explicit): - `StarterPackManifestValidator` + `StarterPackApplyService` (idempotent apply with dry-run conflict reporting) - SignalR realtime baseline: `BoardsHub` with board-scoped subscription authz and application-level board mutation event publishing; **scale-out readiness** (`#105`/ADR-0025): conditional Redis backplane via `Microsoft.AspNetCore.SignalR.StackExchangeRedis` 8.0.28 (pinned in `Directory.Packages.props`; semver-major capped per `#1225`) — enabled when `SignalR:Redis:ConnectionString` is configured, falls back to in-memory when absent; `RedisBackplaneHealthCheck` reports NotConfigured/Healthy/Unhealthy in `/health/ready`; operational runbook at `docs/platform/SIGNALR_SCALEOUT_RUNBOOK.md` - OpenTelemetry baseline for API + worker metrics/traces with configurable OTLP/console exporters - - Polly circuit breakers on OpenAI, Gemini HTTP clients, and OAuth backchannel; `CircuitBreakerStateTracker` singleton reports per-client state (closed/open/half-open) via health endpoint as degraded (not 503); `CircuitBreakerSettings` config class for threshold tuning + - Polly circuit breakers on OpenAI, OpenAICompatible, Gemini, and Ollama HTTP clients, and OAuth backchannel; `CircuitBreakerStateTracker` singleton reports per-client state (closed/open/half-open) via health endpoint as degraded (not 503); `CircuitBreakerSettings` config class for threshold tuning - security logging redaction baseline for capture/auth-sensitive flows: sanitized exception summaries in middleware/workers/providers, generic invalid-source errors, redacted persisted queue/webhook failure messages, and disabled automatic ASP.NET Core trace exception recording - Auth posture today: - JWT middleware is wired @@ -1307,7 +1312,7 @@ Security and identity: - policy decision is now explicit: cross-user authenticated access failures should return `403`; remaining work is consistent enforcement across all families/tests Automation and data: -- active LLM provider policy supports explicit mock vs live-provider switching (`OpenAI`/`Gemini`) with safe defaults for development/test environments +- active LLM provider policy supports explicit mock vs live-provider switching (`OpenAI`/`OpenAICompatible`/`Gemini`/`Ollama`) with safe defaults for development/test environments - managed-key shared-token controls are now more broadly shipped: identity attribution baseline (`#236`), user-facing usage policy (`#240`, `docs/security/MANAGED_KEY_USAGE_POLICY.md`), secrets/config management baseline (SEC-10, `docs/security/SECRETS_MANAGEMENT_BASELINE.md`), incident runbook + drill scripts (SEC-19, `docs/security/MANAGED_KEY_INCIDENT_RUNBOOK.md` + `scripts/drills/`), and abuse detection domain groundwork + operator API (`#238` SEC-18, `AbuseActor`/`AbuseEvent`/`AbuseDetectionService` with 4-state model) are all delivered; remaining automated live-traffic containment and quota enforcement remain tracked in `#237` (kill-switch budget guardrails) and the SEC-18 follow-through slice for live wiring - planner extraction remains rule/regex-based with deterministic validation and expanded board/column operation coverage - database-level export/import now exists as a minimal safe implementation and is restricted to Development sandbox mode diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index 1ef574713..10b08f14e 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -29,6 +29,43 @@ Verification note: - prior recertification: backend 6,336 (2026-05-05 after Paper backend gap PR `#1040`), frontend 2,805 (2026-04-25) - growth since last recertification: backend +278 passing tests, frontend +462 passing tests +## OpenAI-Compatible Provider Replacement Checkpoint (`#1306`) + +Local replacement-branch verification on 2026-07-27 exercises the compatible +provider, the dispatch/accounting seam, and the registered transport chain: + +```powershell +dotnet test backend/tests/Taskdeck.Application.Tests/Taskdeck.Application.Tests.csproj -c Release -m:1 --filter "FullyQualifiedName~OpenAiCompatibleLlmProviderTests|FullyQualifiedName~CircuitBreakerStateTrackerTests|FullyQualifiedName~LlmDispatchTrackingHandlerTests|FullyQualifiedName~ChatServiceTests|FullyQualifiedName~LlmCaptureTriageExtractorTests" +dotnet test backend/tests/Taskdeck.Api.Tests/Taskdeck.Api.Tests.csproj -c Release -m:1 --filter "FullyQualifiedName~LlmProviderRegistrationTests|FullyQualifiedName~ProtectedOutboundTelemetryHandlerTests|FullyQualifiedName~CircuitBreakerTests" +``` + +Results: **136 passed / 0 failed / 0 skipped** in Application and **80 passed / +0 failed / 0 skipped** in API. Coverage includes buffered completion and real +registered-loopback SSE; UTF-8 byte, line, event, and aggregate ceilings; +content-filter/refusal handling without persisting provider detail; zero/known +usage normalization; every-redirect rejection; exact-origin egress and DNS +checks; direct-only proxy and telemetry controls; scoped Sentry removal; 501 +stream fallback outside the compatible Polly failure set; pre-dispatch versus +dispatched quota settlement; and deterministic half-open race/cooldown behavior +across separate Polly and companion circuit states. + +The compatible registration expands the protected-client inventory from the +four clients proved by `#1513` to five: OpenAI, OpenAICompatible, Gemini, Ollama, +and outbound webhook delivery. The same code tree passed the required full gate: + +```powershell +dotnet test backend/Taskdeck.sln -c Release -m:1 +``` + +Result: **7,660 passed, 5 intentional skips, 0 failed** (Domain 1,636; +Application 3,678; API 2,189 + 4 skips; CLI 100; Architecture 22 + 1 skip; +Integration 35). Docs governance, golden-principles governance, GitHub-operations +governance, and `git diff --check` also passed. Independent review and hosted +exact-head evidence remain pending. A +maintainer-supplied compatible-provider key and a visibly incremental stream in +the real UI are separate human gates; loopback transport tests do not prove +either. + ## Proxy-Safe Direct Egress Checkpoint (`#1513`) Local issue-branch verification on 2026-07-27 covers the direct-only primary @@ -54,7 +91,7 @@ to prove normal trace propagation/activity/metric export while protected request configured OpenTelemetry exporter. The metric guarantee is deliberately scoped to Taskdeck's exporter, not arbitrary process-global listeners. Registered-provider controls also prove that outer .NET HTTP EventSource payloads do not contain the configured path/query while the real configured origin reaches the wire, -and that Sentry's outbound handler is removed only from the four protected clients: the unrelated +and that Sentry's outbound handler was removed only from the four then-registered protected clients: the unrelated `GitHubConnectorProvider` client retains the handler and sends `sentry-trace`. Public caller-owned provider clients retain their configured URI, request body, and authentication. The guarantee does not cover independently installed Activity/Meter listeners or transport-stage host/IP observation. diff --git a/docs/platform/CONFIGURATION_REFERENCE.md b/docs/platform/CONFIGURATION_REFERENCE.md index 683a08434..6b546865c 100644 --- a/docs/platform/CONFIGURATION_REFERENCE.md +++ b/docs/platform/CONFIGURATION_REFERENCE.md @@ -216,14 +216,14 @@ default and the only one that ships enabled. See | Key | Type | Default | Description | Required? | | --- | --- | --- | --- | --- | | `Llm:EnableLiveProviders` | `bool` | `false` | Master switch. Live providers (OpenAI, OpenAICompatible, Gemini, Ollama) only run when this is true. | No | -| `Llm:AllowLiveProvidersInDevelopment` | `bool` | `false` | Safety gate — live providers refuse to run in the `Development` environment unless this is also true. | No | +| `Llm:AllowLiveProvidersInDevelopment` | `bool` | `false` | Safety gate — live providers refuse to run in `Development`, `Test`, or `Testing` unless this is also true. | No | | `Llm:Provider` | `string` | `Mock` | Provider selector. `Mock`, `OpenAi`, `OpenAiCompatible`, `Gemini`, or `Ollama`. Resolved by `LlmProviderSelectionPolicy.Evaluate`. | No | | `Llm:OpenAi:ApiKey` | `string` | `""` | OpenAI API key. Required to use the OpenAI provider. Store as a secret. | Only for `Llm:Provider = OpenAi` | -| `Llm:OpenAi:BaseUrl` | `string` | `https://api.openai.com/v1` | OpenAI API base URL. Override for compatible gateways. | No | +| `Llm:OpenAi:BaseUrl` | `string` | `https://api.openai.com/v1` | OpenAI API base URL. Use `OpenAiCompatible` for third-party compatible gateways. | No | | `Llm:OpenAi:Model` | `string` | `gpt-4o-mini` | Model identifier sent in chat requests. | No | | `Llm:OpenAi:TimeoutSeconds` | `int` | `30` | `HttpClient.Timeout` applied to the OpenAI provider. Must be `> 0`: `LlmProviderSelectionPolicy.TryValidateOpenAiSettings` rejects values `<= 0` as invalid and the selection policy falls back to the Mock provider. (The `HttpClient` registration also substitutes `30` when the value is `<= 0`, but only as a safety net — the provider will still not be selected.) | No | | `Llm:OpenAiCompatible:ApiKey` | `string` | `""` | API key for the configured OpenAI-compatible endpoint. Store as a secret. | Only for `Llm:Provider = OpenAiCompatible` | -| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. Production/non-development endpoints must use public HTTPS; plain HTTP is accepted only for gated loopback development. The URL may contain a path but not user information, a query, or a fragment, and passes private-network, metadata-host, egress-envelope, and DNS connection-time SSRF controls. | Only for `Llm:Provider = OpenAiCompatible` | +| `Llm:OpenAiCompatible:BaseUrl` | `string` | `""` | Required OpenAI Chat Completions-compatible base URL, such as `https://openrouter.ai/api/v1`, `https://api.groq.com/openai/v1`, or `https://api.deepseek.com/v1`. Public HTTPS is required except when the host is exactly `localhost`, the environment is `Development`, `Test`, or `Testing`, and `Llm:AllowLiveProvidersInDevelopment=true`; numeric loopback addresses such as `127.0.0.1` and `[::1]`, and all other private/link-local hosts, remain blocked. The URL may contain a path but not user information, a query, or a fragment, and passes private-network, metadata-host, egress-envelope, and DNS connection-time SSRF controls. | Only for `Llm:Provider = OpenAiCompatible` | | `Llm:OpenAiCompatible:Model` | `string` | `""` | Required model identifier supplied by the compatible vendor. | Only for `Llm:Provider = OpenAiCompatible` | | `Llm:OpenAiCompatible:TimeoutSeconds` | `int` | `30` | Full compatible-provider response deadline, including response-body/SSE reads after headers. Valid range: 1--300; invalid values select Mock. | No | | `Llm:OpenAiCompatible:MaxResponseBytes` | `int` | `1048576` | Maximum UTF-8 bytes accepted from a buffered response or aggregate SSE response. Valid range: 1024--4194304. | No | @@ -239,15 +239,16 @@ default and the only one that ships enabled. See | `Llm:Ollama:TimeoutSeconds` | `int` | `120` | `HttpClient.Timeout` applied to Ollama. Must be between `1` and `600`; invalid values make provider selection fall back to Mock. | No | | `Llm:Ollama:AllowLocalhostEndpoints` | `bool` | `false` | Additional Ollama opt-in for exact `localhost`. Effective only in Development/Test/Testing when `Llm:AllowLiveProvidersInDevelopment=true`; it never permits literal loopback/private addresses or Production localhost. | No | -**Outbound transport boundary:** the OpenAI, Gemini, and Ollama primary clients -set `UseProxy = false`. They ignore ambient/system proxy settings so the -configured provider origin remains the target inspected by +**Outbound transport boundary:** the OpenAI, OpenAICompatible, Gemini, and +Ollama primary clients set `UseProxy = false`. They ignore ambient/system proxy +settings so the configured provider origin remains the target inspected by `OutboundWebhookConnectCallback`. The existing development-localhost opt-ins remain unchanged. There is no proxy-aware provider setting: corporate -proxy-only deployments fail closed. This path does not use -`EgressEnvelopeHandler`, and its existing no-auto-redirect and audit behavior is -unchanged. Default `IHttpClientFactory` request logging is removed for these -protected clients. Their primary handlers disable distributed-trace header +proxy-only deployments fail closed. OpenAI, Gemini, and Ollama retain the +dedicated connect-callback boundary without `EgressEnvelopeHandler`; +OpenAICompatible additionally uses a fixed-origin `EgressEnvelopeHandler` and +rejects all redirects. Default `IHttpClientFactory` request logging is removed +for these protected clients. Their primary handlers disable distributed-trace header propagation, and Taskdeck's configured OpenTelemetry pipeline excludes their marked HTTP activities and private-scope HTTP metrics (including destination dimensions such as `server.address` and `server.port`). This is Taskdeck exporter @@ -466,7 +467,7 @@ Bound to `CircuitBreakerSettings`. Consumed by `Taskdeck.Api.Extensions.LlmProviderRegistration.AddLlmProviders` and `Taskdeck.Api.Extensions.AuthenticationRegistration.AddTaskdeckAuthentication`. Polly circuit breaker policies are applied to LLM provider HTTP clients -(OpenAI, Gemini) and OAuth/OIDC backchannel handlers. +(OpenAI, OpenAICompatible, Gemini, and Ollama) and OAuth/OIDC backchannel handlers. | Key | Type | Default | Env override | Description | Restart required? | |-----|------|---------|--------------|-------------|-------------------| @@ -548,7 +549,7 @@ Sentry is fully off until explicitly opted in. When enabled, Taskdeck keeps Sentry's server-side exception tracking and removes Sentry's automatic outbound `IHttpClientFactory` handler only from the registered -OpenAI, Gemini, Ollama, and `OutboundWebhookDelivery` clients. Those protected +OpenAI, OpenAICompatible, Gemini, Ollama, and `OutboundWebhookDelivery` clients. Those protected clients therefore do not acquire Sentry trace/baggage propagation, URL breadcrumbs, or failed-request capture outside their dedicated telemetry boundary. Unrelated factory clients retain normal Sentry instrumentation. diff --git a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md index 547b184d0..3be499479 100644 --- a/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md +++ b/docs/platform/LLM_PROVIDER_SETUP_GUIDE.md @@ -26,15 +26,14 @@ Backend provider runtime now supports: Selection is deterministic through `LlmProviderSelectionPolicy`: - to use live providers (`OpenAI`, `OpenAICompatible`, `Gemini`, or `Ollama`), live providers must be enabled (`EnableLiveProviders=true`) -- to use live providers in development-like environments, explicit live mode is required (`AllowLiveProvidersInDevelopment=true`) +- to use live providers in `Development`, `Test`, or `Testing`, explicit live mode is required (`AllowLiveProvidersInDevelopment=true`) - provider mode may be explicitly set to `Mock`, `OpenAI`, `OpenAICompatible`, `Gemini`, or `Ollama`; this guide's config example intentionally uses `Mock` as the safe default - unknown provider values also fall back deterministically to `Mock` - selected provider config must pass provider-specific validation (`BaseUrl`, `Model`, `TimeoutSeconds`, and an API key where required) -- `BaseUrl` is additionally validated by `SsrfProtectionService.ValidateLlmProviderUrl` (SEC-26 PR `#905`): private IPv4 (`127/8`, `10/8`, `172.16/12`, `192.168/16`), IPv6 ranges (`::1`, `fc00::/7`, `fe80::/10`), IPv4-mapped IPv6, cloud metadata hostnames (`metadata.google.internal`, `metadata.goog`, AWS IMDS `169.254.169.254`, AWS IMDSv2 IPv6 `fd00:ec2::254`, Alibaba `100.100.100.200`), and non-HTTPS URLs are rejected; the selection policy falls back to Mock when validation fails +- `BaseUrl` is additionally validated by `SsrfProtectionService.ValidateLlmProviderUrl` (SEC-26 PR `#905`): private IPv4 (`127/8`, `10/8`, `172.16/12`, `192.168/16`), IPv6 ranges (`::1`, `fc00::/7`, `fe80::/10`), IPv4-mapped IPv6, cloud metadata hostnames (`metadata.google.internal`, `metadata.goog`, AWS IMDS `169.254.169.254`, AWS IMDSv2 IPv6 `fd00:ec2::254`, Alibaba `100.100.100.200`), and non-HTTPS URLs are rejected except for the exact gated `localhost` case below; the selection policy falls back to Mock when validation fails - the OpenAI, OpenAICompatible, Gemini, and Ollama primary `HttpClient` handlers use `OutboundWebhookConnectCallback` for DNS-level SSRF protection and set both `AllowAutoRedirect = false` and `UseProxy = false`; ambient/system proxy settings are ignored so the configured provider origin remains the host validated by the connect callback -These provider transports are direct-only. Taskdeck has no proxy-aware outbound LLM mode, so a deployment that can reach providers only through a corporate proxy fails closed. This is a dedicated connect-callback boundary, not `EgressEnvelopeHandler` enforcement, and it does not change the existing redirect or audit posture. Registered protected clients mask their configured URI immediately before send, then an inner handler restores it for transport; this keeps path/query data out of outer .NET HTTP EventSource payloads. Public caller-owned provider clients do not opt into masking. Protected registrations remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private metric scope that Taskdeck's configured OpenTelemetry pipeline drops alongside marked HTTP activities. Enabled Sentry removes its outbound handler from these protected client pipelines only, so it cannot add separate `sentry-trace`/baggage headers or capture protected URLs and 5xx responses; unrelated clients retain instrumentation and server-side Sentry tracking remains active. Independently installed process-global `ActivityListener`/`MeterListener` and transport-stage host/IP observation remain outside the guarantee. -OpenAICompatible additionally rejects every observed 3xx response in its egress handler rather than following an allowlisted redirect. +These provider transports are direct-only. Taskdeck has no proxy-aware outbound LLM mode, so a deployment that can reach providers only through a corporate proxy fails closed. OpenAI, Gemini, and Ollama retain their dedicated connect-callback boundary without `EgressEnvelopeHandler`; OpenAICompatible additionally passes through a fixed-origin `EgressEnvelopeHandler` immediately inside the protected telemetry handler and rejects every observed 3xx response rather than following an allowlisted redirect. Registered protected clients mask their configured URI immediately before send, then the protected inner handler restores it for validation and transport; this keeps path/query data out of outer .NET HTTP EventSource payloads. Public caller-owned provider clients do not opt into masking. Protected registrations remove default `IHttpClientFactory` request loggers, disable distributed-trace header propagation, and use a private metric scope that Taskdeck's configured OpenTelemetry pipeline drops alongside marked HTTP activities. Enabled Sentry removes its outbound handler from these protected client pipelines only, so it cannot add separate `sentry-trace`/baggage headers or capture protected URLs and 5xx responses; unrelated clients retain instrumentation and server-side Sentry tracking remains active. Independently installed process-global `ActivityListener`/`MeterListener` and transport-stage host/IP observation remain outside the guarantee. If any live-provider condition fails, runtime degrades safely to `Mock`. @@ -44,14 +43,17 @@ To support local LLM workflows (Ollama, LM Studio, LocalAI, etc.), the effective connect-time exception is scoped to the exact `localhost` hostname **only** when both of the following are true: -- the environment is `Development` (or a development-like host name) +- the environment is exactly `Development`, `Test`, or `Testing` - `Llm:AllowLiveProvidersInDevelopment = true` is set explicitly Ollama additionally requires `Llm:Ollama:AllowLocalhostEndpoints = true`. Literal loopback and other private/link-local addresses remain blocked by the connect callback; use the exact `localhost` hostname for an opted-in local provider. -Under this bypass, HTTPS is also not required for `localhost` endpoints. +Plain HTTP is accepted only when the URI host is exactly `localhost`, the +environment is `Development`, `Test`, or `Testing`, and +`Llm:AllowLiveProvidersInDevelopment=true`. Numeric loopback addresses such as +`127.0.0.1` and `[::1]`, and all other private/link-local hosts, remain blocked. This exception is intentionally narrow: Staging/Production deployments can never reach `localhost` LLM endpoints even if the configuration is cloned. All other private IP ranges and cloud metadata hostnames stay blocked even @@ -157,13 +159,15 @@ for chat streams and forwards delta events as they arrive. Set the common safety gates and provider name: - `Llm__EnableLiveProviders=true` -- `Llm__AllowLiveProvidersInDevelopment=true` (only for development-like environments) +- `Llm__AllowLiveProvidersInDevelopment=true` (only for `Development`, `Test`, or `Testing`) - `Llm__Provider=OpenAICompatible` Production and other non-development endpoints must be public HTTPS and pass the same URL and DNS-level SSRF checks as OpenAI. Plain HTTP is accepted only -for loopback development endpoints when both the development environment and -`AllowLiveProvidersInDevelopment` gate permit it. Keep keys in a secret store; +when the URI host is exactly `localhost`, the environment is `Development`, +`Test`, or `Testing`, and `Llm:AllowLiveProvidersInDevelopment=true`. Numeric +loopback addresses such as `127.0.0.1` and `[::1]`, and all other private or +link-local hosts, remain blocked. Keep keys in a secret store; never commit them. Compatible gateways may require optional non-secret headers such as `HTTP-Referer` or `X-Title`; use `Llm__OpenAiCompatible__ExtraHeaders__` for those. @@ -277,7 +281,7 @@ This is intentionally separate from the broader demo tooling so an operator can ## Behavior Guarantees -- LLM-consuming application services remain provider-agnostic (`ChatService` depends on `ILlmProvider` only). Capture triage does not depend on `ILlmProvider` at all — it is a deterministic, offline extractor. +- LLM-consuming application services remain provider-agnostic (`ChatService` and `LlmCaptureTriageExtractor` depend on `ILlmProvider`, not a concrete adapter). - invalid/missing live-provider configuration does not crash requests - provider adapters return deterministic fallback responses when upstream calls fail - capture triage provenance persists `promptVersion`, `provider`, and `model` @@ -288,11 +292,12 @@ This is intentionally separate from the broader demo tooling so an operator can ## Test Coverage Expectations (Implemented) -- selection-policy unit coverage for `Mock`/`OpenAI`/`OpenAICompatible`/`Gemini` and invalid-config fallback +- selection-policy unit coverage for `Mock`/`OpenAI`/`OpenAICompatible`/`Gemini`/`Ollama` and invalid-config fallback - provider adapter unit coverage: - OpenAI: success/failure + metadata checks - - OpenAICompatible: true SSE delta parsing, malformed/mid-stream error, cancellation, and explicit buffered-fallback metadata + - OpenAICompatible: true SSE delta parsing, strict UTF-8 byte ceilings, malformed/mid-stream error, cancellation, zero/known usage, content-filter/refusal handling, fixed-origin egress, registered transport, and explicit buffered-fallback metadata - Gemini: success/failure/invalid-response/invalid-config/cancellation + health + attribution header mapping + - Ollama: success/failure/streaming/structured extraction + localhost-policy checks - API integration coverage: - capture triage provenance includes provider/model - chat flow validated using a non-mock provider stub with attribution assertions