Improve parsing of responses from the AI

OpenConsole.sln

@@ -2334,10 +2334,14 @@ Global
 		{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x64.Build.0 = Release|x64
 		{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x86.ActiveCfg = Release|Win32
 		{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x86.Build.0 = Release|Win32
-        {7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.ActiveCfg = Debug|Win32
-        {7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.ActiveCfg = Release|ARM64
-        {7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.ActiveCfg = Release|x64
-        {7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.ActiveCfg = Release|Win32
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.ActiveCfg = AuditMode|x64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.Build.0 = AuditMode|x64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.ActiveCfg = AuditMode|ARM64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.Build.0 = AuditMode|ARM64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.ActiveCfg = AuditMode|x64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.Build.0 = AuditMode|x64
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.ActiveCfg = AuditMode|Win32
+		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.Build.0 = AuditMode|Win32
 		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|Any CPU.ActiveCfg = Debug|x64
 		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|ARM64.ActiveCfg = Debug|ARM64
 		{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|ARM64.Build.0 = Debug|ARM64

src/cascadia/QueryExtension/ExtensionPalette.cpp

@@ -127,7 +127,7 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
 
     winrt::fire_and_forget ExtensionPalette::_getSuggestions(const winrt::hstring& prompt, const winrt::hstring& currentLocalTime)
     {
-        const auto userMessage = winrt::make<ChatMessage>(prompt, true, false);
+        const auto userMessage = winrt::make<ChatMessage>(prompt, true);
         std::vector<IInspectable> userMessageVector{ userMessage };
         const auto queryAttribution = _lmProvider ? _lmProvider.BrandingData().QueryAttribution() : winrt::hstring{};
         const auto userGroupedMessages = winrt::make<GroupedChatMessages>(currentLocalTime, true, winrt::single_threaded_vector(std::move(userMessageVector)), queryAttribution);
@@ -197,48 +197,35 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
 
     void ExtensionPalette::_splitResponseAndAddToChatHelper(const IResponse response)
     {
-        // this function is dependent on the AI response separating code blocks with
-        // newlines and "```". OpenAI seems to naturally conform to this, though
-        // we could probably engineer the prompt to specify this if we need to.
-        std::wstringstream ss(response.Message().c_str());
-        std::wstring line;
-        std::wstring codeBlock;
-        bool inCodeBlock = false;
         const auto time = _getCurrentLocalTimeHelper();
         std::vector<IInspectable> messageParts;
 
-        while (std::getline(ss, line))
-        {
-            if (!line.empty())
+        const auto chatMsg = winrt::make<ChatMessage>(response.Message(), false);
+        chatMsg.RunCommandClicked([this](auto&&, const auto commandlines) {
+            auto suggestion = winrt::to_string(commandlines);
+            // the AI sometimes sends multiline code blocks
+            // we don't want to run any of those commands when the chat item is clicked,
+            // so we replace newlines with the appropriate delimiter
+            size_t pos = 0;
+            while ((pos = suggestion.find("\n", pos)) != std::string::npos)
             {
-                if (!inCodeBlock && line.find(L"```") == 0)
-                {
-                    inCodeBlock = true;
-                    continue;
-                }
-                if (inCodeBlock && line.find(L"```") == 0)
-                {
-                    inCodeBlock = false;
-                    const auto chatMsg = winrt::make<ChatMessage>(winrt::hstring{ std::move(codeBlock) }, false, true);
-                    messageParts.push_back(chatMsg);
-                    codeBlock.clear();
-                    continue;
-                }
-                if (inCodeBlock)
-                {
-                    if (!codeBlock.empty())
-                    {
-                        codeBlock += L'\n';
-                    }
-                    codeBlock += line;
-                }
-                else
-                {
-                    const auto chatMsg = winrt::make<ChatMessage>(winrt::hstring{ line }, false, false);
-                    messageParts.push_back(chatMsg);
-                }
+                const auto delimiter = (_ActiveCommandline == cmdExe || _ActiveCommandline == cmd) ? cmdCommandDelimiter : commandDelimiter;
+                suggestion.at(pos) = delimiter;
+                pos += 1; // Move past the replaced character
             }
-        }
+            _InputSuggestionRequestedHandlers(*this, winrt::to_hstring(suggestion));
+            _close();
+
+            const auto lmProviderName = _lmProvider ? _lmProvider.BrandingData().Name() : winrt::hstring{};
+            TraceLoggingWrite(
+                g_hQueryExtensionProvider,
+                "AICodeResponseInputted",
+                TraceLoggingDescription("Event emitted when the user clicks on a suggestion to have it be input into their active shell"),
+                TraceLoggingWideString(lmProviderName.c_str(), "LMProviderName", "The name of the connected service provider, if present"),
+                TraceLoggingKeyword(MICROSOFT_KEYWORD_CRITICAL_DATA),
+                TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage));
+        });
+        messageParts.push_back(chatMsg);
 
         const auto brandingData = _lmProvider ? _lmProvider.BrandingData() : nullptr;
         const auto responseAttribution = response.ResponseAttribution().empty() ? _ProfileName : response.ResponseAttribution();
@@ -313,46 +300,6 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
         }
     }
 
-    // Method Description:
-    // - This event is called when the user clicks on a Chat Message. We will
-    //   dispatch the contents of the message to the app to input into the active control.
-    // Arguments:
-    // - e: an ItemClickEventArgs who's ClickedItem() will be the message that was clicked on.
-    // Return Value:
-    // - <none>
-    void ExtensionPalette::_listItemClicked(const Windows::Foundation::IInspectable& /*sender*/,
-                                            const Windows::UI::Xaml::Controls::ItemClickEventArgs& e)
-    {
-        const auto selectedSuggestionItem = e.ClickedItem();
-        const auto selectedItemAsChatMessage = selectedSuggestionItem.as<winrt::Microsoft::Terminal::Query::Extension::ChatMessage>();
-        if (selectedItemAsChatMessage.IsCode())
-        {
-            auto suggestion = winrt::to_string(selectedItemAsChatMessage.MessageContent());
-
-            // the AI sometimes sends multiline code blocks
-            // we don't want to run any of those commands when the chat item is clicked,
-            // so we replace newlines with the appropriate delimiter
-            size_t pos = 0;
-            while ((pos = suggestion.find("\n", pos)) != std::string::npos)
-            {
-                const auto delimiter = (_ActiveCommandline == cmdExe || _ActiveCommandline == cmd) ? cmdCommandDelimiter : commandDelimiter;
-                suggestion.at(pos) = delimiter;
-                pos += 1; // Move past the replaced character
-            }
-            _InputSuggestionRequestedHandlers(*this, winrt::to_hstring(suggestion));
-            _close();
-
-            const auto lmProviderName = _lmProvider ? _lmProvider.BrandingData().Name() : winrt::hstring{};
-            TraceLoggingWrite(
-                g_hQueryExtensionProvider,
-                "AICodeResponseInputted",
-                TraceLoggingDescription("Event emitted when the user clicks on a suggestion to have it be input into their active shell"),
-                TraceLoggingWideString(lmProviderName.c_str(), "LMProviderName", "The name of the connected service provider, if present"),
-                TraceLoggingKeyword(MICROSOFT_KEYWORD_CRITICAL_DATA),
-                TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage));
-        }
-    }
-
     // Method Description:
     // - This event is triggered when someone clicks anywhere in the bounds of
     //   the window that's _not_ the query palette UI. When that happens,
@@ -489,4 +436,49 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
         // Clear the text box each time we close the dialog. This is consistent with VsCode.
         _queryBox().Text(winrt::hstring{});
     }
+
+    ChatMessage::ChatMessage(winrt::hstring content, bool isQuery) :
+        _messageContent{ content },
+        _isQuery{ isQuery },
+        _richBlock{ nullptr }
+    {
+        _richBlock = Microsoft::Terminal::UI::Markdown::Builder::Convert(_messageContent, L"");
+        const auto resources = Application::Current().Resources();
+        const auto textBrushObj = _isQuery ? resources.Lookup(box_value(L"TextOnAccentFillColorPrimaryBrush")) : resources.Lookup(box_value(L"TextFillColorPrimaryBrush"));
+        if (const auto textBrush = textBrushObj.try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
+        {
+            _richBlock.Foreground(textBrush);
+        }
+        if (!_isQuery)
+        {
+            for (const auto& b : _richBlock.Blocks())
+            {
+                if (const auto& p{ b.try_as<Windows::UI::Xaml::Documents::Paragraph>() })
+                {
+                    for (const auto& line : p.Inlines())
+                    {
+                        if (const auto& otherContent{ line.try_as<Windows::UI::Xaml::Documents::InlineUIContainer>() })
+                        {
+                            if (const auto& codeBlock{ otherContent.Child().try_as<Microsoft::Terminal::UI::Markdown::CodeBlock>() })
+                            {
+                                codeBlock.Margin({ 0, 8, 0, 8 });
+                                codeBlock.PlayButtonVisibility(Windows::UI::Xaml::Visibility::Visible);
+                                if (const auto backgroundBrush = resources.Lookup(box_value(L"ControlAltFillColorSecondaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
+                                {
+                                    codeBlock.Background(backgroundBrush);
+                                }
+                                if (const auto foregroundBrush = resources.Lookup(box_value(L"AccentTextFillColorPrimaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
+                                {
+                                    codeBlock.Foreground(foregroundBrush);
+                                }
+                                codeBlock.RequestRunCommands([this, commandlines = codeBlock.Commandlines()](auto&&, auto&&) {
+                                    _RunCommandClickedHandlers(*this, commandlines);
+                                });
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
 }

src/cascadia/QueryExtension/ExtensionPalette.h

@@ -50,7 +50,6 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
 
         void _clearAndInitializeMessages(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
         void _exportMessagesToFile(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
-        void _listItemClicked(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Controls::ItemClickEventArgs& e);
         void _rootPointerPressed(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Input::PointerRoutedEventArgs& e);
         void _backdropPointerPressed(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Input::PointerRoutedEventArgs& e);
         void _lostFocusHandler(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
@@ -63,19 +62,18 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
 
     struct ChatMessage : ChatMessageT<ChatMessage>
     {
-        ChatMessage(winrt::hstring content, bool isQuery, bool isCode) :
-            _messageContent{ content },
-            _isQuery{ isQuery },
-            _isCode{ isCode } {}
+        ChatMessage(winrt::hstring content, bool isQuery);
 
         bool IsQuery() const { return _isQuery; };
-        bool IsCode() const { return _isCode; };
         winrt::hstring MessageContent() const { return _messageContent; };
+        winrt::Windows::UI::Xaml::Controls::RichTextBlock RichBlock() const { return _richBlock; };
+
+        TYPED_EVENT(RunCommandClicked, winrt::Microsoft::Terminal::Query::Extension::ChatMessage, winrt::hstring);
 
     private:
         bool _isQuery;
-        bool _isCode;
         winrt::hstring _messageContent;
+        Windows::UI::Xaml::Controls::RichTextBlock _richBlock;
     };
 
     struct GroupedChatMessages : GroupedChatMessagesT<GroupedChatMessages>

src/cascadia/QueryExtension/ExtensionPalette.idl

@@ -7,10 +7,11 @@ namespace Microsoft.Terminal.Query.Extension
 {
     [default_interface] runtimeclass ChatMessage
     {
-        ChatMessage(String content, Boolean isQuery, Boolean isCode);
+        ChatMessage(String content, Boolean isQuery);
         String MessageContent { get; };
         Boolean IsQuery { get; };
-        Boolean IsCode { get; };
+        Windows.UI.Xaml.Controls.RichTextBlock RichBlock { get; };
+        event Windows.Foundation.TypedEventHandler<ChatMessage, String> RunCommandClicked;
     }
 
     runtimeclass GroupedChatMessages : Windows.Foundation.Collections.IVector<IInspectable>

src/cascadia/QueryExtension/ExtensionPalette.xaml

@@ -46,7 +46,7 @@
             <mux:StackLayout x:Name="VerticalStackLayout"
                              Orientation="Vertical"
                              Spacing="16" />
-            <DataTemplate x:Key="QueryMessageTemplate"
+            <DataTemplate x:Key="RichQueryMessageTemplate"
                           x:DataType="local:ChatMessage">
                 <Grid Height="Auto"
                       Margin="4"
@@ -62,14 +62,11 @@
                                 BorderBrush="{ThemeResource MessageBorderBrush}"
                                 BorderThickness="{ThemeResource MessageBorderThickness}"
                                 CornerRadius="8">
-                        <TextBlock FontSize="14"
-                                   Foreground="{ThemeResource TextOnAccentFillColorPrimaryBrush}"
-                                   Text="{x:Bind MessageContent}"
-                                   TextWrapping="WrapWholeWords" />
+                        <ContentPresenter Content="{x:Bind RichBlock}" />
                     </StackPanel>
                 </Grid>
             </DataTemplate>
-            <DataTemplate x:Key="TextResponseMessageTemplate"
+            <DataTemplate x:Key="RichResponseMessageTemplate"
                           x:DataType="local:ChatMessage">
                 <Grid Height="Auto"
                       Margin="4"
@@ -85,54 +82,13 @@
                                 BorderBrush="{ThemeResource MessageBorderBrush}"
                                 BorderThickness="{ThemeResource MessageBorderThickness}"
                                 CornerRadius="8">
-                        <TextBlock FontSize="14"
-                                   Foreground="{ThemeResource TextFillColorPrimaryBrush}"
-                                   IsTextSelectionEnabled="False"
-                                   Text="{x:Bind MessageContent}"
-                                   TextWrapping="WrapWholeWords" />
-                    </StackPanel>
-                </Grid>
-            </DataTemplate>
-            <DataTemplate x:Key="CodeResponseMessageTemplate"
-                          x:DataType="local:ChatMessage">
-                <Grid Height="Auto"
-                      Margin="4"
-                      HorizontalAlignment="Left">
-                    <Grid.RowDefinitions>
-                        <RowDefinition Height="Auto" />
-                        <RowDefinition Height="Auto" />
-                    </Grid.RowDefinitions>
-                    <StackPanel Grid.Row="1"
-                                Padding="16,8,16,8"
-                                Background="{ThemeResource ControlAltFillColorQuarternaryBrush}"
-                                BorderBrush="{ThemeResource MessageBorderBrush}"
-                                BorderThickness="{ThemeResource MessageBorderThickness}"
-                                CornerRadius="8">
-                        <Border Padding="8,4,8,4"
-                                Background="{ThemeResource ControlAltFillColorSecondaryBrush}"
-                                CornerRadius="4">
-                            <StackPanel Orientation="Horizontal"
-                                        Spacing="8">
-                                <TextBlock MaxWidth="400"
-                                           FontFamily="Cascadia Code"
-                                           FontSize="12"
-                                           Foreground="{ThemeResource AccentTextFillColorPrimaryBrush}"
-                                           IsTextSelectionEnabled="False"
-                                           Text="{x:Bind MessageContent}"
-                                           TextWrapping="Wrap" />
-                                <FontIcon VerticalAlignment="Center"
-                                          FontSize="16"
-                                          Foreground="{ThemeResource TextFillColorPrimaryBrush}"
-                                          Glyph="&#xE8C8;" />
-                            </StackPanel>
-                        </Border>
+                        <ContentPresenter Content="{x:Bind RichBlock}" />
                     </StackPanel>
                 </Grid>
             </DataTemplate>
             <local:ExtensionPaletteMessageTemplateSelector x:Key="ChatMessageTemplateSelector"
-                                                           CodeResponseMessageTemplate="{StaticResource CodeResponseMessageTemplate}"
-                                                           QueryMessageTemplate="{StaticResource QueryMessageTemplate}"
-                                                           TextResponseMessageTemplate="{StaticResource TextResponseMessageTemplate}" />
+                                                           RichQueryMessageTemplate="{StaticResource RichQueryMessageTemplate}"
+                                                           RichResponseMessageTemplate="{StaticResource RichResponseMessageTemplate}" />
             <Style TargetType="ListViewHeaderItem">
                 <Setter Property="FontFamily" Value="{ThemeResource ContentControlThemeFontFamily}" />
                 <Setter Property="FontSize" Value="{ThemeResource ListViewHeaderItemThemeFontSize}" />
@@ -284,7 +240,6 @@
                               VerticalAlignment="Bottom"
                               HorizontalContentAlignment="Stretch"
                               IsItemClickEnabled="True"
-                              ItemClick="_listItemClicked"
                               ItemTemplateSelector="{StaticResource ChatMessageTemplateSelector}"
                               ItemsSource="{Binding Source={StaticResource MessagesCollectionViewSource}}"
                               SelectionMode="None">