Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve parsing of responses from the AI #18220

Open
wants to merge 11 commits into
base: feature/llm
Choose a base branch
from
+100 −157
Open

Improve parsing of responses from the AI #18220

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1c834bf
just in case
PankajBhojwani Nov 14, 2024
e686b57
Merge branch 'feature/llm' of https://github.com/microsoft/terminal i…
PankajBhojwani Nov 18, 2024
d81fd81
brush things
PankajBhojwani Nov 19, 2024
be40fb2
click the play button works
PankajBhojwani Nov 19, 2024
100fcda
neaten
PankajBhojwani Nov 19, 2024
0be1e43
remove cmark
PankajBhojwani Nov 19, 2024
ef9e69e
format
PankajBhojwani Nov 19, 2024
146d027
don't need these anymore
PankajBhojwani Nov 19, 2024
527a01a
wait we still need the multiline check
PankajBhojwani Nov 20, 2024
3fb5cbf
nits
PankajBhojwani Nov 20, 2024
c77e026
slightly cleaner
PankajBhojwani Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions OpenConsole.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2334,10 +2334,14 @@ Global
{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x64.Build.0 = Release|x64
{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x86.ActiveCfg = Release|Win32
{6515F03F-E56D-4DB4-B23D-AC4FB80DB36F}.Release|x86.Build.0 = Release|Win32
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.ActiveCfg = Debug|Win32
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.ActiveCfg = Release|ARM64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.ActiveCfg = Release|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.ActiveCfg = Release|Win32
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.ActiveCfg = AuditMode|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|Any CPU.Build.0 = AuditMode|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.ActiveCfg = AuditMode|ARM64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|ARM64.Build.0 = AuditMode|ARM64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.ActiveCfg = AuditMode|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x64.Build.0 = AuditMode|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.ActiveCfg = AuditMode|Win32
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.AuditMode|x86.Build.0 = AuditMode|Win32
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|Any CPU.ActiveCfg = Debug|x64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|ARM64.ActiveCfg = Debug|ARM64
{7615F03F-E56D-4DB4-B23D-BD4FB80DB36F}.Debug|ARM64.Build.0 = Debug|ARM64
Expand Down
153 changes: 75 additions & 78 deletions src/cascadia/QueryExtension/ExtensionPalette.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation

winrt::fire_and_forget ExtensionPalette::_getSuggestions(const winrt::hstring& prompt, const winrt::hstring& currentLocalTime)
{
const auto userMessage = winrt::make<ChatMessage>(prompt, true, false);
const auto userMessage = winrt::make<ChatMessage>(prompt, true);
std::vector<IInspectable> userMessageVector{ userMessage };
const auto queryAttribution = _lmProvider ? _lmProvider.BrandingData().QueryAttribution() : winrt::hstring{};
const auto userGroupedMessages = winrt::make<GroupedChatMessages>(currentLocalTime, true, winrt::single_threaded_vector(std::move(userMessageVector)), queryAttribution);
Expand Down Expand Up @@ -197,48 +197,35 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation

void ExtensionPalette::_splitResponseAndAddToChatHelper(const IResponse response)
{
// this function is dependent on the AI response separating code blocks with
// newlines and "```". OpenAI seems to naturally conform to this, though
// we could probably engineer the prompt to specify this if we need to.
std::wstringstream ss(response.Message().c_str());
std::wstring line;
std::wstring codeBlock;
bool inCodeBlock = false;
const auto time = _getCurrentLocalTimeHelper();
std::vector<IInspectable> messageParts;

while (std::getline(ss, line))
{
if (!line.empty())
const auto chatMsg = winrt::make<ChatMessage>(winrt::to_hstring(response.Message()), false);
PankajBhojwani marked this conversation as resolved.
Show resolved Hide resolved
chatMsg.RunCommandClicked([this](auto&&, const auto commandlines) {
auto suggestion = winrt::to_string(commandlines);
// the AI sometimes sends multiline code blocks
// we don't want to run any of those commands when the chat item is clicked,
// so we replace newlines with the appropriate delimiter
PankajBhojwani marked this conversation as resolved.
Show resolved Hide resolved
size_t pos = 0;
while ((pos = suggestion.find("\n", pos)) != std::string::npos)
{
if (!inCodeBlock && line.find(L"```") == 0)
{
inCodeBlock = true;
continue;
}
if (inCodeBlock && line.find(L"```") == 0)
{
inCodeBlock = false;
const auto chatMsg = winrt::make<ChatMessage>(winrt::hstring{ std::move(codeBlock) }, false, true);
messageParts.push_back(chatMsg);
codeBlock.clear();
continue;
}
if (inCodeBlock)
{
if (!codeBlock.empty())
{
codeBlock += L'\n';
}
codeBlock += line;
}
else
{
const auto chatMsg = winrt::make<ChatMessage>(winrt::hstring{ line }, false, false);
messageParts.push_back(chatMsg);
}
const auto delimiter = (_ActiveCommandline == cmdExe || _ActiveCommandline == cmd) ? cmdCommandDelimiter : commandDelimiter;
suggestion.at(pos) = delimiter;
pos += 1; // Move past the replaced character
}
}
_InputSuggestionRequestedHandlers(*this, winrt::to_hstring(suggestion));
_close();

const auto lmProviderName = _lmProvider ? _lmProvider.BrandingData().Name() : winrt::hstring{};
TraceLoggingWrite(
g_hQueryExtensionProvider,
"AICodeResponseInputted",
TraceLoggingDescription("Event emitted when the user clicks on a suggestion to have it be input into their active shell"),
TraceLoggingWideString(lmProviderName.c_str(), "LMProviderName", "The name of the connected service provider, if present"),
TraceLoggingKeyword(MICROSOFT_KEYWORD_CRITICAL_DATA),
TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage));
});
messageParts.push_back(chatMsg);

const auto brandingData = _lmProvider ? _lmProvider.BrandingData() : nullptr;
const auto responseAttribution = response.ResponseAttribution().empty() ? _ProfileName : response.ResponseAttribution();
Expand Down Expand Up @@ -313,46 +300,6 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
}
}

// Method Description:
// - This event is called when the user clicks on a Chat Message. We will
// dispatch the contents of the message to the app to input into the active control.
// Arguments:
// - e: an ItemClickEventArgs who's ClickedItem() will be the message that was clicked on.
// Return Value:
// - <none>
void ExtensionPalette::_listItemClicked(const Windows::Foundation::IInspectable& /*sender*/,
const Windows::UI::Xaml::Controls::ItemClickEventArgs& e)
{
const auto selectedSuggestionItem = e.ClickedItem();
const auto selectedItemAsChatMessage = selectedSuggestionItem.as<winrt::Microsoft::Terminal::Query::Extension::ChatMessage>();
if (selectedItemAsChatMessage.IsCode())
{
auto suggestion = winrt::to_string(selectedItemAsChatMessage.MessageContent());

// the AI sometimes sends multiline code blocks
// we don't want to run any of those commands when the chat item is clicked,
// so we replace newlines with the appropriate delimiter
size_t pos = 0;
while ((pos = suggestion.find("\n", pos)) != std::string::npos)
{
const auto delimiter = (_ActiveCommandline == cmdExe || _ActiveCommandline == cmd) ? cmdCommandDelimiter : commandDelimiter;
suggestion.at(pos) = delimiter;
pos += 1; // Move past the replaced character
}
_InputSuggestionRequestedHandlers(*this, winrt::to_hstring(suggestion));
_close();

const auto lmProviderName = _lmProvider ? _lmProvider.BrandingData().Name() : winrt::hstring{};
TraceLoggingWrite(
g_hQueryExtensionProvider,
"AICodeResponseInputted",
TraceLoggingDescription("Event emitted when the user clicks on a suggestion to have it be input into their active shell"),
TraceLoggingWideString(lmProviderName.c_str(), "LMProviderName", "The name of the connected service provider, if present"),
TraceLoggingKeyword(MICROSOFT_KEYWORD_CRITICAL_DATA),
TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage));
}
}

// Method Description:
// - This event is triggered when someone clicks anywhere in the bounds of
// the window that's _not_ the query palette UI. When that happens,
Expand Down Expand Up @@ -489,4 +436,54 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation
// Clear the text box each time we close the dialog. This is consistent with VsCode.
_queryBox().Text(winrt::hstring{});
}

ChatMessage::ChatMessage(winrt::hstring content, bool isQuery) :
_messageContent{ content },
_isQuery{ isQuery }
{
_richBlock = Microsoft::Terminal::UI::Markdown::Builder::Convert(_messageContent, L"");
const auto resources = Application::Current().Resources();
if (_isQuery)
{
if (const auto textBrush = resources.Lookup(box_value(L"TextOnAccentFillColorPrimaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
{
_richBlock.Foreground(textBrush);
}
}
else
{
for (const auto& b : _richBlock.Blocks())
{
if (const auto& p{ b.try_as<Windows::UI::Xaml::Documents::Paragraph>() })
{
for (const auto& line : p.Inlines())
{
if (const auto& otherContent{ line.try_as<Windows::UI::Xaml::Documents::InlineUIContainer>() })
{
if (const auto& codeBlock{ otherContent.Child().try_as<Microsoft::Terminal::UI::Markdown::CodeBlock>() })
{
codeBlock.Margin({ 0, 8, 0, 8 });
codeBlock.PlayButtonVisibility(Windows::UI::Xaml::Visibility::Visible);
if (const auto backgroundBrush = resources.Lookup(box_value(L"ControlAltFillColorSecondaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
PankajBhojwani marked this conversation as resolved.
Show resolved Hide resolved
{
codeBlock.Background(backgroundBrush);
}
if (const auto foregroundBrush = resources.Lookup(box_value(L"AccentTextFillColorPrimaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
{
codeBlock.Foreground(foregroundBrush);
}
codeBlock.RequestRunCommands([this, commandlines = codeBlock.Commandlines()](auto&&, auto&&) {
_RunCommandClickedHandlers(*this, commandlines);
});
}
}
}
}
}
if (const auto textBrush = resources.Lookup(box_value(L"TextFillColorPrimaryBrush")).try_as<Windows::UI::Xaml::Media::SolidColorBrush>())
{
_richBlock.Foreground(textBrush);
}
}
}
}
12 changes: 5 additions & 7 deletions src/cascadia/QueryExtension/ExtensionPalette.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation

void _clearAndInitializeMessages(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
void _exportMessagesToFile(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
void _listItemClicked(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Controls::ItemClickEventArgs& e);
void _rootPointerPressed(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Input::PointerRoutedEventArgs& e);
void _backdropPointerPressed(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::Input::PointerRoutedEventArgs& e);
void _lostFocusHandler(const Windows::Foundation::IInspectable& sender, const Windows::UI::Xaml::RoutedEventArgs& args);
Expand All @@ -63,19 +62,18 @@ namespace winrt::Microsoft::Terminal::Query::Extension::implementation

struct ChatMessage : ChatMessageT<ChatMessage>
{
ChatMessage(winrt::hstring content, bool isQuery, bool isCode) :
_messageContent{ content },
_isQuery{ isQuery },
_isCode{ isCode } {}
ChatMessage(winrt::hstring content, bool isQuery);

bool IsQuery() const { return _isQuery; };
bool IsCode() const { return _isCode; };
winrt::hstring MessageContent() const { return _messageContent; };
winrt::Windows::UI::Xaml::Controls::RichTextBlock RichBlock() const { return _richBlock; };

TYPED_EVENT(RunCommandClicked, winrt::Microsoft::Terminal::Query::Extension::ChatMessage, winrt::hstring);

private:
bool _isQuery;
bool _isCode;
winrt::hstring _messageContent;
Windows::UI::Xaml::Controls::RichTextBlock _richBlock;
};

struct GroupedChatMessages : GroupedChatMessagesT<GroupedChatMessages>
Expand Down
5 changes: 3 additions & 2 deletions src/cascadia/QueryExtension/ExtensionPalette.idl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ namespace Microsoft.Terminal.Query.Extension
{
[default_interface] runtimeclass ChatMessage
{
ChatMessage(String content, Boolean isQuery, Boolean isCode);
ChatMessage(String content, Boolean isQuery);
String MessageContent { get; };
Boolean IsQuery { get; };
Boolean IsCode { get; };
Windows.UI.Xaml.Controls.RichTextBlock RichBlock { get; };
event Windows.Foundation.TypedEventHandler<ChatMessage, String> RunCommandClicked;
}

runtimeclass GroupedChatMessages : Windows.Foundation.Collections.IVector<IInspectable>
Expand Down
57 changes: 6 additions & 51 deletions src/cascadia/QueryExtension/ExtensionPalette.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
<mux:StackLayout x:Name="VerticalStackLayout"
Orientation="Vertical"
Spacing="16" />
<DataTemplate x:Key="QueryMessageTemplate"
<DataTemplate x:Key="RichQueryMessageTemplate"
x:DataType="local:ChatMessage">
<Grid Height="Auto"
Margin="4"
Expand All @@ -62,14 +62,11 @@
BorderBrush="{ThemeResource MessageBorderBrush}"
BorderThickness="{ThemeResource MessageBorderThickness}"
CornerRadius="8">
<TextBlock FontSize="14"
Foreground="{ThemeResource TextOnAccentFillColorPrimaryBrush}"
Text="{x:Bind MessageContent}"
TextWrapping="WrapWholeWords" />
<ContentPresenter Content="{x:Bind RichBlock}" />
</StackPanel>
</Grid>
</DataTemplate>
<DataTemplate x:Key="TextResponseMessageTemplate"
<DataTemplate x:Key="RichResponseMessageTemplate"
x:DataType="local:ChatMessage">
<Grid Height="Auto"
Margin="4"
Expand All @@ -85,54 +82,13 @@
BorderBrush="{ThemeResource MessageBorderBrush}"
BorderThickness="{ThemeResource MessageBorderThickness}"
CornerRadius="8">
<TextBlock FontSize="14"
Foreground="{ThemeResource TextFillColorPrimaryBrush}"
IsTextSelectionEnabled="False"
Text="{x:Bind MessageContent}"
TextWrapping="WrapWholeWords" />
</StackPanel>
</Grid>
</DataTemplate>
<DataTemplate x:Key="CodeResponseMessageTemplate"
x:DataType="local:ChatMessage">
<Grid Height="Auto"
Margin="4"
HorizontalAlignment="Left">
<Grid.RowDefinitions>
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
</Grid.RowDefinitions>
<StackPanel Grid.Row="1"
Padding="16,8,16,8"
Background="{ThemeResource ControlAltFillColorQuarternaryBrush}"
BorderBrush="{ThemeResource MessageBorderBrush}"
BorderThickness="{ThemeResource MessageBorderThickness}"
CornerRadius="8">
<Border Padding="8,4,8,4"
Background="{ThemeResource ControlAltFillColorSecondaryBrush}"
CornerRadius="4">
<StackPanel Orientation="Horizontal"
Spacing="8">
<TextBlock MaxWidth="400"
FontFamily="Cascadia Code"
FontSize="12"
Foreground="{ThemeResource AccentTextFillColorPrimaryBrush}"
IsTextSelectionEnabled="False"
Text="{x:Bind MessageContent}"
TextWrapping="Wrap" />
<FontIcon VerticalAlignment="Center"
FontSize="16"
Foreground="{ThemeResource TextFillColorPrimaryBrush}"
Glyph="&#xE8C8;" />
</StackPanel>
</Border>
<ContentPresenter Content="{x:Bind RichBlock}" />
</StackPanel>
</Grid>
</DataTemplate>
<local:ExtensionPaletteMessageTemplateSelector x:Key="ChatMessageTemplateSelector"
CodeResponseMessageTemplate="{StaticResource CodeResponseMessageTemplate}"
QueryMessageTemplate="{StaticResource QueryMessageTemplate}"
TextResponseMessageTemplate="{StaticResource TextResponseMessageTemplate}" />
RichQueryMessageTemplate="{StaticResource RichQueryMessageTemplate}"
RichResponseMessageTemplate="{StaticResource RichResponseMessageTemplate}" />
<Style TargetType="ListViewHeaderItem">
<Setter Property="FontFamily" Value="{ThemeResource ContentControlThemeFontFamily}" />
<Setter Property="FontSize" Value="{ThemeResource ListViewHeaderItemThemeFontSize}" />
Expand Down Expand Up @@ -284,7 +240,6 @@
VerticalAlignment="Bottom"
HorizontalContentAlignment="Stretch"
IsItemClickEnabled="True"
ItemClick="_listItemClicked"
ItemTemplateSelector="{StaticResource ChatMessageTemplateSelector}"
ItemsSource="{Binding Source={StaticResource MessagesCollectionViewSource}}"
SelectionMode="None">
Expand Down
Loading
Loading