feat: screen aware answers

ui/controls/ChatUserControl.axaml.cs

@@ -1,11 +1,13 @@
 using Avalonia.Controls;
 using System;
 using System.ClientModel;
+using System.Collections.Generic;
+using System.IO;
 using System.Text;
+using System.Threading.Tasks;
 using Avalonia.Input;
 using highminded.utils;
 using OpenAI.Chat;
-using OpenAI;
 using Markdig;
 using Markdown.ColorCode;
 
@@ -13,15 +15,37 @@ namespace highminded.ui.controls;
 
 public partial class ChatUserControl : UserControl
 {
-    
-    private readonly MarkdownPipeline _pipeline = null!; 
-    
+    private readonly MarkdownPipeline _pipeline = null!;
+
     public ChatUserControl()
     {
         InitializeComponent();
-        _pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().UseColorCode().Build(); 
+        _pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().UseColorCode().Build();
     }
-    
+
+    public async void SendScreenshot()
+    {
+        var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
+        var fileName = $"screenshot_{timestamp}.png";
+        var filePath = Path.Combine(Environment.CurrentDirectory, fileName);
+
+        var screenshot = await ScreenCapture.CaptureScreenAsync(filePath);
+        if (!screenshot) return;
+
+        using Stream imageStream = File.OpenRead(filePath);
+        BinaryData imageBytes = BinaryData.FromStream(imageStream);
+
+        List<ChatMessage> messages =
+        [
+            new UserChatMessage(
+                ChatMessageContentPart.CreateTextPart("I'm attaching a screenshot of a problem. I want you to read it and give me the appropriate answer."),
+                ChatMessageContentPart.CreateImagePart(imageBytes, "image/png")
+            )
+        ];
+
+        await ProcessChatStreamAsync(messages);
+    }
+
     private async void PromptBox_OnKeyDown(object? sender, KeyEventArgs e)
     {
         try
@@ -32,25 +56,34 @@ public partial class ChatUserControl : UserControl
             if (prompt is null) return;
             PromptBox.Clear();
 
-            AsyncCollectionResult<StreamingChatCompletionUpdate> completionUpdates =
-                InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(prompt);
-
-            var responseBuilder = new StringBuilder();
-
-            await foreach (var completionUpdate in completionUpdates)
-            {
-                if (completionUpdate.ContentUpdate.Count <= 0) continue;
-
-                var token = completionUpdate.ContentUpdate[0].Text;
-                responseBuilder.Append(token);
-
-                var html = Markdig.Markdown.ToHtml(responseBuilder.ToString(), _pipeline);
-                ResultBlock.Text = html;
-            }
+            await ProcessChatStreamAsync(prompt);
         }
         catch (Exception err)
         {
             ResultBlock.Text = err.Message;
         }
     }
+
+    private async Task ProcessChatStreamAsync(object promptOrMessages)
+    {
+        AsyncCollectionResult<StreamingChatCompletionUpdate> completionUpdates = promptOrMessages switch
+        {
+            string prompt => InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(prompt),
+            IEnumerable<ChatMessage> messages => InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(messages),
+            _ => throw new ArgumentException("Invalid input type", nameof(promptOrMessages))
+        };
+
+        var responseBuilder = new StringBuilder();
+
+        await foreach (var completionUpdate in completionUpdates)
+        {
+            if (completionUpdate.ContentUpdate.Count <= 0) continue;
+
+            var token = completionUpdate.ContentUpdate[0].Text;
+            responseBuilder.Append(token);
+
+            var html = Markdig.Markdown.ToHtml(responseBuilder.ToString(), _pipeline);
+            ResultBlock.Text = html;
+        }
+    }
 }

ui/windows/MainWindow.axaml

@@ -32,6 +32,14 @@
                             <TextBlock FontSize="12" Text="SHIFT + \" />
                         </StackPanel>
                     </Border>
+                    <Border Background="Black" BorderBrush="#19ffffff" BorderThickness="2" CornerRadius="5"
+                            Margin="0 0 10 0">
+                        <StackPanel Orientation="Horizontal" HorizontalAlignment="Center" VerticalAlignment="Center"
+                                    Spacing="5" Margin="10 0">
+                            <TextBlock FontSize="12" Text="Screen:" />
+                            <TextBlock FontSize="12" Text="SHIFT + S" />
+                        </StackPanel>
+                    </Border>
                     <Button Name="ChatBtn" Background="Transparent" Click="ChatBtnClick">
                         <iconPacks:PackIconLucide Grid.Column="1" Kind="Brain" Height="16" Width="16" />
                     </Button>

ui/windows/MainWindow.axaml.cs

@@ -80,12 +80,18 @@ public partial class MainWindow : Window
         bool hasShift = (e.RawEvent.Mask & EventMask.Shift) != EventMask.None;
         bool hasH = e.Data.KeyCode == KeyCode.VcH;
         bool hasBackslash = e.Data.KeyCode == KeyCode.VcBackslash;
+        bool hasS = e.Data.KeyCode == KeyCode.VcS;
 
         if (hasCtrl && hasShift && hasAlt && hasH)
         {
             ShowOverlay();
         }
 
+        if (hasShift && hasS)
+        {
+            Dispatcher.UIThread.Post(() => { _chatUserControl.SendScreenshot(); });
+        }
+
         if (hasShift && hasBackslash)
         {
             Dispatcher.UIThread.Post(() =>

utils/ScreenCapture.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace highminded.utils;
+
+public static class ScreenCapture
+{
+    public static async Task<bool> CaptureScreenAsync(string filePath)
+    {
+        try
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                return await CaptureWindowsAsync(filePath);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                return await CaptureMacAsync(filePath);
+            }
+
+            return false;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    private static async Task<bool> CaptureWindowsAsync(string filePath)
+    {
+        var script = $$"""
+                       Add-Type -AssemblyName System.Windows.Forms
+                       Add-Type -AssemblyName System.Drawing
+
+                       $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+                       $bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
+                       $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+
+                       try {
+                            $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
+                            $bitmap.Save('{{filePath.Replace("\\", @"\\")}}', [System.Drawing.Imaging.ImageFormat]::Png)
+                            Write-Output 'Success'
+                       } catch {
+                            Write-Output 'Error'
+                       } finally {
+                            $graphics.Dispose()
+                            $bitmap.Dispose()
+                       }
+                       """;
+
+        var process = new Process
+        {
+            StartInfo = new ProcessStartInfo
+            {
+                FileName = "powershell.exe",
+                Arguments = $"-Command \"{script}\"",
+                UseShellExecute = false,
+                CreateNoWindow = true,
+                RedirectStandardOutput = true
+            }
+        };
+
+        process.Start();
+        var output = await process.StandardOutput.ReadToEndAsync();
+        await process.WaitForExitAsync();
+
+        return output.Trim() == "Success" && File.Exists(filePath);
+    }
+
+    private static async Task<bool> CaptureMacAsync(string filePath)
+    {
+        var process = new Process
+        {
+            StartInfo = new ProcessStartInfo
+            {
+                FileName = "screencapture",
+                Arguments = $"-x \"{filePath}\"",
+                UseShellExecute = false,
+                CreateNoWindow = true
+            }
+        };
+
+        process.Start();
+        await process.WaitForExitAsync();
+
+        return process.ExitCode == 0 && File.Exists(filePath);
+    }
+}