diff --git a/ui/controls/ChatUserControl.axaml.cs b/ui/controls/ChatUserControl.axaml.cs index d125cd6..a109b7d 100644 --- a/ui/controls/ChatUserControl.axaml.cs +++ b/ui/controls/ChatUserControl.axaml.cs @@ -1,11 +1,13 @@ using Avalonia.Controls; using System; using System.ClientModel; +using System.Collections.Generic; +using System.IO; using System.Text; +using System.Threading.Tasks; using Avalonia.Input; using highminded.utils; using OpenAI.Chat; -using OpenAI; using Markdig; using Markdown.ColorCode; @@ -13,15 +15,37 @@ namespace highminded.ui.controls; public partial class ChatUserControl : UserControl { - - private readonly MarkdownPipeline _pipeline = null!; - + private readonly MarkdownPipeline _pipeline = null!; + public ChatUserControl() { InitializeComponent(); - _pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().UseColorCode().Build(); + _pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().UseColorCode().Build(); } - + + public async void SendScreenshot() + { + var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); + var fileName = $"screenshot_{timestamp}.png"; + var filePath = Path.Combine(Environment.CurrentDirectory, fileName); + + var screenshot = await ScreenCapture.CaptureScreenAsync(filePath); + if (!screenshot) return; + + using Stream imageStream = File.OpenRead(filePath); + BinaryData imageBytes = BinaryData.FromStream(imageStream); + + List messages = + [ + new UserChatMessage( + ChatMessageContentPart.CreateTextPart("I'm attaching a screenshot of a problem. I want you to read it and give me the appropriate answer."), + ChatMessageContentPart.CreateImagePart(imageBytes, "image/png") + ) + ]; + + await ProcessChatStreamAsync(messages); + } + private async void PromptBox_OnKeyDown(object? sender, KeyEventArgs e) { try @@ -32,25 +56,34 @@ public partial class ChatUserControl : UserControl if (prompt is null) return; PromptBox.Clear(); - AsyncCollectionResult completionUpdates = - InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(prompt); - - var responseBuilder = new StringBuilder(); - - await foreach (var completionUpdate in completionUpdates) - { - if (completionUpdate.ContentUpdate.Count <= 0) continue; - - var token = completionUpdate.ContentUpdate[0].Text; - responseBuilder.Append(token); - - var html = Markdig.Markdown.ToHtml(responseBuilder.ToString(), _pipeline); - ResultBlock.Text = html; - } + await ProcessChatStreamAsync(prompt); } catch (Exception err) { ResultBlock.Text = err.Message; } } + + private async Task ProcessChatStreamAsync(object promptOrMessages) + { + AsyncCollectionResult completionUpdates = promptOrMessages switch + { + string prompt => InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(prompt), + IEnumerable messages => InMemoryDb.Obj.ChatClient.CompleteChatStreamingAsync(messages), + _ => throw new ArgumentException("Invalid input type", nameof(promptOrMessages)) + }; + + var responseBuilder = new StringBuilder(); + + await foreach (var completionUpdate in completionUpdates) + { + if (completionUpdate.ContentUpdate.Count <= 0) continue; + + var token = completionUpdate.ContentUpdate[0].Text; + responseBuilder.Append(token); + + var html = Markdig.Markdown.ToHtml(responseBuilder.ToString(), _pipeline); + ResultBlock.Text = html; + } + } } \ No newline at end of file diff --git a/ui/windows/MainWindow.axaml b/ui/windows/MainWindow.axaml index a0db4eb..35e6814 100644 --- a/ui/windows/MainWindow.axaml +++ b/ui/windows/MainWindow.axaml @@ -32,6 +32,14 @@ + + + + + + diff --git a/ui/windows/MainWindow.axaml.cs b/ui/windows/MainWindow.axaml.cs index c7214cf..9a9d60f 100644 --- a/ui/windows/MainWindow.axaml.cs +++ b/ui/windows/MainWindow.axaml.cs @@ -80,12 +80,18 @@ public partial class MainWindow : Window bool hasShift = (e.RawEvent.Mask & EventMask.Shift) != EventMask.None; bool hasH = e.Data.KeyCode == KeyCode.VcH; bool hasBackslash = e.Data.KeyCode == KeyCode.VcBackslash; + bool hasS = e.Data.KeyCode == KeyCode.VcS; if (hasCtrl && hasShift && hasAlt && hasH) { ShowOverlay(); } + if (hasShift && hasS) + { + Dispatcher.UIThread.Post(() => { _chatUserControl.SendScreenshot(); }); + } + if (hasShift && hasBackslash) { Dispatcher.UIThread.Post(() => diff --git a/utils/ScreenCapture.cs b/utils/ScreenCapture.cs new file mode 100644 index 0000000..0bac085 --- /dev/null +++ b/utils/ScreenCapture.cs @@ -0,0 +1,91 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading.Tasks; + +namespace highminded.utils; + +public static class ScreenCapture +{ + public static async Task CaptureScreenAsync(string filePath) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return await CaptureWindowsAsync(filePath); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return await CaptureMacAsync(filePath); + } + + return false; + } + catch + { + return false; + } + } + + private static async Task CaptureWindowsAsync(string filePath) + { + var script = $$""" + Add-Type -AssemblyName System.Windows.Forms + Add-Type -AssemblyName System.Drawing + + $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds + $bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + + try { + $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) + $bitmap.Save('{{filePath.Replace("\\", @"\\")}}', [System.Drawing.Imaging.ImageFormat]::Png) + Write-Output 'Success' + } catch { + Write-Output 'Error' + } finally { + $graphics.Dispose() + $bitmap.Dispose() + } + """; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "powershell.exe", + Arguments = $"-Command \"{script}\"", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true + } + }; + + process.Start(); + var output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + + return output.Trim() == "Success" && File.Exists(filePath); + } + + private static async Task CaptureMacAsync(string filePath) + { + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "screencapture", + Arguments = $"-x \"{filePath}\"", + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + await process.WaitForExitAsync(); + + return process.ExitCode == 0 && File.Exists(filePath); + } +} \ No newline at end of file