XFE Git
XFE Studio Git
Git 首页 全局搜索
XFE 主站 文档 NuGet

XFEWorkflow

【WinUI】XFE基于windows的工作流

公开
关注 0 Fork 0 Star 0
UTF-8
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using Microsoft.UI.Xaml.Automation;
using XFEWorkflow.Abstractions;
using XFEWorkflow.Controls;
using XFEWorkflow.Infrastructure.Storage;
using XFEWorkflow.Views;

namespace XFEWorkflow.Services;

internal static partial class UiSmokeTest
{
    private static async Task SmokeAgentBrowserAsync(string output)
    {
        output = Path.Combine(output, "agent-browser"); Directory.CreateDirectory(output);
        Task Phase(string value) => File.WriteAllTextAsync(Path.Combine(output, "phase.txt"), value);
        var services = AppServices.Current; var shell = AppShellPage.Current!;
        var frame = (Frame)shell.FindName("navigationFrame");
        var root = (FrameworkElement)App.MainWindow.Content;
        var oldTheme = root.RequestedTheme; var oldSize = App.MainWindow.AppWindow.Size;
        const string pluginId = "xfe.agent-browser";
        var oldSettings = services.State.PluginSettings(pluginId);
        var uploadFile = Path.Combine(output, "upload-fixture.txt"); await File.WriteAllTextAsync(uploadFile, "真实上传文件:中文 😀");
        var portSource = new TcpListener(IPAddress.Loopback, 0); portSource.Start(); var port = ((IPEndPoint)portSource.LocalEndpoint).Port; portSource.Stop();
        var url = $"http://127.0.0.1:{port}/";
        using var listener = new HttpListener(); listener.Prefixes.Add(url); listener.Start();
        var requests = new List<JsonObject>(); var serverErrors = new List<string>();
        var server = Task.Run(async () =>
        {
            try
            {
                while (listener.IsListening)
                {
                    var request = await listener.GetContextAsync();
                    try
                    {
                        string response; var type = "text/html; charset=utf-8";
                        if (request.Request.Url!.AbsolutePath == "/chat/completions")
                        {
                            using var reader = new StreamReader(request.Request.InputStream);
                            var input = JsonNode.Parse(await reader.ReadToEndAsync())!.AsObject();
                            lock (requests) requests.Add(input);
                            var messages = input["messages"]!.AsArray();
                            var cancelling = messages.OfType<JsonObject>().Any(x => x.String("role") == "user" && x["content"] is JsonValue && x.String("content") == "测试停止");
                            var tools = messages.OfType<JsonObject>().Count(x => x.String("role") == "tool");
                            var delta = new JsonObject(); var finish = "stop";
                            if (!cancelling && tools >= 3)
                                delta["content"] = "## 操作已完成\n\n**测试联系人 😀** 已填写并点击。\n\n| 验证 | 结果 |\n|---|---|\n| 页面 | 已更新 |\n| iframe | 已读取 |\n\n- [x] 观察页面\n- [x] 操作并截图\n\n> 仅使用本地测试网页。\n\n" + new string('`', 3) + "csharp\nConsole.WriteLine(\"中文 😀\");\n" + new string('`', 3) + "\n<script>window.agentInjected=true</script><img src=\"https://invalid.example/tracker\" onerror=\"window.agentInjected=true\">";
                            else
                            {
                                var name = cancelling || tools == 1 ? "browser_execute" : tools == 0 ? "read_skill" : "browser_screenshot";
                                JsonObject args = cancelling ? new() { ["code"] = "await new Promise(resolve => setTimeout(resolve, 60000)); return 'never';" }
                                    : tools == 0 ? new() { ["path"] = "references/browser-actions.md" }
                                    : tools == 1 ? new() { ["code"] = "await page.locator('#name').fill('测试联系人 😀'); await page.locator('#apply').click(); await page.locator('#upload').setInputFiles(" + JsonSerializer.Serialize(uploadFile) + "); const pendingDownload = page.waitForEvent('download'); await page.locator('#download').click(); const download = await pendingDownload; await download.saveAs(outputDir + '/test-download.txt'); return { result: await page.locator('#result').innerText(), iframe: await page.frames()[1].locator('body').innerText(), uploaded: await page.locator('#upload').evaluate(e => e.files[0].name), download: download.suggestedFilename(), url: page.url() };" }
                                    : new() { ["fullPage"] = false };
                                delta["tool_calls"] = new JsonArray(new JsonObject { ["index"] = 0, ["id"] = "ui-tool-" + tools, ["type"] = "function",
                                    ["function"] = new JsonObject { ["name"] = name, ["arguments"] = args.ToJsonString() } });
                                finish = "tool_calls";
                            }
                            type = "text/event-stream";
                            string Chunk(JsonObject value, string? reason) => "data: " + new JsonObject { ["choices"] = new JsonArray(new JsonObject { ["index"] = 0, ["delta"] = value, ["finish_reason"] = reason }) }.ToJsonString() + "\n\n";
                            response = Chunk(delta, null) + Chunk(new(), finish) + "data: [DONE]\n\n";
                        }
                        else if (request.Request.Url.AbsolutePath == "/download")
                        {
                            type = "application/octet-stream"; request.Response.Headers["Content-Disposition"] = "attachment; filename=agent-download.txt";
                            response = "真实下载文件:中文 😀";
                        }
                        else if (request.Request.Url.AbsolutePath == "/frame") response = "<!doctype html><html lang=zh-CN><body>iframe 真实内容 😀</body></html>";
                        else response = """
                            <!doctype html><html lang=zh-CN><meta charset=utf-8><title>Agent 本地验收</title>
                            <style>body{font:18px 'Segoe UI';padding:50px;background:#f6f8fc;color:#243041}input,button{font:inherit;padding:12px;border:1px solid #b3bfce;border-radius:8px}button{background:#2364c5;color:white}iframe{display:block;margin-top:30px;border:1px solid #aab8cc;border-radius:8px}</style>
                            <h1>WebView2 · Agent 验收</h1><p>本地网页,验证填写、点击、iframe 与截图。</p>
                            <label for=name>联系人</label><input id=name><button id=apply onclick="document.querySelector('#result').textContent='欢迎,'+document.querySelector('#name').value">确认</button>
                            <p id=result>尚未填写</p><iframe src=/frame title=测试框架></iframe>
                            <p><input id=upload type=file><a id=download href=/download>下载测试文件</a></p>
                            </html>
                            """;
                        var bytes = Encoding.UTF8.GetBytes(response); request.Response.ContentType = type; request.Response.ContentLength64 = bytes.Length;
                        await request.Response.OutputStream.WriteAsync(bytes); request.Response.Close();
                    }
                    catch (Exception ex) { lock (serverErrors) serverErrors.Add(ex.ToString()); try { request.Response.Abort(); } catch { } }
                }
            }
            catch (HttpListenerException) { } catch (ObjectDisposedException) { }
        });
        T Control<T>(string id) where T : FrameworkElement => Descendants<T>(frame).Single(x => AutomationProperties.GetAutomationId(x) == id);
        async Task Until(Func<Task<bool>> condition, int seconds = 30)
        {
            var limit = DateTime.UtcNow.AddSeconds(seconds);
            while (DateTime.UtcNow < limit) { if (await condition()) return; await Task.Delay(100); }
            throw new TimeoutException("Agent 浏览器未达到期望状态:" + string.Join(" / ", Descendants<InfoBar>(frame).Where(x => x.IsOpen).Select(x => x.Message)));
        }
        async Task Viewport(WebView2 web, string file)
        {
            var value = JsonNode.Parse(await web.CoreWebView2.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", "{\"format\":\"png\",\"fromSurface\":true,\"captureBeyondViewport\":false}"))!;
            await File.WriteAllBytesAsync(file, Convert.FromBase64String(value["data"]!.GetValue<string>()));
        }
        async Task Composite(string theme, WebView2 web, WebView2 chat)
        {
            // RTB omits WebView2's native surfaces. Combine only this app's own native and CDP renders.
            var native = Path.Combine(output, "page-" + theme + ".png");
            await CaptureWindowAsync(native);
            var browserFile = Path.Combine(output, "browser-" + theme + ".png"); var chatFile = Path.Combine(output, "chat-" + theme + ".png");
            await Viewport(web, browserFile); await Viewport(chat, chatFile);
            using var bitmap = new System.Drawing.Bitmap(native);
            using var graphics = System.Drawing.Graphics.FromImage(bitmap);
            var scale = bitmap.Width / root.ActualWidth;
            foreach (var (view, file) in new[] { (web, browserFile), (chat, chatFile) })
            {
                var position = view.TransformToVisual(root).TransformPoint(new());
                using var pixels = System.Drawing.Image.FromFile(file);
                graphics.DrawImage(pixels, new System.Drawing.RectangleF((float)(position.X * scale), (float)(position.Y * scale), (float)(view.ActualWidth * scale), (float)(view.ActualHeight * scale)));
            }
            bitmap.Save(Path.Combine(output, "page-" + theme + "-composited.png"), System.Drawing.Imaging.ImageFormat.Png);
        }
        try
        {
            var plugin = services.Plugins.Installed.Single(x => x.Manifest.Id == pluginId && x.Enabled);
            services.State.SetPluginSettings(pluginId, new() { ["endpoint"] = url, ["model"] = "deepseek-v4-flash", ["credential"] = "", ["maximumSteps"] = 12 });
            await services.State.SaveAsync();
            var scale = root.XamlRoot.RasterizationScale; App.MainWindow.AppWindow.Resize(new((int)(1500 * scale), (int)(950 * scale)));
            root.RequestedTheme = ElementTheme.Dark;
            shell.ShowBrowser(false);
            var pageRef = new PluginPageReference(pluginId, plugin.Manifest.Version, pluginId + ".page");
            await Phase("navigate");
            shell.ViewModel.NavigationViewService.NavigateTo<PluginPageHostPage>(pageRef);
            await Until(() => Task.FromResult(Descendants<WebView2>(frame).Count(x => x.CoreWebView2 is not null) >= 2));
            var web = Control<WebView2>("AgentBrowserWeb"); var chat = Control<WebView2>("AgentChatWeb");
            await Until(async () => await chat.ExecuteScriptAsync("Boolean(window.agentChatReady)") == "true");
            await Phase("webviews-ready");
            web.Source = new Uri(url);
            await Until(async () => JsonNode.Parse(await web.ExecuteScriptAsync("document.title"))?.GetValue<string>() == "Agent 本地验收");
            if (web.ActualWidth < 400 || chat.ActualHeight < 300) throw new InvalidOperationException("浏览器或聊天视图没有填满布局");
            var input = Control<TextBox>("AgentComposer"); var send = Control<Button>("AgentSend"); var stop = Control<Button>("AgentStop");
            input.Text = "填写联系人并截图"; await Task.Delay(80); await InvokeButtonAsync(send);
            await Phase("agent-sent");
            await Until(() => Task.FromResult(!send.IsEnabled));
            await Until(() => Task.FromResult(send.IsEnabled), 40);
            if (Descendants<InfoBar>(frame).Any(x => x.IsOpen)) throw new InvalidOperationException(string.Join(" / ", Descendants<InfoBar>(frame).Select(x => x.Message)));
            await Phase("agent-complete");
            var actual = JsonNode.Parse(await web.ExecuteScriptAsync("document.querySelector('#result').textContent"))!.GetValue<string>();
            if (actual != "欢迎,测试联系人 😀") throw new InvalidOperationException("Agent 没有真正填写并点击:" + actual);
            var downloaded = Directory.GetFiles(Path.Combine(services.Paths.Root, "PluginData", pluginId, "Conversations"), "test-download.txt", SearchOption.AllDirectories).Single();
            if (await File.ReadAllTextAsync(downloaded) != "真实下载文件:中文 😀") throw new InvalidOperationException("Agent 下载文件内容不正确");
            var render = JsonNode.Parse(await chat.ExecuteScriptAsync("JSON.stringify({table:document.querySelectorAll('.markdown table').length,code:document.querySelectorAll('.markdown pre code').length,task:document.querySelectorAll('.markdown input[type=checkbox]').length,tools:document.querySelectorAll('details.tool').length,images:document.querySelectorAll('img.screenshot').length,injected:!!window.agentInjected,remote:document.querySelectorAll('img[src^=http]').length,theme:document.documentElement.dataset.theme})"))!.GetValue<string>();
            var rendered = JsonNode.Parse(render)!.AsObject();
            if (rendered.Integer("table") != 1 || rendered.Integer("code") != 1 || rendered.Integer("task") != 2 || rendered.Integer("tools") < 4 || rendered.Integer("images") != 1 || rendered.Boolean("injected") || rendered.Integer("remote") != 0)
                throw new InvalidOperationException("Markdown / 工具 / 截图或脚本防护验证失败:" + render);
            await chat.ExecuteScriptAsync("scrollTo(0,document.documentElement.scrollHeight)"); await Task.Delay(100);
            await Composite("dark", web, chat);
            await WebViewCapture.SaveAsync(web.CoreWebView2, Path.Combine(output, "browser-result.png"), fromSurface: true);
            root.RequestedTheme = ElementTheme.Light;
            await Until(async () => await chat.ExecuteScriptAsync("document.documentElement.dataset.theme") == "\"light\"");
            await Composite("light", web, chat);
            var splitter = Descendants<PaneSplitter>(frame).Single();
            var initialWidth = chat.ActualWidth; services.State.SetPaneSize(pluginId + ".chat", splitter.PreferredSize + 70); splitter.ReloadPreference(); await Task.Delay(100);
            if (Math.Abs(chat.ActualWidth - initialWidth) < 30) throw new InvalidOperationException("Agent 聊天栏分隔条不能调整宽度");
            shell.ViewModel.NavigationViewService.NavigateTo<WorkflowListPage>(); await Task.Delay(80);
            shell.ViewModel.NavigationViewService.NavigateTo<PluginPageHostPage>(pageRef); await Task.Delay(150);
            if (Control<WebView2>("AgentBrowserWeb") != web) throw new InvalidOperationException("切换页面丢失浏览器会话");
            input = Control<TextBox>("AgentComposer"); send = Control<Button>("AgentSend"); stop = Control<Button>("AgentStop");
            input.Text = "测试停止"; await Task.Delay(80); await InvokeButtonAsync(send);
            await Phase("cancel-test");
            await Until(async () => await chat.ExecuteScriptAsync("Boolean(document.querySelector('details.running'))") == "true");
            await InvokeButtonAsync(stop);
            await Until(() => Task.FromResult(send.IsEnabled), 10);
            if (Control<TextBlock>("AgentStatus").Text != "已停止 · 已完成的网页操作保留") throw new InvalidOperationException("停止任务状态不正确");
            await InvokeButtonAsync(Control<Button>("AgentSettings"));
            ContentDialog? settingsDialog = null;
            await Until(() => Task.FromResult((settingsDialog = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetOpenPopupsForXamlRoot(root.XamlRoot)
                .SelectMany(x => Descendants<ContentDialog>(x.Child)).FirstOrDefault(x => x.Title as string == "DeepSeek 浏览器 Agent 设置")) is not null));
            var keyField = Descendants<PasswordBox>(settingsDialog!).Single();
            var modelField = Descendants<ComboBox>(settingsDialog!).Single(x => x.Header as string == "模型");
            await Until(() => Task.FromResult(Descendants<TextBox>(modelField).Any(x => x.Text == "deepseek-v4-flash")));
            if (settingsDialog!.ActualTheme != root.ActualTheme) throw new InvalidOperationException("Agent 设置弹窗没有跟随主题");
            var credentialField = Descendants<ComboBox>(settingsDialog!).Single(x => x.Header as string == "API Key 凭据名称");
            credentialField.Text = "agent-browser.ui-fixture"; keyField.Password = "agent-local-fixture-secret";
            await InvokeButtonAsync(Descendants<Button>(settingsDialog!).Single(x => x.Content as string == "检查技能与运行环境"));
            await Until(() => Task.FromResult(Descendants<TextBlock>(settingsDialog!).Any(x => x.Text.StartsWith("技能已就绪", StringComparison.Ordinal))));
            await CaptureElementAsync(settingsDialog!, Path.Combine(output, "settings.png"));
            await InvokeButtonAsync(Descendants<Button>(settingsDialog!).Single(x => x.Name == "PrimaryButton"));
            await Until(() => Task.FromResult(services.State.PluginSettings(pluginId).String("credential") == "agent-browser.ui-fixture"));
            if (services.Credentials.Get("agent-browser.ui-fixture") != "agent-local-fixture-secret" || services.State.PluginSettings(pluginId).ToJsonString().Contains("agent-local-fixture-secret"))
                throw new InvalidOperationException("Agent 设置没有仅保存加密凭据引用");
            await services.Plugins.SetEnabledAsync(plugin, false);
            await Phase("disable-page");
            await Until(() => Task.FromResult(!Descendants<WebView2>(frame).Any()));
            if (services.Plugins.PageCatalog().Any(x => x.Page.Id == pageRef.PageId)) throw new InvalidOperationException("禁用后插件页面仍可导航");
            await services.Plugins.SetEnabledAsync(plugin, true);
            shell.ViewModel.NavigationViewService.NavigateTo<WorkflowListPage>(); await Task.Delay(80);
            shell.ViewModel.NavigationViewService.NavigateTo<PluginPageHostPage>(pageRef);
            await Until(() => Task.FromResult(Descendants<WebView2>(frame).Count(x => x.CoreWebView2 is not null) >= 2));
            await Until(async () => await Control<WebView2>("AgentChatWeb").ExecuteScriptAsync("Boolean(window.agentChatReady)") == "true");
            lock (requests)
            {
                if (!requests[0]["messages"]![0]!["content"]!.GetValue<string>().Contains("<browser-debugger-skill>") || requests.Count < 5) throw new InvalidOperationException("技能未注入或工具循环未完成");
            }
            var captures = Directory.GetFiles(Path.Combine(services.Paths.Root, "PluginData", pluginId, "Conversations"), "network.cdp.har", SearchOption.AllDirectories);
            if (captures.Length == 0) throw new InvalidOperationException("技能没有保存网络捕获");
            await AtomicJson.WriteAsync(Path.Combine(output, "verified.json"), new { actual, rendered, requests = requests.Count, captureCount = captures.Length, cancelled = true, persistedAcrossNavigation = true, pageDisabledAndReenabled = true, serverErrors });
        }
        finally
        {
            shell.ViewModel.NavigationViewService.NavigateTo<MainPage>(); root.RequestedTheme = oldTheme; App.MainWindow.AppWindow.Resize(oldSize);
            services.State.SetPluginSettings(pluginId, oldSettings); await services.State.SaveAsync();
            listener.Stop(); await server;
        }
    }
}