using System.Drawing; using System.Drawing.Drawing2D; using System.Drawing.Imaging; using System.Runtime.InteropServices; using System.Text.Json.Nodes; using System.Text.RegularExpressions; using Tesseract; using XFEWorkflow.Abstractions; namespace XFEWorkflow.Plugins.Vision; public sealed class OcrRecognizer : IDisposable { private static readonly object nativeGate = new(); private static bool nativeLoaded; private readonly TesseractEngine engine; public OcrRecognizer(string language = "chi_sim+eng") { if (RuntimeInformation.ProcessArchitecture is not (Architecture.X64 or Architecture.X86)) throw new PlatformNotSupportedException("当前离线 OCR 引擎支持 Windows x64/x86;请使用 x64 客户端进行文字识别。"); if (language is not ("chi_sim+eng" or "chi_sim" or "eng")) throw new ArgumentException("识别语言应为 chi_sim+eng、chi_sim 或 eng。"); var root = Path.GetDirectoryName(typeof(OcrRecognizer).Assembly.Location)!; foreach (var code in language.Split('+')) if (!File.Exists(Path.Combine(root, "tessdata", code + ".traineddata"))) throw new FileNotFoundException("OCR 模型缺失,请重新安装屏幕与识别插件:" + code); TesseractEnviornment.CustomSearchPath = root; try { lock (nativeGate) { if (!nativeLoaded) { var runtime = Path.Combine(root, "NativeRuntime", RuntimeInformation.ProcessArchitecture == Architecture.X64 ? "win-x64" : "win-x86"); foreach (var file in new[] { "vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll" }) if (File.Exists(Path.Combine(runtime, file))) NativeLibrary.Load(Path.Combine(runtime, file)); nativeLoaded = true; } } engine = new TesseractEngine(Path.Combine(root, "tessdata"), language, EngineMode.LstmOnly); } catch (Exception ex) when (ex is DllNotFoundException or TypeInitializationException) { throw new InvalidOperationException("无法加载离线 OCR 引擎,请检查插件原生库与 Microsoft Visual C++ 2015–2022 运行库。", ex); } } public JsonObject Recognize(ScreenFrame frame, JsonObject options, CancellationToken ct) { ct.ThrowIfCancellationRequested(); var scale = VisionParameters.Number(options, "scale", 2, 1, 4); var confidence = VisionParameters.Number(options, "minimumConfidence", 35, 0, 100); var segmentation = options.String("segmentation", "SparseText") switch { "SparseText" => PageSegMode.SparseText, "Auto" => PageSegMode.Auto, "SingleBlock" => PageSegMode.SingleBlock, "SingleLine" => PageSegMode.SingleLine, _ => throw new ArgumentException("文字排版参数无效。") }; var width = checked((int)Math.Round(frame.Bitmap.Width * scale)); var height = checked((int)Math.Round(frame.Bitmap.Height * scale)); if ((long)width * height > ScreenRegion.MaximumPixels) throw new ArgumentException("放大后的 OCR 图像超过 4000 万像素,请缩小区域或降低放大倍数。"); using var resized = new Bitmap(width, height, PixelFormat.Format24bppRgb); using (var graphics = Graphics.FromImage(resized)) { graphics.InterpolationMode = InterpolationMode.HighQualityBicubic; graphics.Clear(Color.White); graphics.DrawImage(frame.Bitmap, 0, 0, width, height); } using var stream = new MemoryStream(); resized.Save(stream, System.Drawing.Imaging.ImageFormat.Png); using var pix = Pix.LoadFromMemory(stream.ToArray()); using var page = engine.Process(pix, segmentation); ct.ThrowIfCancellationRequested(); JsonArray words = [], lines = []; using (var iterator = page.GetIterator()) { iterator.Begin(); var line = -1; do { ct.ThrowIfCancellationRequested(); if (line < 0 || iterator.IsAtBeginningOf(PageIteratorLevel.TextLine)) { line++; if (iterator.TryGetBoundingBox(PageIteratorLevel.TextLine, out var lineBounds)) { var entry = Bounds(lineBounds, frame.Region, scale); entry["text"] = (iterator.GetText(PageIteratorLevel.TextLine) ?? "").Trim(); entry["line"] = line; lines.Add(entry); } } var text = (iterator.GetText(PageIteratorLevel.Word) ?? "").Trim(); var score = iterator.GetConfidence(PageIteratorLevel.Word); if (text.Length == 0 || score < confidence || !iterator.TryGetBoundingBox(PageIteratorLevel.Word, out var bounds)) continue; var word = Bounds(bounds, frame.Region, scale); word["text"] = text; word["confidence"] = Math.Round(score, 2); word["line"] = line; words.Add(word); } while (iterator.Next(PageIteratorLevel.Word)); } return new() { ["text"] = page.GetText().Trim(), ["words"] = words, ["lines"] = lines, ["confidence"] = Math.Round(page.GetMeanConfidence() * 100, 2), ["coordinateSpace"] = frame.CoordinateSpace, ["region"] = frame.Region.ToJson() }; } private static JsonObject Bounds(Tesseract.Rect bounds, ScreenRegion origin, double scale) { var x = (int)Math.Floor(bounds.X1 / scale); var y = (int)Math.Floor(bounds.Y1 / scale); return new ScreenRegion(origin.X + x, origin.Y + y, Math.Max(1, (int)Math.Ceiling(bounds.X2 / scale) - x), Math.Max(1, (int)Math.Ceiling(bounds.Y2 / scale) - y)).ToJson(); } public void Dispose() => engine.Dispose(); } public static class OcrTextSearch { public static JsonObject Find(JsonObject recognized, string target, string mode = "contains", bool ignoreCase = true) { if (string.IsNullOrWhiteSpace(target)) throw new ArgumentException("请输入要查找的文字。"); if (mode is not ("contains" or "exact" or "regex")) throw new ArgumentException("文字匹配方式无效。"); var comparison = ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; var regex = mode == "regex" ? new Regex(target, ignoreCase ? RegexOptions.IgnoreCase | RegexOptions.CultureInvariant : RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(500)) : null; JsonArray matches = []; var groups = (recognized["words"] as JsonArray ?? []).OfType().GroupBy(w => w.Integer("line")); foreach (var group in groups) { var words = group.ToArray(); var text = string.Join(" ", words.Select(w => w.String("text"))); var offsets = new List<(int Start, int End, JsonObject Word)>(); var position = 0; foreach (var word in words) { var wordText = word.String("text"); offsets.Add((position, position + wordText.Length, word)); position += wordText.Length + 1; } // Chinese OCR often inserts spaces between words. Match without whitespace, // while retaining a map to original characters for precise click bounds. var map = Enumerable.Range(0, text.Length).Where(i => !char.IsWhiteSpace(text[i])).ToArray(); var compact = new string(map.Select(i => text[i]).ToArray()); var needle = new string(target.Where(c => !char.IsWhiteSpace(c)).ToArray()); var spans = new List<(int Start, int Length, bool Compact)>(); if (regex is not null) { foreach (Match match in regex.Matches(text)) if (match.Length > 0) spans.Add((match.Index, match.Length, false)); } else if (mode == "exact") { if (compact.Equals(needle, comparison)) spans.Add((0, compact.Length, true)); } else for (var start = 0; needle.Length > 0 && start < compact.Length;) { var index = compact.IndexOf(needle, start, comparison); if (index < 0) break; spans.Add((index, needle.Length, true)); start = index + needle.Length; } foreach (var span in spans) { var start = span.Compact ? map[span.Start] : span.Start; var end = span.Compact ? map[span.Start + span.Length - 1] + 1 : span.Start + span.Length; var selected = offsets.Where(w => w.End > start && w.Start < end).Select(w => w.Word).ToArray(); if (selected.Length == 0) continue; var x = selected.Min(w => w.Integer("x")); var y = selected.Min(w => w.Integer("y")); var right = selected.Max(w => w.Integer("x") + w.Integer("width")); var bottom = selected.Max(w => w.Integer("y") + w.Integer("height")); var bounds = new ScreenRegion(x, y, right - x, bottom - y); var match = bounds.ToJson(); match["text"] = text[start..end]; match["point"] = VisionParameters.Point(bounds.CenterX, bounds.CenterY); matches.Add(match); if (matches.Count >= 100) break; } if (matches.Count >= 100) break; } return VisionParameters.Matches(matches, recognized.String("coordinateSpace"), recognized.String("text")); } }