diff --git a/src/OnscreenTranslator/DirectXOverlay.cs b/src/OnscreenTranslator/DirectXOverlay.cs index a4a2954..842f8df 100644 --- a/src/OnscreenTranslator/DirectXOverlay.cs +++ b/src/OnscreenTranslator/DirectXOverlay.cs @@ -1,13 +1,13 @@ -using System.Diagnostics; +using System.Diagnostics; using System.Drawing; using System.IO; -using LibreTranslate.Net.Enhanced.Constants; +using System.Net; +using System.Text.RegularExpressions; using LibreTranslate.Net.Enhanced.Models; using Overlay.NET.Common; using Overlay.NET.Directx; using Process.NET.Windows; using TesseractOCR; -using TesseractOCR.Enums; using Image = TesseractOCR.Pix.Image; using ImageFormat = System.Drawing.Imaging.ImageFormat; using Point = System.Drawing.Point; @@ -17,9 +17,22 @@ namespace OnscreenTranslator; public class DirectXOverlay : DirectXOverlayPlugin { + private const string TessdataPath = "C:/Users/luis-/Downloads/tessdata-4.1.0/tessdata-4.1.0"; + + private static readonly Regex LineRegex = + new(@"class=['""]ocr_(?:line|textline|caption|header)['""][^>]*?title=['""]bbox (\d+) (\d+) (\d+) (\d+)"); + + private static readonly Regex WordRegex = new(@"class=['""]ocrx_word['""][^>]*>(.*?)", RegexOptions.Singleline); + private static readonly Regex TagRegex = new("<[^>]+>"); + private readonly TickEngine _tickEngine = new(); public readonly ISettings Settings = new SerializableSettings(); + private readonly LibreTranslate.Net.Enhanced.LibreTranslate _libreTranslate = new("http://127.0.0.1:5000", ""); + private int _backgroundBrush; private int _displayFps; + private Engine? _engine; + private TranslationLanguage? _engineLanguage; + private bool _excludedFromCapture; private int _font; private int _hugeFont; private int _i; @@ -27,9 +40,21 @@ public class DirectXOverlay : DirectXOverlayPlugin private int _redBrush; private int _redOpacityBrush; private float _rotation; + private int _textBrush; + private bool _translating; private Stopwatch _watch; - private readonly List TranslatedElements = new(); - private List UntranslatedElements = new(); + + // Replaced as a whole when a translation finishes and never mutated, so rendering can't see a half-built list + private IReadOnlyList _translatedElements = []; + + public TranslationLanguage SourceLanguage { get; set; } = TranslationLanguage.Japanese; + public TranslationLanguage TargetLanguage { get; set; } = TranslationLanguage.German; + + /// + /// Skip LibreTranslate and show Tesseract's text as-is, for checking OCR results. + /// Applies to the next TriggerTranslation. + /// + public bool ShowOcrText { get; set; } public override void Initialize(IWindow targetWindow) { @@ -55,11 +80,15 @@ public class DirectXOverlay : DirectXOverlayPlugin //Console.Title = @"OverlayExample"; OverlayWindow = new DirectXOverlayWindow(targetWindow.Handle, false); + // Keep the overlay out of the screenshots fed to Tesseract + _excludedFromCapture = OverlayWindow.ExcludeFromCapture(); _watch = Stopwatch.StartNew(); _redBrush = OverlayWindow.Graphics.CreateBrush(0x7FFF0000); _redOpacityBrush = OverlayWindow.Graphics.CreateBrush(Color.FromArgb(80, 255, 0, 0)); _interiorBrush = OverlayWindow.Graphics.CreateBrush(0x7FFFFF00); + _backgroundBrush = OverlayWindow.Graphics.CreateBrush(Color.FromArgb(200, 0, 0, 0)); + _textBrush = OverlayWindow.Graphics.CreateBrush(Color.FromArgb(255, 255, 255, 255)); _font = OverlayWindow.Graphics.CreateFont("Arial", 20); _hugeFont = OverlayWindow.Graphics.CreateFont("Arial", 50, true); @@ -121,7 +150,7 @@ public class DirectXOverlay : DirectXOverlayPlugin OverlayWindow.Graphics.BeginScene(); OverlayWindow.Graphics.ClearScene(); - RenderElements(TranslatedElements); + RenderElements(_translatedElements); if (_watch.ElapsedMilliseconds > 1000) { @@ -138,110 +167,151 @@ public class DirectXOverlay : DirectXOverlayPlugin OverlayWindow.Graphics.EndScene(); } - public void TriggerTranslation() + /// + /// Captures the target window, reads its text and replaces the overlay with the translation. + /// Must be called from the UI thread; calls made while a translation is running are ignored. + /// + public async Task TriggerTranslation() { - var HTML = GetScreenHTML(); + if (_translating || OverlayWindow == null) return; + _translating = true; - UntranslatedElements = GetScreenElements(HTML); - TranslateElements(); + try + { + using var screenshot = await CaptureTargetWindow(); + var source = SourceLanguage; + var target = TargetLanguage; + + // OCR takes a while, keep it off the UI thread so rendering continues + var html = await Task.Run(() => GetScreenHTML(screenshot, source)); + var untranslatedElements = GetScreenElements(html, source); + var translatedElements = ShowOcrText + ? ShowUntranslated(untranslatedElements) + : await TranslateElements(untranslatedElements, source, target); + + if (OverlayWindow == null) return; // disposed while translating + + foreach (var element in translatedElements) + { + // Cover the original line, growing the box if the translation is longer + var size = OverlayWindow.Graphics.MeasureText(element.Output, _font); + element.BoxWidth = Math.Max(element.Width, (int)Math.Ceiling(size.Width)); + element.BoxHeight = Math.Max(element.Height, (int)Math.Ceiling(size.Height)); + } + + _translatedElements = translatedElements; + // DrawText caches a layout per distinct string; drop the ones from the previous translation + OverlayWindow.Graphics.DeleteLayoutContainer(); + } + finally + { + _translating = false; + } } - private string GetScreenHTML() + private string GetScreenHTML(MemoryStream screenshot, TranslationLanguage language) { - var mem = GetScreenDataFromTargetWindow(); - var languages = new List(); - languages.Add(Language.Japanese); - languages.Add(Language.JapaneseVertical); - languages.Add(Language.English); - var engine = new Engine("C:/Users/luis-/Downloads/tessdata-4.1.0/tessdata-4.1.0", languages); - var img = Image.LoadFromMemory(mem); - var page = engine.Process(img); + // Loading the models is slow, so the engine is kept until the source language changes + if (_engine == null || _engineLanguage != language) + { + _engine?.Dispose(); + _engine = new Engine(TessdataPath, language.OcrLanguages.ToList()); + _engineLanguage = language; + } + + using var img = Image.LoadFromMemory(screenshot); + using var page = _engine.Process(img); return page.HOcrText(); } - private List GetScreenElements(string HTML) + /// + /// Turns hOCR into one element per text line, positioned at the line's bounding box. + /// + private static List GetScreenElements(string html, TranslationLanguage language) { var elements = new List(); + var lines = LineRegex.Matches(html); - var Index = 0; - while (Index != -1) + for (var i = 0; i < lines.Count; i++) { - Index = HTML.IndexOf("class='ocrx_word'", Index); - if (Index == -1) break; - var BbboxIndex = HTML.IndexOf("bbox", Index); - if (BbboxIndex == -1) break; - var XIndex = BbboxIndex + 4; - var YIndex = HTML.IndexOf(" ", XIndex + 1); - if (YIndex == -1) break; - ++YIndex; - var YEnd = HTML.IndexOf(" ", YIndex + 1); - if (YEnd == -1) break; + var line = lines[i]; + var end = i + 1 < lines.Count ? lines[i + 1].Index : html.Length; + var lineHtml = html.Substring(line.Index, end - line.Index); - var WordStart = HTML.IndexOf(">", YIndex); - if (WordStart == -1) break; - ++WordStart; - var WordEnd = HTML.IndexOf("<", WordStart); - if (WordEnd == -1) break; + // Word text may be wrapped in / and contain entities like & + var words = WordRegex.Matches(lineHtml) + .Select(word => WebUtility.HtmlDecode(TagRegex.Replace(word.Groups[1].Value, "")).Trim()) + .Where(word => word.Length > 0); + var text = string.Join(language.JoinWordsWithSpace ? " " : "", words); + if (text.Length == 0) continue; - var NewElement = new Element(); - var XString = HTML.Substring(XIndex, YIndex - XIndex); - var YString = HTML.Substring(YIndex, YEnd - YIndex); - NewElement.X = int.Parse(XString); - NewElement.Y = int.Parse(YString); - NewElement.Input = HTML.Substring(WordStart, WordEnd - WordStart); - elements.Add(NewElement); - - Index = WordEnd; + var x0 = int.Parse(line.Groups[1].Value); + var y0 = int.Parse(line.Groups[2].Value); + elements.Add(new Element + { + X = x0, + Y = y0, + Width = int.Parse(line.Groups[3].Value) - x0, + Height = int.Parse(line.Groups[4].Value) - y0, + Input = text + }); } return elements; } - private async void TranslateElements() + private async Task> TranslateElements(List elements, TranslationLanguage source, + TranslationLanguage target) { - var libreTranslate = new LibreTranslate.Net.Enhanced.LibreTranslate("http://127.0.0.1:5000", ""); + var translated = new List(); - TranslatedElements.Clear(); - - foreach (var element in UntranslatedElements) + foreach (var element in elements) { - if (element.Input == "") - continue; - - var Detection = await libreTranslate.DetectAsync(new Detect + var targetText = await _libreTranslate.TranslateAsync(new Translate { + Source = source.Code, + Target = target.Code, Text = element.Input }); + if (string.IsNullOrWhiteSpace(targetText)) continue; - if (Detection != null) - { - var First = Detection.FirstOrDefault(); - if (First != null) - if (First.Confidence == 0) - continue; - } - - var TargetText = await libreTranslate.TranslateAsync(new Translate - { - Source = LanguageCode.English, - Target = LanguageCode.German, - Text = element.Input - }); - var New = element; - New.Output = TargetText; - TranslatedElements.Add(New); + element.Output = targetText; + translated.Add(element); } + + return translated; } - private void RenderElements(List elements) + private static List ShowUntranslated(List elements) { + foreach (var element in elements) element.Output = element.Input; + return elements; + } + + private void RenderElements(IReadOnlyList elements) + { + const int padding = 4; + foreach (var element in elements) - OverlayWindow.Graphics.DrawText(element.Input, _font, _redBrush, element.X, element.Y); + { + OverlayWindow.Graphics.FillRectangle(element.X - padding, element.Y - padding, + element.BoxWidth + padding * 2, element.BoxHeight + padding * 2, _backgroundBrush); + OverlayWindow.Graphics.DrawText(element.Output, _font, _textBrush, element.X, element.Y); + } } public override void Dispose() { - if (OverlayWindow != null) OverlayWindow.Dispose(); + _tickEngine.IsTicking = false; + _tickEngine.PreTick -= OnPreTick; + _tickEngine.Tick -= OnTick; + + OverlayWindow?.Dispose(); + OverlayWindow = null; + + _engine?.Dispose(); + _engine = null; + base.Dispose(); } @@ -252,31 +322,53 @@ public class DirectXOverlay : DirectXOverlayPlugin OverlayWindow.Graphics.EndScene(); } - private MemoryStream GetScreenDataFromTargetWindow() + private async Task CaptureTargetWindow() { - var bounds = new Rectangle(TargetWindow.X, - TargetWindow.Y, - TargetWindow.Width, - TargetWindow.Height); - - using var bitmap = new Bitmap(bounds.Width, bounds.Height); - using (var g = Graphics.FromImage(bitmap)) + // Without capture exclusion (Windows before 10 2004) the overlay would end up in its own screenshot + var hideOverlay = !_excludedFromCapture && OverlayWindow.IsVisible; + if (hideOverlay) { - g.CopyFromScreen(new Point(bounds.Left, bounds.Top), Point.Empty, bounds.Size); + OverlayWindow.Hide(); + await Task.Delay(50); // let DWM compose a frame without the overlay } - // bitmap.Save("C://test.jpg", ImageFormat.Jpeg); - System.Drawing.Image im = System.Drawing.Image.FromHbitmap(bitmap.GetHbitmap()); - var mem = new MemoryStream(); - im.Save(mem, ImageFormat.Bmp); - return mem; + try + { + // Use the overlay's bounds (GetWindowRect of the target), not TargetWindow.X/Y/Width/Height: + // Process.NET reports the restored placement rect, which is wrong for snapped or maximized windows + OverlayWindow.Update(); + var bounds = new Rectangle(OverlayWindow.X, + OverlayWindow.Y, + OverlayWindow.Width, + OverlayWindow.Height); + + using var bitmap = new Bitmap(bounds.Width, bounds.Height); + using (var g = Graphics.FromImage(bitmap)) + { + g.CopyFromScreen(new Point(bounds.Left, bounds.Top), Point.Empty, bounds.Size); + } + + // bitmap.Save("C://test.jpg", ImageFormat.Jpeg); + var mem = new MemoryStream(); + bitmap.Save(mem, ImageFormat.Bmp); + mem.Position = 0; + return mem; + } + finally + { + if (hideOverlay) OverlayWindow.Show(); + } } - private struct Element + private class Element { public int X; public int Y; - public string Input; - public string Output; + public int Width; + public int Height; + public int BoxWidth; + public int BoxHeight; + public string Input = ""; + public string Output = ""; } -} \ No newline at end of file +} diff --git a/src/OnscreenTranslator/MainWindow.xaml b/src/OnscreenTranslator/MainWindow.xaml index d541db9..4507e79 100644 --- a/src/OnscreenTranslator/MainWindow.xaml +++ b/src/OnscreenTranslator/MainWindow.xaml @@ -9,6 +9,9 @@