first commit
This commit is contained in:
Luis Reich
282
src/OnscreenTranslator/DirectXOverlay.cs
Normal file
282
src/OnscreenTranslator/DirectXOverlay.cs
Normal file
@@ -0,0 +1,282 @@
|
||||
using System.Diagnostics;
|
||||
using System.Drawing;
|
||||
using System.IO;
|
||||
using LibreTranslate.Net.Enhanced.Constants;
|
||||
using LibreTranslate.Net.Enhanced.Models;
|
||||
using Overlay.NET.Common;
|
||||
using Overlay.NET.Directx;
|
||||
using Process.NET.Windows;
|
||||
using TesseractOCR;
|
||||
using TesseractOCR.Enums;
|
||||
using Image = TesseractOCR.Pix.Image;
|
||||
using ImageFormat = System.Drawing.Imaging.ImageFormat;
|
||||
using Point = System.Drawing.Point;
|
||||
using Rectangle = System.Drawing.Rectangle;
|
||||
|
||||
namespace OnscreenTranslator;
|
||||
|
||||
public class DirectXOverlay : DirectXOverlayPlugin
|
||||
{
|
||||
private readonly TickEngine _tickEngine = new();
|
||||
public readonly ISettings<OverlaySettings> Settings = new SerializableSettings<OverlaySettings>();
|
||||
private int _displayFps;
|
||||
private int _font;
|
||||
private int _hugeFont;
|
||||
private int _i;
|
||||
private int _interiorBrush;
|
||||
private int _redBrush;
|
||||
private int _redOpacityBrush;
|
||||
private float _rotation;
|
||||
private Stopwatch _watch;
|
||||
private readonly List<Element> TranslatedElements = new();
|
||||
private List<Element> UntranslatedElements = new();
|
||||
|
||||
public override void Initialize(IWindow targetWindow)
|
||||
{
|
||||
// Set target window by calling the base method
|
||||
base.Initialize(targetWindow);
|
||||
|
||||
// For demo, show how to use settings
|
||||
var current = Settings.Current;
|
||||
var type = GetType();
|
||||
|
||||
if (current.UpdateRate == 0)
|
||||
current.UpdateRate = 1000 / 60;
|
||||
|
||||
current.Author = GetAuthor(type);
|
||||
current.Description = GetDescription(type);
|
||||
current.Identifier = GetIdentifier(type);
|
||||
current.Name = GetName(type);
|
||||
current.Version = GetVersion(type);
|
||||
|
||||
// File is made from above info
|
||||
Settings.Save();
|
||||
Settings.Load();
|
||||
//Console.Title = @"OverlayExample";
|
||||
|
||||
OverlayWindow = new DirectXOverlayWindow(targetWindow.Handle, false);
|
||||
_watch = Stopwatch.StartNew();
|
||||
|
||||
_redBrush = OverlayWindow.Graphics.CreateBrush(0x7FFF0000);
|
||||
_redOpacityBrush = OverlayWindow.Graphics.CreateBrush(Color.FromArgb(80, 255, 0, 0));
|
||||
_interiorBrush = OverlayWindow.Graphics.CreateBrush(0x7FFFFF00);
|
||||
|
||||
_font = OverlayWindow.Graphics.CreateFont("Arial", 20);
|
||||
_hugeFont = OverlayWindow.Graphics.CreateFont("Arial", 50, true);
|
||||
|
||||
_rotation = 0.0f;
|
||||
_displayFps = 0;
|
||||
_i = 0;
|
||||
// Set up update interval and register events for the tick engine.
|
||||
|
||||
_tickEngine.PreTick += OnPreTick;
|
||||
_tickEngine.Tick += OnTick;
|
||||
}
|
||||
|
||||
private void OnPreTick(object sender, EventArgs e)
|
||||
{
|
||||
var targetWindowIsActivated = TargetWindow.IsActivated;
|
||||
if (!targetWindowIsActivated && OverlayWindow.IsVisible)
|
||||
{
|
||||
_watch.Stop();
|
||||
ClearScreen();
|
||||
OverlayWindow.Hide();
|
||||
}
|
||||
else if (targetWindowIsActivated && !OverlayWindow.IsVisible)
|
||||
{
|
||||
OverlayWindow.Show();
|
||||
}
|
||||
}
|
||||
|
||||
private void OnTick(object sender, EventArgs e)
|
||||
{
|
||||
if (!OverlayWindow.IsVisible) return;
|
||||
|
||||
OverlayWindow.Update();
|
||||
InternalRender();
|
||||
}
|
||||
|
||||
public override void Enable()
|
||||
{
|
||||
_tickEngine.Interval = Settings.Current.UpdateRate.Milliseconds();
|
||||
_tickEngine.IsTicking = true;
|
||||
base.Enable();
|
||||
}
|
||||
|
||||
public override void Disable()
|
||||
{
|
||||
_tickEngine.IsTicking = false;
|
||||
base.Disable();
|
||||
}
|
||||
|
||||
public override void Update()
|
||||
{
|
||||
_tickEngine.Pulse();
|
||||
}
|
||||
|
||||
protected void InternalRender()
|
||||
{
|
||||
if (!_watch.IsRunning) _watch.Start();
|
||||
|
||||
OverlayWindow.Graphics.BeginScene();
|
||||
OverlayWindow.Graphics.ClearScene();
|
||||
|
||||
RenderElements(TranslatedElements);
|
||||
|
||||
if (_watch.ElapsedMilliseconds > 1000)
|
||||
{
|
||||
_i = _displayFps;
|
||||
_displayFps = 0;
|
||||
_watch.Restart();
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
_displayFps++;
|
||||
}
|
||||
|
||||
OverlayWindow.Graphics.EndScene();
|
||||
}
|
||||
|
||||
public void TriggerTranslation()
|
||||
{
|
||||
var HTML = GetScreenHTML();
|
||||
|
||||
UntranslatedElements = GetScreenElements(HTML);
|
||||
TranslateElements();
|
||||
}
|
||||
|
||||
private string GetScreenHTML()
|
||||
{
|
||||
var mem = GetScreenDataFromTargetWindow();
|
||||
var languages = new List<Language>();
|
||||
languages.Add(Language.Japanese);
|
||||
languages.Add(Language.JapaneseVertical);
|
||||
languages.Add(Language.English);
|
||||
var engine = new Engine("F:/Downloads/tessdata-4.1.0/tessdata-4.1.0", languages);
|
||||
var img = Image.LoadFromMemory(mem);
|
||||
var page = engine.Process(img);
|
||||
return page.HOcrText();
|
||||
}
|
||||
|
||||
private List<Element> GetScreenElements(string HTML)
|
||||
{
|
||||
var elements = new List<Element>();
|
||||
|
||||
var Index = 0;
|
||||
while (Index != -1)
|
||||
{
|
||||
Index = HTML.IndexOf("class='ocrx_word'", Index);
|
||||
if (Index == -1) break;
|
||||
var BbboxIndex = HTML.IndexOf("bbox", Index);
|
||||
if (BbboxIndex == -1) break;
|
||||
var XIndex = BbboxIndex + 4;
|
||||
var YIndex = HTML.IndexOf(" ", XIndex + 1);
|
||||
if (YIndex == -1) break;
|
||||
++YIndex;
|
||||
var YEnd = HTML.IndexOf(" ", YIndex + 1);
|
||||
if (YEnd == -1) break;
|
||||
|
||||
var WordStart = HTML.IndexOf(">", YIndex);
|
||||
if (WordStart == -1) break;
|
||||
++WordStart;
|
||||
var WordEnd = HTML.IndexOf("<", WordStart);
|
||||
if (WordEnd == -1) break;
|
||||
|
||||
var NewElement = new Element();
|
||||
var XString = HTML.Substring(XIndex, YIndex - XIndex);
|
||||
var YString = HTML.Substring(YIndex, YEnd - YIndex);
|
||||
NewElement.X = int.Parse(XString);
|
||||
NewElement.Y = int.Parse(YString);
|
||||
NewElement.Input = HTML.Substring(WordStart, WordEnd - WordStart);
|
||||
elements.Add(NewElement);
|
||||
|
||||
Index = WordEnd;
|
||||
}
|
||||
|
||||
return elements;
|
||||
}
|
||||
|
||||
private async void TranslateElements()
|
||||
{
|
||||
var libreTranslate = new LibreTranslate.Net.Enhanced.LibreTranslate("http://127.0.0.1:5000", "");
|
||||
|
||||
TranslatedElements.Clear();
|
||||
|
||||
foreach (var element in UntranslatedElements)
|
||||
{
|
||||
if (element.Input == "")
|
||||
continue;
|
||||
|
||||
var Detection = await libreTranslate.DetectAsync(new Detect
|
||||
{
|
||||
Text = element.Input
|
||||
});
|
||||
|
||||
if (Detection != null)
|
||||
{
|
||||
var First = Detection.FirstOrDefault();
|
||||
if (First != null)
|
||||
if (First.Confidence == 0)
|
||||
continue;
|
||||
}
|
||||
|
||||
var TargetText = await libreTranslate.TranslateAsync(new Translate
|
||||
{
|
||||
Source = LanguageCode.English,
|
||||
Target = LanguageCode.German,
|
||||
Text = element.Input
|
||||
});
|
||||
var New = element;
|
||||
New.Output = TargetText;
|
||||
TranslatedElements.Add(New);
|
||||
}
|
||||
}
|
||||
|
||||
private void RenderElements(List<Element> elements)
|
||||
{
|
||||
foreach (var element in elements)
|
||||
OverlayWindow.Graphics.DrawText(element.Input, _font, _redBrush, element.X, element.Y);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (OverlayWindow != null) OverlayWindow.Dispose();
|
||||
base.Dispose();
|
||||
}
|
||||
|
||||
public void ClearScreen()
|
||||
{
|
||||
OverlayWindow.Graphics.BeginScene();
|
||||
OverlayWindow.Graphics.ClearScene();
|
||||
OverlayWindow.Graphics.EndScene();
|
||||
}
|
||||
|
||||
private MemoryStream GetScreenDataFromTargetWindow()
|
||||
{
|
||||
var bounds = new Rectangle(TargetWindow.X,
|
||||
TargetWindow.Y,
|
||||
TargetWindow.Width,
|
||||
TargetWindow.Height);
|
||||
|
||||
using var bitmap = new Bitmap(bounds.Width, bounds.Height);
|
||||
using (var g = Graphics.FromImage(bitmap))
|
||||
{
|
||||
g.CopyFromScreen(new Point(bounds.Left, bounds.Top), Point.Empty, bounds.Size);
|
||||
}
|
||||
|
||||
// bitmap.Save("C://test.jpg", ImageFormat.Jpeg);
|
||||
System.Drawing.Image im = System.Drawing.Image.FromHbitmap(bitmap.GetHbitmap());
|
||||
var mem = new MemoryStream();
|
||||
im.Save(mem, ImageFormat.Bmp);
|
||||
return mem;
|
||||
}
|
||||
|
||||
private struct Element
|
||||
{
|
||||
public int X;
|
||||
public int Y;
|
||||
public string Input;
|
||||
public string Output;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user