diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bdb0cab --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49f18f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,303 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# ========================= +# Operating System Files +# ========================= + +# OSX +# ========================= + +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows +# ========================= + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk diff --git a/READMEog.md b/READMEog.md new file mode 100644 index 0000000..fff909c --- /dev/null +++ b/READMEog.md @@ -0,0 +1,34 @@ +# Overlay.NET + +## Synopsis + +Overlay.NET aims to be An easy-to-use overlay library that allows you to render visuals to other windows, and offers lots of overlay options, from external WPF windows to shared process directx hooks to do the overlay rendering. + + + +## NuGet: https://www.nuget.org/packages/Overlay.NET/ + +## Features +- WPF based overlay. +- Directx based overlay. +- DirectX code is 99% from this project, check it out if interested in dx based overlays - https://github.com/michel-pi/GameOverlay.Net +- Soon: Shared process, or 'in-process' directX engine to use the games directx device instead. + +## Demos / Examples +You may find the demo code here: https://github.com/lolp1/Overlay.NET/tree/master/src/Overlay.NET.Demo + +You may view a quick demo video of the WPF/directx overlay below. + + + +## Contributing (please do!) +- If you like it, star it. +- If you love it, contribute. +- If you want to add to the sample/demo apps, include a working example in your pull request, preferably added to the demo project. + +## Credits +- Julian Forrester aka Arcanaeum for showing me how to make a basic wpf overlay. +- https://github.com/michel-pi/GameOverlay.Net - The external directx2D overlay implementation. +- xaviermonin (https://github.com/xaviermonin) - fixed the most annoying directx overlay bug for the creators window update. diff --git a/src/OnscreenTranslator/App.xaml b/src/OnscreenTranslator/App.xaml new file mode 100644 index 0000000..a3e7d7f --- /dev/null +++ b/src/OnscreenTranslator/App.xaml @@ -0,0 +1,9 @@ + + + + + diff --git a/src/OnscreenTranslator/App.xaml.cs b/src/OnscreenTranslator/App.xaml.cs new file mode 100644 index 0000000..e1cc4fd --- /dev/null +++ b/src/OnscreenTranslator/App.xaml.cs @@ -0,0 +1,14 @@ +using System.Configuration; +using System.Data; +using System.Windows; +using System.Windows.Input; +using Process.NET.Windows; + +namespace OnscreenTranslator; + +/// +/// Interaction logic for App.xaml +/// +public partial class App : Application +{ +} \ No newline at end of file diff --git a/src/OnscreenTranslator/AssemblyInfo.cs b/src/OnscreenTranslator/AssemblyInfo.cs new file mode 100644 index 0000000..4a05c7d --- /dev/null +++ b/src/OnscreenTranslator/AssemblyInfo.cs @@ -0,0 +1,10 @@ +using System.Windows; + +[assembly: ThemeInfo( + ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located + //(used if a resource is not found in the page, + // or application resource dictionaries) + ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located + //(used if a resource is not found in the page, + // app, or any theme specific resource dictionaries) +)] \ No newline at end of file diff --git a/src/OnscreenTranslator/DirectXOverlay.cs b/src/OnscreenTranslator/DirectXOverlay.cs new file mode 100644 index 0000000..600b089 --- /dev/null +++ b/src/OnscreenTranslator/DirectXOverlay.cs @@ -0,0 +1,282 @@ +using System.Diagnostics; +using System.Drawing; +using System.IO; +using LibreTranslate.Net.Enhanced.Constants; +using LibreTranslate.Net.Enhanced.Models; +using Overlay.NET.Common; +using Overlay.NET.Directx; +using Process.NET.Windows; +using TesseractOCR; +using TesseractOCR.Enums; +using Image = TesseractOCR.Pix.Image; +using ImageFormat = System.Drawing.Imaging.ImageFormat; +using Point = System.Drawing.Point; +using Rectangle = System.Drawing.Rectangle; + +namespace OnscreenTranslator; + +public class DirectXOverlay : DirectXOverlayPlugin +{ + private readonly TickEngine _tickEngine = new(); + public readonly ISettings Settings = new SerializableSettings(); + private int _displayFps; + private int _font; + private int _hugeFont; + private int _i; + private int _interiorBrush; + private int _redBrush; + private int _redOpacityBrush; + private float _rotation; + private Stopwatch _watch; + private readonly List TranslatedElements = new(); + private List UntranslatedElements = new(); + + public override void Initialize(IWindow targetWindow) + { + // Set target window by calling the base method + base.Initialize(targetWindow); + + // For demo, show how to use settings + var current = Settings.Current; + var type = GetType(); + + if (current.UpdateRate == 0) + current.UpdateRate = 1000 / 60; + + current.Author = GetAuthor(type); + current.Description = GetDescription(type); + current.Identifier = GetIdentifier(type); + current.Name = GetName(type); + current.Version = GetVersion(type); + + // File is made from above info + Settings.Save(); + Settings.Load(); + //Console.Title = @"OverlayExample"; + + OverlayWindow = new DirectXOverlayWindow(targetWindow.Handle, false); + _watch = Stopwatch.StartNew(); + + _redBrush = OverlayWindow.Graphics.CreateBrush(0x7FFF0000); + _redOpacityBrush = OverlayWindow.Graphics.CreateBrush(Color.FromArgb(80, 255, 0, 0)); + _interiorBrush = OverlayWindow.Graphics.CreateBrush(0x7FFFFF00); + + _font = OverlayWindow.Graphics.CreateFont("Arial", 20); + _hugeFont = OverlayWindow.Graphics.CreateFont("Arial", 50, true); + + _rotation = 0.0f; + _displayFps = 0; + _i = 0; + // Set up update interval and register events for the tick engine. + + _tickEngine.PreTick += OnPreTick; + _tickEngine.Tick += OnTick; + } + + private void OnPreTick(object sender, EventArgs e) + { + var targetWindowIsActivated = TargetWindow.IsActivated; + if (!targetWindowIsActivated && OverlayWindow.IsVisible) + { + _watch.Stop(); + ClearScreen(); + OverlayWindow.Hide(); + } + else if (targetWindowIsActivated && !OverlayWindow.IsVisible) + { + OverlayWindow.Show(); + } + } + + private void OnTick(object sender, EventArgs e) + { + if (!OverlayWindow.IsVisible) return; + + OverlayWindow.Update(); + InternalRender(); + } + + public override void Enable() + { + _tickEngine.Interval = Settings.Current.UpdateRate.Milliseconds(); + _tickEngine.IsTicking = true; + base.Enable(); + } + + public override void Disable() + { + _tickEngine.IsTicking = false; + base.Disable(); + } + + public override void Update() + { + _tickEngine.Pulse(); + } + + protected void InternalRender() + { + if (!_watch.IsRunning) _watch.Start(); + + OverlayWindow.Graphics.BeginScene(); + OverlayWindow.Graphics.ClearScene(); + + RenderElements(TranslatedElements); + + if (_watch.ElapsedMilliseconds > 1000) + { + _i = _displayFps; + _displayFps = 0; + _watch.Restart(); + } + + else + { + _displayFps++; + } + + OverlayWindow.Graphics.EndScene(); + } + + public void TriggerTranslation() + { + var HTML = GetScreenHTML(); + + UntranslatedElements = GetScreenElements(HTML); + TranslateElements(); + } + + private string GetScreenHTML() + { + var mem = GetScreenDataFromTargetWindow(); + var languages = new List(); + languages.Add(Language.Japanese); + languages.Add(Language.JapaneseVertical); + languages.Add(Language.English); + var engine = new Engine("F:/Downloads/tessdata-4.1.0/tessdata-4.1.0", languages); + var img = Image.LoadFromMemory(mem); + var page = engine.Process(img); + return page.HOcrText(); + } + + private List GetScreenElements(string HTML) + { + var elements = new List(); + + var Index = 0; + while (Index != -1) + { + Index = HTML.IndexOf("class='ocrx_word'", Index); + if (Index == -1) break; + var BbboxIndex = HTML.IndexOf("bbox", Index); + if (BbboxIndex == -1) break; + var XIndex = BbboxIndex + 4; + var YIndex = HTML.IndexOf(" ", XIndex + 1); + if (YIndex == -1) break; + ++YIndex; + var YEnd = HTML.IndexOf(" ", YIndex + 1); + if (YEnd == -1) break; + + var WordStart = HTML.IndexOf(">", YIndex); + if (WordStart == -1) break; + ++WordStart; + var WordEnd = HTML.IndexOf("<", WordStart); + if (WordEnd == -1) break; + + var NewElement = new Element(); + var XString = HTML.Substring(XIndex, YIndex - XIndex); + var YString = HTML.Substring(YIndex, YEnd - YIndex); + NewElement.X = int.Parse(XString); + NewElement.Y = int.Parse(YString); + NewElement.Input = HTML.Substring(WordStart, WordEnd - WordStart); + elements.Add(NewElement); + + Index = WordEnd; + } + + return elements; + } + + private async void TranslateElements() + { + var libreTranslate = new LibreTranslate.Net.Enhanced.LibreTranslate("http://127.0.0.1:5000", ""); + + TranslatedElements.Clear(); + + foreach (var element in UntranslatedElements) + { + if (element.Input == "") + continue; + + var Detection = await libreTranslate.DetectAsync(new Detect + { + Text = element.Input + }); + + if (Detection != null) + { + var First = Detection.FirstOrDefault(); + if (First != null) + if (First.Confidence == 0) + continue; + } + + var TargetText = await libreTranslate.TranslateAsync(new Translate + { + Source = LanguageCode.English, + Target = LanguageCode.German, + Text = element.Input + }); + var New = element; + New.Output = TargetText; + TranslatedElements.Add(New); + } + } + + private void RenderElements(List elements) + { + foreach (var element in elements) + OverlayWindow.Graphics.DrawText(element.Input, _font, _redBrush, element.X, element.Y); + } + + public override void Dispose() + { + if (OverlayWindow != null) OverlayWindow.Dispose(); + base.Dispose(); + } + + public void ClearScreen() + { + OverlayWindow.Graphics.BeginScene(); + OverlayWindow.Graphics.ClearScene(); + OverlayWindow.Graphics.EndScene(); + } + + private MemoryStream GetScreenDataFromTargetWindow() + { + var bounds = new Rectangle(TargetWindow.X, + TargetWindow.Y, + TargetWindow.Width, + TargetWindow.Height); + + using var bitmap = new Bitmap(bounds.Width, bounds.Height); + using (var g = Graphics.FromImage(bitmap)) + { + g.CopyFromScreen(new Point(bounds.Left, bounds.Top), Point.Empty, bounds.Size); + } + + // bitmap.Save("C://test.jpg", ImageFormat.Jpeg); + System.Drawing.Image im = System.Drawing.Image.FromHbitmap(bitmap.GetHbitmap()); + var mem = new MemoryStream(); + im.Save(mem, ImageFormat.Bmp); + return mem; + } + + private struct Element + { + public int X; + public int Y; + public string Input; + public string Output; + } +} \ No newline at end of file diff --git a/src/OnscreenTranslator/MainWindow.xaml b/src/OnscreenTranslator/MainWindow.xaml new file mode 100644 index 0000000..d541db9 --- /dev/null +++ b/src/OnscreenTranslator/MainWindow.xaml @@ -0,0 +1,15 @@ + + + +