Add "borderless window workaround" (DXVK DWM) hack toggle

CustomResolution/Cmds/BorderlessWindowedCmd.cs

@@ -0,0 +1,47 @@
+using static FFXIVClientStructs.FFXIV.Client.UI.AddonRelicNoteBook;
+using System.Collections;
+using System.Globalization;
+using System;
+
+namespace CustomResolution.Cmds;
+
+public sealed class BorderlessWindowedCmd : Cmd
+{
+    public override string Name => "cresbw";
+
+    public override string HelpMessage => $"Tweak the \"Apply borderless window workaround\" toggle.\n" +
+        $"\tExamples:\n" +
+        $"\tTo enable / disable it:\n\t\t{FullName} on\n\t\t{FullName} off\n\t\t{FullName} toggle";
+
+    public override void Run(string arguments)
+    {
+        if (string.IsNullOrEmpty(arguments))
+        {
+            Service.PrintChat("Invalid parameters.");
+            return;
+        }
+
+        switch (arguments.ToLowerInvariant())
+        {
+            case "on":
+                Service.Config.IsDXVKDWMHackEnabled = true;
+                Service.Config.Save();
+                Service.PrintChat("Enabled borderless window workaround.");
+                return;
+
+            case "off":
+                Service.Config.IsDXVKDWMHackEnabled = false;
+                Service.Config.Save();
+                Service.PrintChat("Disabled borderless window workaround.");
+                return;
+
+            case "toggle":
+                Service.Config.IsDXVKDWMHackEnabled = !Service.Config.IsDXVKDWMHackEnabled;
+                Service.Config.Save();
+                Service.PrintChat($"{(Service.Config.IsDXVKDWMHackEnabled ? "Enabled" : "Disabled")} borderless window workaround.");
+                return;
+        }
+
+        Service.PrintChat("Invalid parameters.");
+    }
+}

CustomResolution/Configuration.cs

@@ -15,6 +15,8 @@ public class Configuration : IPluginConfiguration
     public uint Width = 1024;
     public uint Height = 1024;
 
+    public bool IsDXVKDWMHackEnabled = false;
+
     [NonSerialized]
     private DalamudPluginInterface? pluginInterface;
 
@@ -28,17 +30,3 @@ public class Configuration : IPluginConfiguration
         pluginInterface!.SavePluginConfig(this);
     }
 }
-
-public enum CullingMode
-{
-    None,
-    OnlyInFront,
-    OnlyInView
-}
-
-public enum DutyMode
-{
-    Always,
-    OutsideContent,
-    InContent
-}

CustomResolution/CustomResolution.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors>0x0ade</Authors>
     <Company></Company>
-    <Version>0.1.0.3</Version>
+    <Version>0.1.0.5</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl></PackageProjectUrl>

CustomResolution/Hooks/CursorPosHooks.cs

@@ -34,6 +34,7 @@ public sealed unsafe class CursorPosHooks : IDisposable
     public void Dispose()
     {
         _getCursorPosHook.Dispose();
+        _setCursorPosHook.Dispose();
     }
 
     private bool GetCursorPosDetour(POINT* lpPoint)

CustomResolution/Plugin.cs

@@ -3,25 +3,31 @@ using Dalamud.Plugin;
 using Dalamud.Plugin.Services;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Kernel;
 using FFXIVClientStructs.FFXIV.Client.UI;
+using FFXIVClientStructs.Interop;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using TerraFX.Interop.Windows;
-using static FFXIVClientStructs.FFXIV.Client.UI.Misc.GroupPoseModule;
+using static TerraFX.Interop.Windows.Windows;
 
 namespace CustomResolution;
 
 public sealed unsafe class Plugin : IDalamudPlugin
 {
+    private readonly HRGN _invisibleRgn;
+
     private readonly List<Cmd> _cmds;
     private bool _unloading = false;
     private int _tickCount = 0;
     private HWND _currentHwnd;
     private RECT _currentClientRect;
     private RECT _currentWindowRect;
+    private bool _currentDXVKDWMHack = false;
 
     public Plugin([RequiredVersion("1.0")] DalamudPluginInterface pluginInterface)
     {
+        _invisibleRgn = CreateRectRgn(0, 0, -1, -1);
+
         pluginInterface.Create<Service>();
 
         Service.Plugin = this;
@@ -116,12 +122,12 @@ public sealed unsafe class Plugin : IDalamudPlugin
 
         var p = new POINT(x, y);
 
-        TerraFX.Interop.Windows.Windows.ScreenToClient(_currentHwnd, &p);
+        ScreenToClient(_currentHwnd, &p);
 
         p.x = (int) Math.Round(p.x * scaleX);
         p.y = (int) Math.Round(p.y * scaleY);
 
-        TerraFX.Interop.Windows.Windows.ClientToScreen(_currentHwnd, &p);
+        ClientToScreen(_currentHwnd, &p);
 
         x = p.x;
         y = p.y;
@@ -139,12 +145,12 @@ public sealed unsafe class Plugin : IDalamudPlugin
 
         var p = new POINT(x, y);
 
-        TerraFX.Interop.Windows.Windows.ScreenToClient(_currentHwnd, &p);
+        ScreenToClient(_currentHwnd, &p);
 
         p.x = (int) Math.Round(p.x * scaleX);
         p.y = (int) Math.Round(p.y * scaleY);
 
-        TerraFX.Interop.Windows.Windows.ClientToScreen(_currentHwnd, &p);
+        ClientToScreen(_currentHwnd, &p);
 
         x = p.x;
         y = p.y;
@@ -198,6 +204,29 @@ public sealed unsafe class Plugin : IDalamudPlugin
             dev->RequestResolutionChange = 1;
         }
 
+        if (Service.Config.IsDXVKDWMHackEnabled && !_unloading)
+        {
+#if false
+            Service.PluginLog.Info($"STYLE: 0x{GetWindowLong(_currentHwnd, GWL.GWL_STYLE):X8}");
+            Service.PluginLog.Info($"EXSTYLE: 0x{GetWindowLong(_currentHwnd, GWL.GWL_EXSTYLE):X8}");
+
+            Span<ushort> name = stackalloc ushort[256];
+            GetClassName(_currentHwnd, name.GetPointer(0), name.Length);
+            WNDCLASSEXW wce;
+            GetClassInfoEx(GetModuleHandle(null), name.GetPointer(0), &wce);
+
+            Service.PluginLog.Info($"CLASS: {new string((char*) name.GetPointer(0))}");
+            Service.PluginLog.Info($"CLASS.style: 0x{wce.style:X8}");
+#endif
+
+            SetDXVKDWMHack(true);
+        }
+        else if (!Service.Config.IsDXVKDWMHackEnabled && _currentDXVKDWMHack)
+        {
+            SetDXVKDWMHack(false);
+        }
+
+        _currentDXVKDWMHack = Service.Config.IsDXVKDWMHackEnabled;
 
         CurrentWidth = width;
         CurrentHeight = height;
@@ -205,6 +234,36 @@ public sealed unsafe class Plugin : IDalamudPlugin
         CurrentWindowHeight = (uint) rectHeight;
     }
 
+    private void SetDXVKDWMHack(bool enabled)
+    {
+        /* Default maximized style / exstyle is 0x95000000 / 0.
+         * WS.WS_POPUP | WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_MAXIMIZE
+         * Default windowed style / exstyle is 0x14CF0000 / 0.
+         * WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_CAPTION | WS.WS_SYSMENU | WS.WS_THICKFRAME | WS.WS_MINIMIZEBOX | WS.WS_MAXIMIZEBOX
+         */
+
+        uint style = (uint) GetWindowLong(_currentHwnd, GWL.GWL_STYLE);
+        bool fullscreen = (style & WS.WS_SYSMENU) == 0;
+
+        /* Alternative hacks:
+         * - WS_EX_CLIENTEDGE, at the cost of having a 2px border on each side.
+         */
+
+        if (fullscreen)
+        {
+            if (enabled)
+            {
+                style &= ~WS.WS_POPUP;
+            }
+            else
+            {
+                style |= WS.WS_POPUP;
+            }
+        }
+
+        SetWindowLong(_currentHwnd, GWL.GWL_STYLE, (int) style);
+    }
+
     private void OnFrameworkUpdate(IFramework framework)
     {
         var dev = Device.Instance();
@@ -214,8 +273,8 @@ public sealed unsafe class Plugin : IDalamudPlugin
         fixed (RECT* currentClientRectPtr = &_currentClientRect)
         fixed (RECT* currentWindowRectPtr = &_currentWindowRect)
         {
-            TerraFX.Interop.Windows.Windows.GetClientRect(_currentHwnd, currentClientRectPtr);
+            GetClientRect(_currentHwnd, currentClientRectPtr);
-            TerraFX.Interop.Windows.Windows.GetWindowRect(_currentHwnd, currentWindowRectPtr);
+            GetWindowRect(_currentHwnd, currentWindowRectPtr);
         }
 
         if (_tickCount++ >= 10)

CustomResolution/Windows/ConfigWindow.cs

@@ -14,13 +14,14 @@ public class ConfigWindow : Window, IDisposable
     private bool _configIsScale;
     private float _configScale;
     private int[] _configWH = new int[2];
+    private bool _configIsDXVKDWMHackEnabled;
 
     public ConfigWindow() : base(
         "CustomResolution",
         ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar |
         ImGuiWindowFlags.NoScrollWithMouse)
     {
-        Size = new Vector2(430, 184);
+        Size = new Vector2(430, 212);
         SizeCondition = ImGuiCond.Always;
 
         UpdateFromConfig();
@@ -38,6 +39,7 @@ public class ConfigWindow : Window, IDisposable
         _configScale = config.Scale;
         _configWH[0] = (int) config.Width;
         _configWH[1] = (int) config.Height;
+        _configIsDXVKDWMHackEnabled = config.IsDXVKDWMHackEnabled;
     }
 
     public void UpdateToConfig()
@@ -49,6 +51,7 @@ public class ConfigWindow : Window, IDisposable
         config.Scale = _configScale;
         config.Width = (uint) _configWH[0];
         config.Height = (uint) _configWH[1];
+        config.IsDXVKDWMHackEnabled = _configIsDXVKDWMHackEnabled;
 
         config.Save();
     }
@@ -70,6 +73,11 @@ public class ConfigWindow : Window, IDisposable
 
         ImGui.Checkbox("Enabled", ref _configIsEnabled);
 
+        if (!_configIsEnabled)
+        {
+            ImGui.BeginDisabled();
+        }
+
         ImGui.Checkbox("Use scale", ref _configIsScale);
 
         if (_configIsScale)
@@ -84,6 +92,24 @@ public class ConfigWindow : Window, IDisposable
             ImGui.InputInt2("Size in pixels", ref _configWH[0]);
         }
 
+        if (!_configIsEnabled)
+        {
+            ImGui.EndDisabled();
+        }
+
+        ImGui.Checkbox("Apply borderless window workaround", ref _configIsDXVKDWMHackEnabled);
+        if (ImGui.IsItemHovered())
+        {
+            ImGui.SetTooltip(@"Fixes DXVK borderless window acting like exclusive fullscreen.
+In other words: Fixes black screen flashing when alt-tabbing.
+This can *possibly* impact performance, depending on your Windows version and GPU.
+Feel free to experiment with this toggle.
+
+Works even with the scaling above disabled.
+
+Not intended to be used with proper fullscreen.");
+        }
+
         if (ImGui.Button("Save and apply"))
         {
             save = true;