Allow switching between different borderless window hacks

CustomResolution/Cmds/BorderlessWindowedCmd.cs

@@ -1,47 +0,0 @@
-using static FFXIVClientStructs.FFXIV.Client.UI.AddonRelicNoteBook;
-using System.Collections;
-using System.Globalization;
-using System;
-
-namespace CustomResolution.Cmds;
-
-public sealed class BorderlessWindowedCmd : Cmd
-{
-    public override string Name => "cresbw";
-
-    public override string HelpMessage => $"Tweak the \"Apply borderless window workaround\" toggle.\n" +
-        $"\tExamples:\n" +
-        $"\tTo enable / disable it:\n\t\t{FullName} on\n\t\t{FullName} off\n\t\t{FullName} toggle";
-
-    public override void Run(string arguments)
-    {
-        if (string.IsNullOrEmpty(arguments))
-        {
-            Service.PrintChat("Invalid parameters.");
-            return;
-        }
-
-        switch (arguments.ToLowerInvariant())
-        {
-            case "on":
-                Service.Config.IsDXVKDWMHackEnabled = true;
-                Service.Config.Save();
-                Service.PrintChat("Enabled borderless window workaround.");
-                return;
-
-            case "off":
-                Service.Config.IsDXVKDWMHackEnabled = false;
-                Service.Config.Save();
-                Service.PrintChat("Disabled borderless window workaround.");
-                return;
-
-            case "toggle":
-                Service.Config.IsDXVKDWMHackEnabled = !Service.Config.IsDXVKDWMHackEnabled;
-                Service.Config.Save();
-                Service.PrintChat($"{(Service.Config.IsDXVKDWMHackEnabled ? "Enabled" : "Disabled")} borderless window workaround.");
-                return;
-        }
-
-        Service.PrintChat("Invalid parameters.");
-    }
-}

CustomResolution/Cmds/MainCmd.cs

@@ -64,6 +64,21 @@ public sealed class MainCmd : Cmd
                 Service.Config.Save();
                 Service.PrintChat($"{(Service.Config.IsEnabled ? "Enabled" : "Disabled")} custom resolution.");
                 return;
+
+            case "debugon":
+                Service.Plugin.IsDebug = true;
+                Service.PrintChat("Enabled cres debug.");
+                return;
+
+            case "debugoff":
+                Service.Plugin.IsDebug = false;
+                Service.PrintChat("Disabled cres debug.");
+                return;
+
+            case "debug":
+                Service.Plugin.IsDebug = !Service.Plugin.IsDebug;
+                Service.PrintChat($"{(Service.Plugin.IsDebug ? "Enabled" : "Disabled")} cres debug.");
+                return;
         }
 
         if (!float.TryParse(arguments, CultureInfo.InvariantCulture, out float value))

CustomResolution/Configuration.cs

@@ -15,7 +15,7 @@ public class Configuration : IPluginConfiguration
     public uint Width = 1024;
     public uint Height = 1024;
 
-    public bool IsDXVKDWMHackEnabled = false;
+    public DXVKDWMHackMode DXVKDWMHackMode = DXVKDWMHackMode.Off;
 
     [NonSerialized]
     private DalamudPluginInterface? pluginInterface;
@@ -30,3 +30,29 @@ public class Configuration : IPluginConfiguration
         pluginInterface!.SavePluginConfig(this);
     }
 }
+
+// Must explicitly map to integer values, blame Newtonsoft.JSON
+public enum DXVKDWMHackMode
+{
+    Off = 0,
+    UnsetPopup = 1,
+    SetClientEdge = 2
+}
+
+public static class DXVKDWMHackModeExt
+{
+    public static string ToHumanNameString(this DXVKDWMHackMode mode) => mode switch
+    {
+        DXVKDWMHackMode.Off => "Off",
+        DXVKDWMHackMode.UnsetPopup => "Best case: Disable WS_POPUP",
+        DXVKDWMHackMode.SetClientEdge => "Worst case: Enable WS_EX_CLIENTEDGE",
+        _ => mode.ToString(),
+    };
+
+    public static string? ToHumanInfoString(this DXVKDWMHackMode mode) => mode switch
+    {
+        DXVKDWMHackMode.UnsetPopup => "Works best with NVIDIA GPUs.",
+        DXVKDWMHackMode.SetClientEdge => "Adds a 2 pixel border around the game.",
+        _ => null
+    };
+}

CustomResolution/CustomResolution.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors>0x0ade</Authors>
     <Company></Company>
-    <Version>0.1.0.5</Version>
+    <Version>0.2.0.0</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl></PackageProjectUrl>

CustomResolution/Hooks/WndProcHook.cs

@@ -4,16 +4,14 @@ using System;
 using System.Linq;
 using System.Reflection.Emit;
 using TerraFX.Interop.Windows;
+using static TerraFX.Interop.Windows.Windows;
 
 
 namespace CustomResolution.Hooks;
 
 // THIS. IS. UGLY.
-public sealed class WndProcHook : IDisposable
+public sealed unsafe class WndProcHook : IDisposable
 {
-    private const uint WM_MOUSEFIRST = 0x0200;
-    private const uint WM_MOUSELAST = 0x0209;
-
     private static WndProcHook? _instance;
 
     private readonly WndProcHookManager _manager;
@@ -75,18 +73,20 @@ public sealed class WndProcHook : IDisposable
 
     private void Invoke(WndProcEventArgs args)
     {
-        if (!(WM_MOUSEFIRST <= args.Message && args.Message <= WM_MOUSELAST))
+        if (Service.Plugin is not { } plugin)
         {
             return;
         }
 
+        if (WM.WM_MOUSEFIRST <= args.Message && args.Message <= WM.WM_MOUSELAST)
+        {
             ParamToCoords(args.LParam, out int x, out int y);
 
 #if false
             Service.PluginLog.Debug($"WM_MOUSE A @ {x} {y}");
 #endif
 
-        Service.Plugin.ConvertCoordsWinToGame(ref x, ref y);
+            plugin.ConvertCoordsWinToGame(ref x, ref y);
 
 #if false
             Service.PluginLog.Debug($"WM_MOUSE B @ {x} {y}");
@@ -95,6 +95,25 @@ public sealed class WndProcHook : IDisposable
             args.LParam = CoordsToParam(x, y);
         }
 
+        if (args.Message == WM.WM_NCCALCSIZE && args.WParam != 0 && plugin.CurrentBorderlessFullscreen &&
+            Service.Config.DXVKDWMHackMode >= DXVKDWMHackMode.SetClientEdge)
+        {
+            NCCALCSIZE_PARAMS* ncsize = (NCCALCSIZE_PARAMS*) args.LParam;
+            MONITORINFO monitorInfo = new()
+            {
+                cbSize = (uint) sizeof(MONITORINFO)
+            };
+
+            if (MonitorFromWindow(plugin.CurrentHWND, MONITOR.MONITOR_DEFAULTTONEAREST) is { } monitor && monitor != HMONITOR.NULL &&
+                GetMonitorInfo(monitor, &monitorInfo))
+            {
+                ncsize->rgrc[0] = monitorInfo.rcMonitor;
+
+                args.SuppressCall = true;
+            }
+        }
+    }
+
     private void Apply()
     {
         if (!_manager.Refresh())

CustomResolution/Plugin.cs

@@ -22,7 +22,7 @@ public sealed unsafe class Plugin : IDalamudPlugin
     private HWND _currentHwnd;
     private RECT _currentClientRect;
     private RECT _currentWindowRect;
-    private bool _currentDXVKDWMHack = false;
+    private DXVKDWMHackMode _currentDXVKDWMHackMode = DXVKDWMHackMode.Off;
 
     public Plugin([RequiredVersion("1.0")] DalamudPluginInterface pluginInterface)
     {
@@ -55,11 +55,17 @@ public sealed unsafe class Plugin : IDalamudPlugin
 
     public string Name => "CustomResolution";
 
+    public HWND CurrentHWND { get; private set; }
+
     public uint CurrentWidth { get; private set; }
     public uint CurrentHeight { get; private set; }
     public uint CurrentWindowWidth { get; private set; }
     public uint CurrentWindowHeight { get; private set; }
 
+    public bool CurrentBorderlessFullscreen { get; private set; }
+
+    public bool IsDebug { get; set; }
+
     public void Dispose()
     {
         _tickCount = 0;
@@ -204,10 +210,43 @@ public sealed unsafe class Plugin : IDalamudPlugin
             dev->RequestResolutionChange = 1;
         }
 
-        if (Service.Config.IsDXVKDWMHackEnabled && !_unloading)
+        // TODO: This isn't accurate! Figure out how to read the game's settings instead.
+        CurrentBorderlessFullscreen = (GetWindowLong(_currentHwnd, GWL.GWL_STYLE) & WS.WS_SYSMENU) == 0;
+
+        if (Service.Config.DXVKDWMHackMode != DXVKDWMHackMode.Off && !_unloading)
         {
-#if false
+            SetDXVKDWMHack(Service.Config.DXVKDWMHackMode);
-            Service.PluginLog.Info($"STYLE: 0x{GetWindowLong(_currentHwnd, GWL.GWL_STYLE):X8}");
+        }
+        else if (Service.Config.DXVKDWMHackMode == DXVKDWMHackMode.Off && _currentDXVKDWMHackMode != DXVKDWMHackMode.Off)
+        {
+            SetDXVKDWMHack(DXVKDWMHackMode.Off);
+        }
+
+        CurrentWidth = width;
+        CurrentHeight = height;
+        CurrentWindowWidth = (uint) rectWidth;
+        CurrentWindowHeight = (uint) rectHeight;
+    }
+
+    private void SetDXVKDWMHack(DXVKDWMHackMode mode)
+    {
+        /* Default maximized style / exstyle is 0x95000000 / 0.
+         * WS.WS_POPUP | WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_MAXIMIZE
+         * Default windowed style / exstyle is 0x14CF0000 / 0.
+         * WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_CAPTION | WS.WS_SYSMENU | WS.WS_THICKFRAME | WS.WS_MINIMIZEBOX | WS.WS_MAXIMIZEBOX
+         */
+        uint styleOrig = (uint) GetWindowLong(_currentHwnd, GWL.GWL_STYLE);
+        uint exstyleOrig = (uint) GetWindowLong(_currentHwnd, GWL.GWL_EXSTYLE);
+
+        uint style = styleOrig;
+        uint exstyle = exstyleOrig;
+
+        bool fullscreen = (style & WS.WS_SYSMENU) == 0;
+
+        if (IsDebug)
+        {
+            Service.PluginLog.Info("--------");
+            Service.PluginLog.Info($"STYLE: 0x{style:X8}");
             Service.PluginLog.Info($"EXSTYLE: 0x{GetWindowLong(_currentHwnd, GWL.GWL_EXSTYLE):X8}");
 
             Span<ushort> name = stackalloc ushort[256];
@@ -217,41 +256,11 @@ public sealed unsafe class Plugin : IDalamudPlugin
 
             Service.PluginLog.Info($"CLASS: {new string((char*) name.GetPointer(0))}");
             Service.PluginLog.Info($"CLASS.style: 0x{wce.style:X8}");
-#endif
-
-            SetDXVKDWMHack(true);
         }
-        else if (!Service.Config.IsDXVKDWMHackEnabled && _currentDXVKDWMHack)
-        {
-            SetDXVKDWMHack(false);
-        }
-
-        _currentDXVKDWMHack = Service.Config.IsDXVKDWMHackEnabled;
-
-        CurrentWidth = width;
-        CurrentHeight = height;
-        CurrentWindowWidth = (uint) rectWidth;
-        CurrentWindowHeight = (uint) rectHeight;
-    }
-
-    private void SetDXVKDWMHack(bool enabled)
-    {
-        /* Default maximized style / exstyle is 0x95000000 / 0.
-         * WS.WS_POPUP | WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_MAXIMIZE
-         * Default windowed style / exstyle is 0x14CF0000 / 0.
-         * WS.WS_VISIBLE | WS.WS_CLIPSIBLINGS | WS.WS_CAPTION | WS.WS_SYSMENU | WS.WS_THICKFRAME | WS.WS_MINIMIZEBOX | WS.WS_MAXIMIZEBOX
-         */
-
-        uint style = (uint) GetWindowLong(_currentHwnd, GWL.GWL_STYLE);
-        bool fullscreen = (style & WS.WS_SYSMENU) == 0;
-
-        /* Alternative hacks:
-         * - WS_EX_CLIENTEDGE, at the cost of having a 2px border on each side.
-         */
 
         if (fullscreen)
         {
-            if (enabled)
+            if (mode >= DXVKDWMHackMode.UnsetPopup)
             {
                 style &= ~WS.WS_POPUP;
             }
@@ -261,7 +270,42 @@ public sealed unsafe class Plugin : IDalamudPlugin
             }
         }
 
+        if (fullscreen && mode >= DXVKDWMHackMode.SetClientEdge)
+        {
+            exstyle |= WS.WS_EX_CLIENTEDGE;
+            exstyle |= WS.WS_EX_COMPOSITED;
+        }
+        else
+        {
+            exstyle &= ~(uint) WS.WS_EX_CLIENTEDGE;
+            exstyle &= ~(uint) WS.WS_EX_COMPOSITED;
+        }
+
+        if (IsDebug)
+        {
+            Service.PluginLog.Info($"NEWSTYLE: 0x{style:X8}");
+            Service.PluginLog.Info($"NEWEXSTYLE: 0x{exstyle:X8}");
+        }
+
+        if (style != styleOrig || exstyle != exstyleOrig)
+        {
+            if (IsDebug)
+            {
+                Service.PluginLog.Info("UPDATE");
+            }
+
             SetWindowLong(_currentHwnd, GWL.GWL_STYLE, (int) style);
+            SetWindowLong(_currentHwnd, GWL.GWL_EXSTYLE, (int) exstyle);
+
+            SetWindowPos(_currentHwnd, HWND.NULL, 0, 0, 0, 0, SWP.SWP_NOZORDER | SWP.SWP_NOMOVE | SWP.SWP_NOSIZE | SWP.SWP_NOACTIVATE | SWP.SWP_DRAWFRAME);
+            ShowWindow(_currentHwnd, SW.SW_SHOW);
+        }
+        else if (IsDebug)
+        {
+            Service.PluginLog.Info("SAME");
+        }
+
+        _currentDXVKDWMHackMode = mode;
     }
 
     private void OnFrameworkUpdate(IFramework framework)

CustomResolution/Windows/ConfigWindow.cs

@@ -14,16 +14,13 @@ public class ConfigWindow : Window, IDisposable
     private bool _configIsScale;
     private float _configScale;
     private int[] _configWH = new int[2];
-    private bool _configIsDXVKDWMHackEnabled;
+    private DXVKDWMHackMode _configDXVKDWMHackMode;
 
     public ConfigWindow() : base(
         "CustomResolution",
         ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar |
-        ImGuiWindowFlags.NoScrollWithMouse)
+        ImGuiWindowFlags.NoScrollWithMouse | ImGuiWindowFlags.AlwaysAutoResize)
     {
-        Size = new Vector2(430, 212);
-        SizeCondition = ImGuiCond.Always;
-
         UpdateFromConfig();
     }
 
@@ -39,7 +36,7 @@ public class ConfigWindow : Window, IDisposable
         _configScale = config.Scale;
         _configWH[0] = (int) config.Width;
         _configWH[1] = (int) config.Height;
-        _configIsDXVKDWMHackEnabled = config.IsDXVKDWMHackEnabled;
+        _configDXVKDWMHackMode = config.DXVKDWMHackMode;
     }
 
     public void UpdateToConfig()
@@ -51,7 +48,7 @@ public class ConfigWindow : Window, IDisposable
         config.Scale = _configScale;
         config.Width = (uint) _configWH[0];
         config.Height = (uint) _configWH[1];
-        config.IsDXVKDWMHackEnabled = _configIsDXVKDWMHackEnabled;
+        config.DXVKDWMHackMode = _configDXVKDWMHackMode;
 
         config.Save();
     }
@@ -97,13 +94,31 @@ public class ConfigWindow : Window, IDisposable
             ImGui.EndDisabled();
         }
 
-        ImGui.Checkbox("Apply borderless window workaround", ref _configIsDXVKDWMHackEnabled);
+        if (ImGui.BeginCombo("Borderless window workaround", _configDXVKDWMHackMode.ToHumanNameString()))
+        {
+            foreach (var mode in Enum.GetValues<DXVKDWMHackMode>())
+            {
+                if (ImGui.Selectable(mode.ToHumanNameString(), _configDXVKDWMHackMode == mode, ImGuiSelectableFlags.None))
+                {
+                    _configDXVKDWMHackMode = mode;
+                }
+
+                if (ImGui.IsItemHovered() && mode.ToHumanInfoString() is { } info)
+                {
+                    ImGui.SetTooltip(info);
+                }
+            }
+
+            ImGui.EndCombo();
+        }
+
         if (ImGui.IsItemHovered())
         {
-            ImGui.SetTooltip(@"Fixes DXVK borderless window acting like exclusive fullscreen.
+            ImGui.SetTooltip(@"Fixes DXVK borderless window causing black screens when alt-tabbing.
-In other words: Fixes black screen flashing when alt-tabbing.
 This can *possibly* impact performance, depending on your Windows version and GPU.
+
 Feel free to experiment with this toggle.
+Make sure to switch the game to windowed and then back to borderless windowed when changing.
 
 Works even with the scaling above disabled.