[WINLOGON] Workaround buggy 3rd-party DLLs that use a wrong notification handler calling convention (#8640)

CORE-20279

PRELIMINARY REMARK: The described bug and code workaround only applies
for x86 32-bit builds.

----

While the Winlogon notification handlers[^1] actually use a `STDCALL`
calling convention, which can be trivially verified by debugging the
official Windows <= 2003 winlogon.exe and its notification extensions,
there exist 3rd-party Winlogon notification DLLs, like the `Ati2evxx.dll`
one from AMD/ATI XP video drivers, that use a `CDECL` calling convention,
or an invalid number (zero) of parameters.

I think the reason why this happens is as follows.
The official documentation[^1] indicates that the handlers have the
following prototype:
```c
void Event_Handler_Function_Name(
  _In_ PWLX_NOTIFICATION_INFO pInfo
);
```
The documentation (and possibly the internal header Windows is using for
Winlogon) is sloppy, because it doesn't tell whether the convention is
`STDCALL` or `CDECL`. When compiling routines with such a signature, the
compiler will employ whatever default convention it is set to use.

Windows code is typically compiled with `STDCALL` convention as the default
(see e.g. how the Windows Development Kit is set up), thus, such a
function signature would default to `STDCALL`. Observation (with debugger)
shows that it is what Windows' winlogon.exe is indeed expecting.

However, 3rd-party code using a different development environment, could
set the compiler to use `CDECL` as the default calling convention. As a
result, the function signature from above would use `CDECL` instead.

The difference between the `STDCALL` and `CDECL` conventions is how the
function parameters are passed on the stack and how the stack is cleaned
at the end (`STDCALL`: the function unwinds the stack; `CDECL`: the caller
does it). A calling convention mismatch would therefore corrupt the stack,
and this is exactly what happens with the `Ati2evxx.dll` from the AMD/ATI
drivers, see CORE-20279.

The ReactOS Winlogon crashes from the `_RTC_Failure()` handler just after
the 3rd-party handler returns, since we compile our code with runtime checks
enabled. Windows' winlogon.exe doesn't apparently crash, because neither
in Release nor in Checked/Debug mode did they compile winlogon.exe with
RTC enabled. However, its stack would become more corrupt with time.

In order to alleviate this in ReactOS' winlogon.exe, I decided to use
a "generic" workaround, manually calling the handler with inline ASM
(which is OK since the problem and solution is x86-specific only).
It does something similar to what the RTC support does: it checks the
stack pointer after the call and restores it if needed.
An informative message is then emitted in the debugger telling which DLL
is buggy and needs to be fixed.

[^1]: https://learn.microsoft.com/en-us/windows/win32/secauthn/event-handler-function-prototype

base/system/winlogon/notify.c

@@ -6,11 +6,11 @@
  * PROGRAMMERS:     Eric Kohl
  */
 
-/* INCLUDES *****************************************************************/
+/* INCLUDES ******************************************************************/
 
 #include "winlogon.h"
 
-/* GLOBALS ******************************************************************/
+/* GLOBALS *******************************************************************/
 
 typedef VOID (WINAPI *PWLX_NOTIFY_HANDLER)(PWLX_NOTIFICATION_INFO pInfo);
 
@@ -45,10 +45,8 @@ typedef struct _NOTIFICATION_ITEM
     PWLX_NOTIFY_HANDLER Handler[LastHandler];
 } NOTIFICATION_ITEM, *PNOTIFICATION_ITEM;
 
-
 static LIST_ENTRY NotificationDllListHead;
 
-
 /* FUNCTIONS *****************************************************************/
 
 /**
@@ -415,7 +413,6 @@ done:
     RegCloseKey(hDllKey);
 }
 
-
 BOOL
 InitNotifications(VOID)
 {
@@ -467,7 +464,6 @@ InitNotifications(VOID)
     return TRUE;
 }
 
-
 static
 VOID
 CallNotificationDll(
@@ -511,7 +507,55 @@ CallNotificationDll(
     /* Call the notification handler in SEH to prevent any Winlogon crashes */
     _SEH2_TRY
     {
+#ifdef _M_IX86 // CORE-20279
+        /* Workaround for buggy 3rd-party notification DLLs: Handle broken ones
+         * that use a CDECL calling convention instead of the correct STDCALL */
+        BOOLEAN Success;
+    #if defined(__GNUC__)
+        register ULONG_PTR StackPtr;
+        __asm__ __volatile__
+        (
+            "movl %%esp, %[StackPtr]\n\t"   // Save current ESP
+            /*"leal %[Info], %%eax\n\t"       // Push parameter
+            "pushl %%eax\n\t"*/ "pushl %[Info]\n\t"
+            "call *%[pNotifyHandler]\n\t"   // Invoke STDCALL notification handler
+            "cmpl %%esp, %[StackPtr]\n\t"   // Check whether ESP is messed up
+            "je 1f\n\t"                     // Exit if everything is fine
+            "movl %[StackPtr], %%esp\n\t"   // Restore correct ESP
+            "1:\n\t"
+            //"seteb %[Success]"              // Set success or failure
+            :
+              [StackPtr]"=&S"(StackPtr), [Success]/*"=rm"*/"=@cce"(Success)
+            :
+              [pNotifyHandler]"m"(pNotifyHandler), [Info]/*"m"(Info)*/"r"(&Info)
+            :
+              /*"%esp", "%eax",*/ "cc", "memory"
+        );
+    #elif defined(_MSC_VER) // && !defined(__clang__)
+        __asm
+        {
+            mov esi, esp                // Save current ESP
+            lea eax, dword ptr [Info]   // Push parameter
+            push eax
+            call [pNotifyHandler]       // Invoke STDCALL notification handler
+            cmp esi, esp                // Check whether ESP is messed up
+            je l1f                      // Exit if everything is fine
+            mov esp, esi                // Restore correct ESP
+            l1f:
+            sete byte ptr [Success]     // Set success or failure
+        }
+    #else
+    #error Unsupported compiler
+    #endif
+        if (!Success)
+        {
+            ERR("WL: The notification DLL '%ws' uses a wrong calling convention or number of parameters. "
+                "Please contact your software vendor for a fixed DLL!\n",
+                NotificationDll->pszDllName);
+        }
+#else
         pNotifyHandler(&Info);
+#endif // _M_IX86
     }
     _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
     {
@@ -525,7 +569,6 @@ CallNotificationDll(
         RevertToSelf();
 }
 
-
 VOID
 CallNotificationDlls(
     PWLSESSION pSession,
@@ -588,7 +631,6 @@ CallNotificationDlls(
     }
 }
 
-
 VOID
 CleanupNotifications(VOID)
 {