Update mouse transparency when the window shape changes

Fixes github.com/libsdl-org/SDL/issues/7617

src/video/cocoa/SDL_cocoashape.h

@@ -0,0 +1,28 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifndef SDL_cocoashape_h_
+#define SDL_cocoashape_h_
+
+extern int Cocoa_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape);
+
+#endif /* SDL_cocoashape_h_ */

src/video/cocoa/SDL_cocoashape.m

@@ -0,0 +1,54 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_COCOA
+
+#include "SDL_cocoavideo.h"
+#include "SDL_cocoashape.h"
+
+
+int Cocoa_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape)
+{
+    SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->driverdata;
+    BOOL ignoresMouseEvents = NO;
+
+    if (shape) {
+        SDL_FPoint point;
+        SDL_GetGlobalMouseState(&point.x, &point.y);
+        point.x -= window->x;
+        point.y -= window->y;
+        if (point.x >= 0.0f && point.x < window->w &&
+            point.y >= 0.0f && point.y < window->h) {
+            int x = (int)SDL_roundf((point.x / (window->w - 1)) * (shape->w - 1));
+            int y = (int)SDL_roundf((point.y / (window->h - 1)) * (shape->h - 1));
+            Uint8 a;
+
+            if (SDL_ReadSurfacePixel(shape, x, y, NULL, NULL, NULL, &a) < 0 || a == SDL_ALPHA_TRANSPARENT) {
+                ignoresMouseEvents = YES;
+            }
+        }
+    }
+    data.nswindow.ignoresMouseEvents = ignoresMouseEvents;
+    return 0;
+}
+
+#endif /* SDL_VIDEO_DRIVER_COCOA */

src/video/cocoa/SDL_cocoavideo.m

@@ -31,6 +31,7 @@
 #include "SDL_cocoametalview.h"
 #include "SDL_cocoaopengles.h"
 #include "SDL_cocoamessagebox.h"
+#include "SDL_cocoashape.h"
 
 @implementation SDL_CocoaVideoData
 
@@ -115,6 +116,7 @@ static SDL_VideoDevice *Cocoa_CreateDevice(void)
         device->DestroyWindow = Cocoa_DestroyWindow;
         device->SetWindowHitTest = Cocoa_SetWindowHitTest;
         device->AcceptDragAndDrop = Cocoa_AcceptDragAndDrop;
+        device->UpdateWindowShape = Cocoa_UpdateWindowShape;
         device->FlashWindow = Cocoa_FlashWindow;
         device->SetWindowFocusable = Cocoa_SetWindowFocusable;
         device->SyncWindow = Cocoa_SyncWindow;

src/video/x11/SDL_x11video.c

@@ -208,8 +208,8 @@ static SDL_VideoDevice *X11_CreateDevice(void)
     device->DestroyWindowFramebuffer = X11_DestroyWindowFramebuffer;
     device->SetWindowHitTest = X11_SetWindowHitTest;
     device->AcceptDragAndDrop = X11_AcceptDragAndDrop;
-    device->FlashWindow = X11_FlashWindow;
     device->UpdateWindowShape = X11_UpdateWindowShape;
+    device->FlashWindow = X11_FlashWindow;
     device->ShowWindowSystemMenu = X11_ShowWindowSystemMenu;
     device->SetWindowFocusable = X11_SetWindowFocusable;
     device->SyncWindow = X11_SyncWindow;