Added support for 0-copy decode and display using Apple VideoToolbox

test/CMakeLists.txt

@@ -184,7 +184,12 @@ endif()
 
 include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake")
 if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
-    add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
+    if(APPLE)
+        add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header})
+        target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal")
+    else()
+        add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
+    endif()
     target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
     if(HAVE_OPENGLES_V2)
         #message(STATUS "Enabling EGL support in testffmpeg")

test/testffmpeg.c

@@ -36,6 +36,10 @@
 #endif
 #endif
 
+#ifdef __APPLE__
+#include "testffmpeg_videotoolbox.h"
+#endif
+
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavutil/avutil.h>
@@ -64,6 +68,9 @@ static SDL_bool has_EGL_EXT_image_dma_buf_import;
 static PFNGLACTIVETEXTUREARBPROC glActiveTextureARBFunc;
 static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
 #endif
+#ifdef __APPLE__
+static SDL_bool has_videotoolbox_output;
+#endif
 static int done;
 
 static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
@@ -87,7 +94,7 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
     SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 6);
     SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 5);
 
-    if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, SDL_WINDOW_HIDDEN, &window, &renderer) < 0) {
+    if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, window_flags, &window, &renderer) < 0) {
         return SDL_FALSE;
     }
 
@@ -116,6 +123,10 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
     }
 #endif /* HAVE_EGL */
 
+#ifdef __APPLE__
+    has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
+#endif
+
     return SDL_TRUE;
 }
 
@@ -222,6 +233,11 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
         (format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) {
         return SDL_TRUE;
     }
+#ifdef __APPLE__
+    if (has_videotoolbox_output && format == AV_PIX_FMT_VIDEOTOOLBOX) {
+        return SDL_TRUE;
+    }
+#endif
 
     if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
         return SDL_TRUE;
@@ -474,11 +490,41 @@ static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
     }
 }
 
-static void HandleVideoFrame(AVFrame *frame, double pts)
+static void DisplayVideoTexture(AVFrame *frame)
 {
     /* Update the video texture */
     GetTextureForFrame(frame, &video_texture);
 
+    if (frame->linesize[0] < 0) {
+        SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
+    } else {
+        SDL_RenderTexture(renderer, video_texture, NULL, NULL);
+    }
+}
+
+static void DisplayVideoToolbox(AVFrame *frame)
+{
+#ifdef __APPLE__
+    SDL_Rect viewport;
+    SDL_GetRenderViewport(renderer, &viewport);
+    DisplayVideoToolboxFrame(renderer, frame->data[3], 0, 0, frame->width, frame->height, viewport.x, viewport.y, viewport.w, viewport.h);
+#endif
+}
+
+static void DisplayVideoFrame(AVFrame *frame)
+{
+    switch (frame->format) {
+    case AV_PIX_FMT_VIDEOTOOLBOX:
+        DisplayVideoToolbox(frame);
+        break;
+    default:
+        DisplayVideoTexture(frame);
+        break;
+    }
+}
+
+static void HandleVideoFrame(AVFrame *frame, double pts)
+{
     /* Quick and dirty PTS handling */
     if (!video_start) {
         video_start = SDL_GetTicks();
@@ -489,11 +535,7 @@ static void HandleVideoFrame(AVFrame *frame, double pts)
         now = (double)(SDL_GetTicks() - video_start) / 1000.0;
     }
 
-    if (frame->linesize[0] < 0) {
-        SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
-    } else {
-        SDL_RenderTexture(renderer, video_texture, NULL, NULL);
-    }
+    DisplayVideoFrame(frame);
 
     /* Render any bouncing balls */
     MoveSprite();
@@ -658,7 +700,7 @@ int main(int argc, char *argv[])
         goto quit;
     }
 
-    window_flags = SDL_WINDOW_HIDDEN;
+    window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_HIGH_PIXEL_DENSITY;
 #ifdef __APPLE__
     window_flags |= SDL_WINDOW_METAL;
 #elif !defined(__WIN32__)
@@ -832,6 +874,9 @@ int main(int argc, char *argv[])
     }
     return_code = 0;
 quit:
+#ifdef __APPLE__
+    CleanupVideoToolboxOutput();
+#endif
     SDL_free(positions);
     SDL_free(velocities);
     av_frame_free(&frame);

test/testffmpeg_videotoolbox.h

@@ -0,0 +1,15 @@
+/*
+  Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely.
+*/
+
+extern SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer);
+extern SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH );
+extern void CleanupVideoToolboxOutput();

test/testffmpeg_videotoolbox.m

@@ -0,0 +1,147 @@
+/*
+  Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely.
+*/
+#include <SDL3/SDL.h>
+
+#include "testffmpeg_videotoolbox.h"
+
+#include <CoreVideo/CoreVideo.h>
+#include <Metal/Metal.h>
+#include <QuartzCore/CAMetalLayer.h>
+#include <simd/simd.h>
+
+
+// Metal BT.601 to RGB conversion shader
+static NSString *drawMetalShaderSource =
+@"    using namespace metal;\n"
+"\n"
+"    struct Vertex\n"
+"    {\n"
+"        float4 position [[position]];\n"
+"        float2 texCoords;\n"
+"    };\n"
+"\n"
+"    constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);\n"
+"\n"
+"    vertex Vertex draw_vs(constant Vertex *vertices [[ buffer(0) ]], uint vid [[ vertex_id ]])\n"
+"    {\n"
+"        return vertices[ vid ];\n"
+"    }\n"
+"\n"
+"    fragment float4 draw_ps_bt601(Vertex in [[ stage_in ]],\n"
+"                                   texture2d<float> textureY [[ texture(0) ]],\n"
+"                                   texture2d<float> textureUV [[ texture(1) ]])\n"
+"    {\n"
+"        float3 yuv = float3(textureY.sample(s, in.texCoords).r, textureUV.sample(s, in.texCoords).rg);\n"
+"        float3 rgb;\n"
+"        yuv += float3(-0.0627451017, -0.501960814, -0.501960814);\n"
+"        rgb.r = dot(yuv, float3(1.1644,  0.000,   1.596));\n"
+"        rgb.g = dot(yuv, float3(1.1644, -0.3918, -0.813));\n"
+"        rgb.b = dot(yuv, float3(1.1644,  2.0172,  0.000));\n"
+"        return float4(rgb, 1.0);\n"
+"    }\n"
+;
+
+// keep this structure aligned with the proceeding drawMetalShaderSource's struct Vertex
+typedef struct Vertex
+{
+    vector_float4 position;
+    vector_float2 texCoord;
+} Vertex;
+
+static void SetVertex(Vertex *vertex, float x, float y, float s, float t)
+{
+    vertex->position[ 0 ] = x;
+    vertex->position[ 1 ] = y;
+    vertex->position[ 2 ] = 0.0f;
+    vertex->position[ 3 ] = 1.0f;
+    vertex->texCoord[ 0 ] = s;
+    vertex->texCoord[ 1 ] = t;
+}
+
+static CAMetalLayer *metal_layer;
+static id<MTLLibrary> library;
+static id<MTLRenderPipelineState> video_pipeline;
+
+SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer)
+{ @autoreleasepool {
+    NSError *error;
+    
+    // Create the metal view
+    metal_layer = (CAMetalLayer *)SDL_GetRenderMetalLayer(renderer);
+    if (!metal_layer) {
+        return SDL_FALSE;
+    }
+
+    // FIXME: Handle other colorspaces besides BT.601
+    library = [metal_layer.device newLibraryWithSource:drawMetalShaderSource options:nil error:&error];
+
+    MTLRenderPipelineDescriptor *videoPipelineDescriptor = [[MTLRenderPipelineDescriptor new] autorelease];
+    videoPipelineDescriptor.vertexFunction = [library newFunctionWithName:@"draw_vs"];
+    videoPipelineDescriptor.fragmentFunction = [library newFunctionWithName:@"draw_ps_bt601"];
+    videoPipelineDescriptor.colorAttachments[ 0 ].pixelFormat = metal_layer.pixelFormat;
+
+    video_pipeline = [metal_layer.device newRenderPipelineStateWithDescriptor:videoPipelineDescriptor error:nil];
+    if (!video_pipeline) {
+        SDL_SetError("Couldn't create video pipeline");
+        return SDL_FALSE;
+    }
+
+    return true;
+}}
+
+SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH )
+{ @autoreleasepool {
+    CVPixelBufferRef pPixelBuffer = (CVPixelBufferRef)buffer;
+    size_t nPixelBufferWidth = CVPixelBufferGetWidthOfPlane(pPixelBuffer, 0);
+    size_t nPixelBufferHeight = CVPixelBufferGetHeightOfPlane(pPixelBuffer, 0);
+    id<MTLTexture> videoFrameTextureY = nil;
+    id<MTLTexture> videoFrameTextureUV = nil;
+
+    IOSurfaceRef pSurface = CVPixelBufferGetIOSurface(pPixelBuffer);
+
+    MTLTextureDescriptor *textureDescriptorY = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm width:nPixelBufferWidth height:nPixelBufferHeight mipmapped:NO];
+    MTLTextureDescriptor *textureDescriptorUV = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRG8Unorm width:CVPixelBufferGetWidthOfPlane(pPixelBuffer, 1) height:CVPixelBufferGetHeightOfPlane(pPixelBuffer, 1) mipmapped:NO];
+
+    videoFrameTextureY = [[metal_layer.device newTextureWithDescriptor:textureDescriptorY iosurface:pSurface plane:0] autorelease];
+    videoFrameTextureUV = [[metal_layer.device newTextureWithDescriptor:textureDescriptorUV iosurface:pSurface plane:1] autorelease];
+
+    float flMinSrcX = ( srcX + 0.5f ) / nPixelBufferWidth;
+    float flMaxSrcX = ( srcX + srcW + 0.5f ) / nPixelBufferWidth;
+    float flMinSrcY = ( srcY + 0.5f ) / nPixelBufferHeight;
+    float flMaxSrcY = ( srcY + srcH + 0.5f ) / nPixelBufferHeight;
+
+    int nOutputWidth, nOutputHeight;
+    nOutputWidth = metal_layer.drawableSize.width;
+    nOutputHeight = metal_layer.drawableSize.height;
+    float flMinDstX = 2.0f * ( ( dstX + 0.5f ) / nOutputWidth ) - 1.0f;
+    float flMaxDstX = 2.0f * ( ( dstX + dstW + 0.5f ) / nOutputWidth ) - 1.0f;
+    float flMinDstY = 2.0f * ( ( nOutputHeight - dstY - 0.5f ) / nOutputHeight ) - 1.0f;
+    float flMaxDstY = 2.0f * ( ( nOutputHeight - ( dstY + dstH ) - 0.5f ) / nOutputHeight ) - 1.0f;
+
+    Vertex arrVerts[4];
+    SetVertex(&arrVerts[0], flMinDstX, flMaxDstY, flMinSrcX, flMaxSrcY);
+    SetVertex(&arrVerts[1], flMinDstX, flMinDstY, flMinSrcX, flMinSrcY);
+    SetVertex(&arrVerts[2], flMaxDstX, flMaxDstY, flMaxSrcX, flMaxSrcY);
+    SetVertex(&arrVerts[3], flMaxDstX, flMinDstY, flMaxSrcX, flMinSrcY);
+
+    id<MTLRenderCommandEncoder> renderEncoder = (id<MTLRenderCommandEncoder>)SDL_GetRenderMetalCommandEncoder(renderer);
+    [renderEncoder setRenderPipelineState:video_pipeline];
+    [renderEncoder setFragmentTexture:videoFrameTextureY atIndex:0];
+    [renderEncoder setFragmentTexture:videoFrameTextureUV atIndex:1];
+    [renderEncoder setVertexBytes:arrVerts length:sizeof(arrVerts) atIndex:0];
+    [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:SDL_arraysize(arrVerts)];
+    return SDL_TRUE;
+}}
+
+void CleanupVideoToolboxOutput()
+{
+}