From aa3781fde9b9f4037cf3b3cd1dce9af12fe2281b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 4 Feb 2022 21:32:04 +0100 Subject: [PATCH] mac-virtualcam: Use IOSurface to share output with virtual cameras This change updates the virtual camera implementation on macOS to utilize IOSurface to share the output feed with the virtual cameras. By using IOSurface, we remove the need for copying the frames across multiple buffers, since they can be shared across Mach connections using zero-copy. --- .../src/dal-plugin/CMakeLists.txt | 8 ++- .../src/dal-plugin/OBSDALMachClient.h | 10 ++-- .../src/dal-plugin/OBSDALMachClient.mm | 42 ++++++++------ .../src/dal-plugin/OBSDALPlugIn.mm | 25 +++++---- .../src/dal-plugin/OBSDALStream.h | 10 ++-- .../src/dal-plugin/OBSDALStream.mm | 40 +++++++++++--- .../src/obs-plugin/CMakeLists.txt | 9 ++- .../src/obs-plugin/OBSDALMachServer.h | 10 ++-- .../src/obs-plugin/OBSDALMachServer.mm | 39 ++++++------- .../src/obs-plugin/plugin-main.mm | 55 ++++++++++++++++--- 10 files changed, 161 insertions(+), 87 deletions(-) diff --git a/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt b/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt index f7747b680..655fbd76c 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt +++ b/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt @@ -4,8 +4,9 @@ find_library(COCOA Cocoa) find_library(COREMEDIA CoreMedia) find_library(COREMEDIAIO CoreMediaIO) find_library(COREVIDEO CoreVideo) +find_library(IOSURFACE IOSurface) -mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO) +mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO IOSURFACE) add_library(mac-dal-plugin MODULE) add_library(OBS::mac-dal-plugin ALIAS mac-dal-plugin) @@ -37,8 +38,9 @@ target_include_directories( target_compile_options(mac-dal-plugin PRIVATE -fobjc-arc -fobjc-weak) -target_link_libraries(mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA} - ${COREMEDIAIO} ${COREVIDEO}) +target_link_libraries( + mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA} ${COREMEDIAIO} ${COREVIDEO} + ${IOSURFACE}) set(MACOSX_PLUGIN_BUNDLE_TYPE "BNDL") target_sources(mac-dal-plugin PRIVATE placeholder.png) diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h index 54ba4bee3..3323432d4 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h @@ -6,16 +6,16 @@ // #import +#import NS_ASSUME_NONNULL_BEGIN @protocol MachClientDelegate -- (void)receivedFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData; +- (void)receivedPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; - (void)receivedStop; @end diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm index 8279cf330..452ee6fbc 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm @@ -101,29 +101,37 @@ break; case MachMsgIdFrame: VLog(@"Received frame message"); - if (components.count >= 6) { - CGFloat width; - [components[0] getBytes:&width length:sizeof(width)]; - CGFloat height; - [components[1] getBytes:&height length:sizeof(height)]; + if (components.count >= 4) { + NSMachPort *framePort = (NSMachPort *)components[0]; + IOSurfaceRef surface = IOSurfaceLookupFromMachPort( + [framePort machPort]); + + CVPixelBufferRef frame; + CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, + surface, NULL, &frame); + uint64_t timestamp; - [components[2] getBytes:×tamp + [components[1] getBytes:×tamp length:sizeof(timestamp)]; - VLog(@"Received frame data: %fx%f (%llu)", width, - height, timestamp); - NSData *frameData = components[3]; + + VLog(@"Received frame data: %zux%zu (%llu)", + CVPixelBufferGetWidth(frame), + CVPixelBufferGetHeight(frame), timestamp); + uint32_t fpsNumerator; - [components[4] getBytes:&fpsNumerator + [components[2] getBytes:&fpsNumerator length:sizeof(fpsNumerator)]; uint32_t fpsDenominator; - [components[5] getBytes:&fpsDenominator + [components[3] getBytes:&fpsDenominator length:sizeof(fpsDenominator)]; - [self.delegate - receivedFrameWithSize:NSMakeSize(width, height) - timestamp:timestamp - fpsNumerator:fpsNumerator - fpsDenominator:fpsDenominator - frameData:frameData]; + + [self.delegate receivedPixelBuffer:frame + timestamp:timestamp + fpsNumerator:fpsNumerator + fpsDenominator:fpsDenominator]; + + CVPixelBufferRelease(frame); + CFRelease(surface); } break; case MachMsgIdStop: diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm index c8aa75535..c1dcaa36c 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm @@ -203,19 +203,21 @@ typedef enum { #pragma mark - MachClientDelegate -- (void)receivedFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData +- (void)receivedPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { + size_t width = CVPixelBufferGetWidth(frame); + size_t height = CVPixelBufferGetHeight(frame); + dispatch_sync(_stateQueue, ^{ if (_state == PlugInStateWaitingForServer) { NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults]; - [defaults setInteger:size.width + [defaults setInteger:(long)width forKey:kTestCardWidthKey]; - [defaults setInteger:size.height + [defaults setInteger:(long)height forKey:kTestCardHeightKey]; [defaults setDouble:(double)fpsNumerator / (double)fpsDenominator @@ -234,11 +236,10 @@ typedef enum { dispatch_time(DISPATCH_TIME_NOW, 5.0 * NSEC_PER_SEC), 5.0 * NSEC_PER_SEC, (1ull * NSEC_PER_SEC) / 10); - [self.stream queueFrameWithSize:size - timestamp:timestamp - fpsNumerator:fpsNumerator - fpsDenominator:fpsDenominator - frameData:frameData]; + [self.stream queuePixelBuffer:frame + timestamp:timestamp + fpsNumerator:fpsNumerator + fpsDenominator:fpsDenominator]; } - (void)receivedStop diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h index d660b71c2..9d841ecea 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h @@ -18,6 +18,7 @@ // along with obs-mac-virtualcam. If not, see . #import "OBSDALObjectStore.h" +#import NS_ASSUME_NONNULL_BEGIN @@ -35,11 +36,10 @@ NS_ASSUME_NONNULL_BEGIN - (void)stopServingDefaultFrames; -- (void)queueFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData; +- (void)queuePixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; @end diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm index 730bb4cc2..fa19bc6ac 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm @@ -349,11 +349,10 @@ } } -- (void)queueFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData +- (void)queuePixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { if (CMSimpleQueueGetFullness(self.queue) >= 1.0) { DLog(@"Queue is full, bailing out"); @@ -374,9 +373,34 @@ self.sequenceNumber = CMIOGetNextSequenceNumber(self.sequenceNumber); CMSampleBufferRef sampleBuffer; - CMSampleBufferCreateFromData(size, timingInfo, self.sequenceNumber, - frameData, &sampleBuffer); - CMSimpleQueueEnqueue(self.queue, sampleBuffer); + + // Generate the video format description from that pixel buffer + CMVideoFormatDescriptionRef format; + err = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, + frame, &format); + if (err != noErr) { + DLog(@"CMVideoFormatDescriptionCreateForImageBuffer err %d", + err); + return; + } + + err = CMIOSampleBufferCreateForImageBuffer( + kCFAllocatorDefault, frame, format, &timingInfo, + self.sequenceNumber, kCMIOSampleBufferNoDiscontinuities, + &sampleBuffer); + + CFRelease(format); + + if (err != noErr) { + DLog(@"CMIOSampleBufferCreateForImageBuffer err %d", err); + return; + } + + err = CMSimpleQueueEnqueue(self.queue, sampleBuffer); + if (err != noErr) { + DLog(@"CMSimpleQueueEnqueue err %d", err); + return; + } // Inform the clients that the queue has been altered if (self.alteredProc != NULL) { diff --git a/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt b/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt index 6e5826829..3818c5a55 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt +++ b/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt @@ -1,8 +1,10 @@ project(mac-virtualcam) find_library(APPKIT AppKit) +find_library(COREVIDEO CoreVideo) +find_library(IOSURFACE IOSurface) -mark_as_advanced(APPKIT) +mark_as_advanced(APPKIT COREVIDEO IOSURFACE) add_library(mac-virtualcam MODULE) add_library(OBS::virtualcam ALIAS mac-virtualcam) @@ -15,8 +17,9 @@ target_include_directories( mac-virtualcam PRIVATE "$/../common") -target_link_libraries(mac-virtualcam PRIVATE OBS::libobs OBS::frontend-api - ${APPKIT}) +target_link_libraries( + mac-virtualcam PRIVATE OBS::libobs OBS::frontend-api ${APPKIT} ${COREVIDEO} + ${IOSURFACE}) target_compile_features(mac-virtualcam PRIVATE cxx_deleted_functions cxx_rvalue_references cxx_std_17) diff --git a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h index c6ab0305b..bd825b2a7 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h +++ b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h @@ -6,6 +6,7 @@ // #import +#import NS_ASSUME_NONNULL_BEGIN @@ -16,11 +17,10 @@ NS_ASSUME_NONNULL_BEGIN /*! Will eventually be used for sending frames to all connected clients */ -- (void)sendFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameBytes:(uint8_t *)frameBytes; +- (void)sendPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; - (void)stop; diff --git a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm index f7bfb051d..a1b141a45 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm +++ b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm @@ -7,6 +7,7 @@ #import "OBSDALMachServer.h" #include +#include #include "MachProtocol.h" #include "Defines.h" @@ -121,23 +122,16 @@ [self.clientPorts minusSet:removedPorts]; } -- (void)sendFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameBytes:(uint8_t *)frameBytes +- (void)sendPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { if ([self.clientPorts count] <= 0) { return; } @autoreleasepool { - CGFloat width = size.width; - NSData *widthData = [NSData dataWithBytes:&width - length:sizeof(width)]; - CGFloat height = size.height; - NSData *heightData = [NSData dataWithBytes:&height - length:sizeof(height)]; NSData *timestampData = [NSData dataWithBytes:×tamp length:sizeof(timestamp)]; @@ -148,19 +142,20 @@ dataWithBytes:&fpsDenominator length:sizeof(fpsDenominator)]; - // NOTE: I'm not totally sure about the safety of dataWithBytesNoCopy in this context. - // Seems like there could potentially be an issue if the frameBuffer went away before the - // mach message finished sending. But it seems to be working and avoids a memory copy. Alternately - // we could do something like - // NSData *frameData = [NSData dataWithBytes:(void *)frameBytes length:size.width * size.height * 2]; - NSData *frameData = [NSData - dataWithBytesNoCopy:(void *)frameBytes - length:size.width * size.height * 2 - freeWhenDone:NO]; + NSPort *framePort = [NSMachPort + portWithMachPort:IOSurfaceCreateMachPort( + CVPixelBufferGetIOSurface( + frame))]; + + if (!framePort) { + blog(LOG_ERROR, + "unable to allocate mach port for pixel buffer"); + return; + } + [self sendMessageToClientsWithMsgId:MachMsgIdFrame components:@[ - widthData, heightData, - timestampData, frameData, + framePort, timestampData, fpsNumeratorData, fpsDenominatorData ]]; diff --git a/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm b/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm index 34e3b087a..cd01b3d70 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm +++ b/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm @@ -182,14 +182,55 @@ static void virtualcam_output_raw_video(void *data, struct video_data *frame) (videoInfo.output_width * 2), frame->linesize[0]); } - CGFloat width = videoInfo.output_width; - CGFloat height = videoInfo.output_height; + size_t width = videoInfo.output_width; + size_t height = videoInfo.output_height; - [sMachServer sendFrameWithSize:NSMakeSize(width, height) - timestamp:frame->timestamp - fpsNumerator:videoInfo.fps_num - fpsDenominator:videoInfo.fps_den - frameBytes:outData]; + NSDictionary *pbAttr = @{(id)kCVPixelBufferIOSurfacePropertiesKey: @{}}; + CVPixelBufferRef frameRef = NULL; + CVReturn status = CVPixelBufferCreate(NULL, width, height, + kCVPixelFormatType_422YpCbCr8, + (__bridge CFDictionaryRef)pbAttr, + &frameRef); + + if (status != kCVReturnSuccess) { + blog(LOG_ERROR, "unable to allocate pixel buffer (error %d)", + status); + return; + } + + // Copy memory into the pixel buffer + CVPixelBufferLockBaseAddress(frameRef, 0); + uint8_t *dest = + (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(frameRef, 0); + uint8_t *src = outData; + + size_t destBytesPerRow = + CVPixelBufferGetBytesPerRowOfPlane(frameRef, 0); + size_t srcBytesPerRow = frame->linesize[0]; + + // Sometimes CVPixelBufferCreate will create a pixelbuffer that's a different + // size than necessary to hold the frame (probably for some optimization reason). + // If that is the case this will do a row-by-row copy into the buffer. + if (destBytesPerRow == srcBytesPerRow) { + memcpy(dest, src, destBytesPerRow * height); + } else { + for (int line = 0; line < height; line++) { + memcpy(dest, src, srcBytesPerRow); + src += srcBytesPerRow; + dest += destBytesPerRow; + } + } + memcpy(dest, outData, srcBytesPerRow * height); + + CVPixelBufferUnlockBaseAddress(frameRef, 0); + + // Share pixel buffer with clients + [sMachServer sendPixelBuffer:frameRef + timestamp:frame->timestamp + fpsNumerator:videoInfo.fps_num + fpsDenominator:videoInfo.fps_den]; + + CVPixelBufferRelease(frameRef); } struct obs_output_info virtualcam_output_info = {