diff --git a/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt b/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt index f7747b680..655fbd76c 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt +++ b/plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt @@ -4,8 +4,9 @@ find_library(COCOA Cocoa) find_library(COREMEDIA CoreMedia) find_library(COREMEDIAIO CoreMediaIO) find_library(COREVIDEO CoreVideo) +find_library(IOSURFACE IOSurface) -mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO) +mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO IOSURFACE) add_library(mac-dal-plugin MODULE) add_library(OBS::mac-dal-plugin ALIAS mac-dal-plugin) @@ -37,8 +38,9 @@ target_include_directories( target_compile_options(mac-dal-plugin PRIVATE -fobjc-arc -fobjc-weak) -target_link_libraries(mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA} - ${COREMEDIAIO} ${COREVIDEO}) +target_link_libraries( + mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA} ${COREMEDIAIO} ${COREVIDEO} + ${IOSURFACE}) set(MACOSX_PLUGIN_BUNDLE_TYPE "BNDL") target_sources(mac-dal-plugin PRIVATE placeholder.png) diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h index 54ba4bee3..3323432d4 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h @@ -6,16 +6,16 @@ // #import +#import NS_ASSUME_NONNULL_BEGIN @protocol MachClientDelegate -- (void)receivedFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData; +- (void)receivedPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; - (void)receivedStop; @end diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm index 8279cf330..452ee6fbc 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm @@ -101,29 +101,37 @@ break; case MachMsgIdFrame: VLog(@"Received frame message"); - if (components.count >= 6) { - CGFloat width; - [components[0] getBytes:&width length:sizeof(width)]; - CGFloat height; - [components[1] getBytes:&height length:sizeof(height)]; + if (components.count >= 4) { + NSMachPort *framePort = (NSMachPort *)components[0]; + IOSurfaceRef surface = IOSurfaceLookupFromMachPort( + [framePort machPort]); + + CVPixelBufferRef frame; + CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, + surface, NULL, &frame); + uint64_t timestamp; - [components[2] getBytes:×tamp + [components[1] getBytes:×tamp length:sizeof(timestamp)]; - VLog(@"Received frame data: %fx%f (%llu)", width, - height, timestamp); - NSData *frameData = components[3]; + + VLog(@"Received frame data: %zux%zu (%llu)", + CVPixelBufferGetWidth(frame), + CVPixelBufferGetHeight(frame), timestamp); + uint32_t fpsNumerator; - [components[4] getBytes:&fpsNumerator + [components[2] getBytes:&fpsNumerator length:sizeof(fpsNumerator)]; uint32_t fpsDenominator; - [components[5] getBytes:&fpsDenominator + [components[3] getBytes:&fpsDenominator length:sizeof(fpsDenominator)]; - [self.delegate - receivedFrameWithSize:NSMakeSize(width, height) - timestamp:timestamp - fpsNumerator:fpsNumerator - fpsDenominator:fpsDenominator - frameData:frameData]; + + [self.delegate receivedPixelBuffer:frame + timestamp:timestamp + fpsNumerator:fpsNumerator + fpsDenominator:fpsDenominator]; + + CVPixelBufferRelease(frame); + CFRelease(surface); } break; case MachMsgIdStop: diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm index c8aa75535..c1dcaa36c 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm @@ -203,19 +203,21 @@ typedef enum { #pragma mark - MachClientDelegate -- (void)receivedFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData +- (void)receivedPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { + size_t width = CVPixelBufferGetWidth(frame); + size_t height = CVPixelBufferGetHeight(frame); + dispatch_sync(_stateQueue, ^{ if (_state == PlugInStateWaitingForServer) { NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults]; - [defaults setInteger:size.width + [defaults setInteger:(long)width forKey:kTestCardWidthKey]; - [defaults setInteger:size.height + [defaults setInteger:(long)height forKey:kTestCardHeightKey]; [defaults setDouble:(double)fpsNumerator / (double)fpsDenominator @@ -234,11 +236,10 @@ typedef enum { dispatch_time(DISPATCH_TIME_NOW, 5.0 * NSEC_PER_SEC), 5.0 * NSEC_PER_SEC, (1ull * NSEC_PER_SEC) / 10); - [self.stream queueFrameWithSize:size - timestamp:timestamp - fpsNumerator:fpsNumerator - fpsDenominator:fpsDenominator - frameData:frameData]; + [self.stream queuePixelBuffer:frame + timestamp:timestamp + fpsNumerator:fpsNumerator + fpsDenominator:fpsDenominator]; } - (void)receivedStop diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h index d660b71c2..9d841ecea 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h @@ -18,6 +18,7 @@ // along with obs-mac-virtualcam. If not, see . #import "OBSDALObjectStore.h" +#import NS_ASSUME_NONNULL_BEGIN @@ -35,11 +36,10 @@ NS_ASSUME_NONNULL_BEGIN - (void)stopServingDefaultFrames; -- (void)queueFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData; +- (void)queuePixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; @end diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm index 730bb4cc2..fa19bc6ac 100644 --- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm +++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm @@ -349,11 +349,10 @@ } } -- (void)queueFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameData:(NSData *)frameData +- (void)queuePixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { if (CMSimpleQueueGetFullness(self.queue) >= 1.0) { DLog(@"Queue is full, bailing out"); @@ -374,9 +373,34 @@ self.sequenceNumber = CMIOGetNextSequenceNumber(self.sequenceNumber); CMSampleBufferRef sampleBuffer; - CMSampleBufferCreateFromData(size, timingInfo, self.sequenceNumber, - frameData, &sampleBuffer); - CMSimpleQueueEnqueue(self.queue, sampleBuffer); + + // Generate the video format description from that pixel buffer + CMVideoFormatDescriptionRef format; + err = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, + frame, &format); + if (err != noErr) { + DLog(@"CMVideoFormatDescriptionCreateForImageBuffer err %d", + err); + return; + } + + err = CMIOSampleBufferCreateForImageBuffer( + kCFAllocatorDefault, frame, format, &timingInfo, + self.sequenceNumber, kCMIOSampleBufferNoDiscontinuities, + &sampleBuffer); + + CFRelease(format); + + if (err != noErr) { + DLog(@"CMIOSampleBufferCreateForImageBuffer err %d", err); + return; + } + + err = CMSimpleQueueEnqueue(self.queue, sampleBuffer); + if (err != noErr) { + DLog(@"CMSimpleQueueEnqueue err %d", err); + return; + } // Inform the clients that the queue has been altered if (self.alteredProc != NULL) { diff --git a/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt b/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt index 6e5826829..3818c5a55 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt +++ b/plugins/mac-virtualcam/src/obs-plugin/CMakeLists.txt @@ -1,8 +1,10 @@ project(mac-virtualcam) find_library(APPKIT AppKit) +find_library(COREVIDEO CoreVideo) +find_library(IOSURFACE IOSurface) -mark_as_advanced(APPKIT) +mark_as_advanced(APPKIT COREVIDEO IOSURFACE) add_library(mac-virtualcam MODULE) add_library(OBS::virtualcam ALIAS mac-virtualcam) @@ -15,8 +17,9 @@ target_include_directories( mac-virtualcam PRIVATE "$/../common") -target_link_libraries(mac-virtualcam PRIVATE OBS::libobs OBS::frontend-api - ${APPKIT}) +target_link_libraries( + mac-virtualcam PRIVATE OBS::libobs OBS::frontend-api ${APPKIT} ${COREVIDEO} + ${IOSURFACE}) target_compile_features(mac-virtualcam PRIVATE cxx_deleted_functions cxx_rvalue_references cxx_std_17) diff --git a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h index c6ab0305b..bd825b2a7 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h +++ b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.h @@ -6,6 +6,7 @@ // #import +#import NS_ASSUME_NONNULL_BEGIN @@ -16,11 +17,10 @@ NS_ASSUME_NONNULL_BEGIN /*! Will eventually be used for sending frames to all connected clients */ -- (void)sendFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameBytes:(uint8_t *)frameBytes; +- (void)sendPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator; - (void)stop; diff --git a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm index f7bfb051d..a1b141a45 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm +++ b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm @@ -7,6 +7,7 @@ #import "OBSDALMachServer.h" #include +#include #include "MachProtocol.h" #include "Defines.h" @@ -121,23 +122,16 @@ [self.clientPorts minusSet:removedPorts]; } -- (void)sendFrameWithSize:(NSSize)size - timestamp:(uint64_t)timestamp - fpsNumerator:(uint32_t)fpsNumerator - fpsDenominator:(uint32_t)fpsDenominator - frameBytes:(uint8_t *)frameBytes +- (void)sendPixelBuffer:(CVPixelBufferRef)frame + timestamp:(uint64_t)timestamp + fpsNumerator:(uint32_t)fpsNumerator + fpsDenominator:(uint32_t)fpsDenominator { if ([self.clientPorts count] <= 0) { return; } @autoreleasepool { - CGFloat width = size.width; - NSData *widthData = [NSData dataWithBytes:&width - length:sizeof(width)]; - CGFloat height = size.height; - NSData *heightData = [NSData dataWithBytes:&height - length:sizeof(height)]; NSData *timestampData = [NSData dataWithBytes:×tamp length:sizeof(timestamp)]; @@ -148,19 +142,20 @@ dataWithBytes:&fpsDenominator length:sizeof(fpsDenominator)]; - // NOTE: I'm not totally sure about the safety of dataWithBytesNoCopy in this context. - // Seems like there could potentially be an issue if the frameBuffer went away before the - // mach message finished sending. But it seems to be working and avoids a memory copy. Alternately - // we could do something like - // NSData *frameData = [NSData dataWithBytes:(void *)frameBytes length:size.width * size.height * 2]; - NSData *frameData = [NSData - dataWithBytesNoCopy:(void *)frameBytes - length:size.width * size.height * 2 - freeWhenDone:NO]; + NSPort *framePort = [NSMachPort + portWithMachPort:IOSurfaceCreateMachPort( + CVPixelBufferGetIOSurface( + frame))]; + + if (!framePort) { + blog(LOG_ERROR, + "unable to allocate mach port for pixel buffer"); + return; + } + [self sendMessageToClientsWithMsgId:MachMsgIdFrame components:@[ - widthData, heightData, - timestampData, frameData, + framePort, timestampData, fpsNumeratorData, fpsDenominatorData ]]; diff --git a/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm b/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm index 34e3b087a..cd01b3d70 100644 --- a/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm +++ b/plugins/mac-virtualcam/src/obs-plugin/plugin-main.mm @@ -182,14 +182,55 @@ static void virtualcam_output_raw_video(void *data, struct video_data *frame) (videoInfo.output_width * 2), frame->linesize[0]); } - CGFloat width = videoInfo.output_width; - CGFloat height = videoInfo.output_height; + size_t width = videoInfo.output_width; + size_t height = videoInfo.output_height; - [sMachServer sendFrameWithSize:NSMakeSize(width, height) - timestamp:frame->timestamp - fpsNumerator:videoInfo.fps_num - fpsDenominator:videoInfo.fps_den - frameBytes:outData]; + NSDictionary *pbAttr = @{(id)kCVPixelBufferIOSurfacePropertiesKey: @{}}; + CVPixelBufferRef frameRef = NULL; + CVReturn status = CVPixelBufferCreate(NULL, width, height, + kCVPixelFormatType_422YpCbCr8, + (__bridge CFDictionaryRef)pbAttr, + &frameRef); + + if (status != kCVReturnSuccess) { + blog(LOG_ERROR, "unable to allocate pixel buffer (error %d)", + status); + return; + } + + // Copy memory into the pixel buffer + CVPixelBufferLockBaseAddress(frameRef, 0); + uint8_t *dest = + (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(frameRef, 0); + uint8_t *src = outData; + + size_t destBytesPerRow = + CVPixelBufferGetBytesPerRowOfPlane(frameRef, 0); + size_t srcBytesPerRow = frame->linesize[0]; + + // Sometimes CVPixelBufferCreate will create a pixelbuffer that's a different + // size than necessary to hold the frame (probably for some optimization reason). + // If that is the case this will do a row-by-row copy into the buffer. + if (destBytesPerRow == srcBytesPerRow) { + memcpy(dest, src, destBytesPerRow * height); + } else { + for (int line = 0; line < height; line++) { + memcpy(dest, src, srcBytesPerRow); + src += srcBytesPerRow; + dest += destBytesPerRow; + } + } + memcpy(dest, outData, srcBytesPerRow * height); + + CVPixelBufferUnlockBaseAddress(frameRef, 0); + + // Share pixel buffer with clients + [sMachServer sendPixelBuffer:frameRef + timestamp:frame->timestamp + fpsNumerator:videoInfo.fps_num + fpsDenominator:videoInfo.fps_den]; + + CVPixelBufferRelease(frameRef); } struct obs_output_info virtualcam_output_info = {