diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index dbca9df9073..093599da6b9 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -35,6 +35,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES ${CORE_MEDIA_LIBRARY} ${CORE_VIDEO_LIBRARY} ${FOUNDATION_LIBRARY} + ${SCREEN_CAPTURE_KIT_LIBRARY} ${VIDEO_TOOLBOX_LIBRARY}) set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in") @@ -55,6 +56,8 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sckit_video.h" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sckit_video.mm" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h" ${APPLE_PLIST_FILE}) diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 5e225fdac21..e202cfb277b 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -9,6 +9,7 @@ FIND_LIBRARY(CORE_AUDIO_LIBRARY CoreAudio) FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia) FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) if(SUNSHINE_ENABLE_TRAY) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index be124b2d331..065fc530218 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -7,9 +7,9 @@ #include "src/logging.h" #include "src/platform/common.h" #include "src/platform/macos/av_img_t.h" -#include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" +#include "src/platform/macos/sckit_video.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg @@ -22,15 +22,15 @@ using namespace std::literals; struct av_display_t: public display_t { - AVVideo *av_capture {}; + SCKitVideo *capture_backend {}; CGDirectDisplayID display_id {}; ~av_display_t() override { - [av_capture release]; + [capture_backend release]; } capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { - auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) { + auto signal = [capture_backend capture:^(CMSampleBufferRef sampleBuffer) { auto new_sample_buffer = std::make_shared(sampleBuffer); auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); @@ -56,6 +56,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img_out->pixel_pitch = img_out->row_pitch / img_out->width; + img_out->frame_timestamp = std::chrono::steady_clock::now(); old_data_retainer = nullptr; @@ -80,13 +81,13 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const std::unique_ptr make_avcodec_encode_device(pix_fmt_e pix_fmt) override { if (pix_fmt == pix_fmt_e::yuv420p) { - av_capture.pixelFormat = kCVPixelFormatType_32BGRA; + capture_backend.pixelFormat = kCVPixelFormatType_32BGRA; return std::make_unique(); } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init(static_cast(av_capture), pix_fmt, setResolution, setPixelFormat); + device->init(static_cast(capture_backend), pix_fmt, setResolution, setPixelFormat); return device; } else { @@ -96,41 +97,94 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } int dummy_img(img_t *img) override { - if (!platf::is_screen_capture_allowed()) { - // If we don't have the screen capture permission, this function will hang - // indefinitely without doing anything useful. Exit instead to avoid this. - // A non-zero return value indicates failure to the calling function. + auto av_img = (av_img_t *) img; + const auto width = capture_backend.frameWidth; + const auto height = capture_backend.frameHeight; + const auto pixel_format = capture_backend.pixelFormat; + + CVPixelBufferRef pixel_buffer = nullptr; + NSDictionary *attributes = @{ + (id) kCVPixelBufferIOSurfacePropertiesKey: @{} + }; + auto status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + pixel_format, + (CFDictionaryRef) attributes, + &pixel_buffer + ); + if (status != kCVReturnSuccess || pixel_buffer == nullptr) { + BOOST_LOG(error) << "Failed to allocate macOS dummy pixel buffer: " << status; return 1; } - auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) { - auto new_sample_buffer = std::make_shared(sampleBuffer); - auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + CVPixelBufferLockBaseAddress(pixel_buffer, 0); + if (CVPixelBufferIsPlanar(pixel_buffer)) { + for (size_t plane = 0; plane < CVPixelBufferGetPlaneCount(pixel_buffer); ++plane) { + auto base = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane)); + auto bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); + auto plane_height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + memset(base, plane == 0 ? 0x00 : 0x80, bytes_per_row * plane_height); + } + } else { + auto base = static_cast(CVPixelBufferGetBaseAddress(pixel_buffer)); + auto bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer); + memset(base, 0x00, bytes_per_row * height); + } + CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); - auto av_img = (av_img_t *) img; + CMVideoFormatDescriptionRef format_description = nullptr; + auto cm_status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, &format_description); + if (cm_status != noErr || format_description == nullptr) { + BOOST_LOG(error) << "Failed to create macOS dummy video format description: " << cm_status; + CVPixelBufferRelease(pixel_buffer); + return 1; + } - auto old_data_retainer = std::make_shared( - av_img->sample_buffer, - av_img->pixel_buffer, - img->data - ); + CMSampleTimingInfo timing_info {}; + timing_info.duration = kCMTimeInvalid; + timing_info.presentationTimeStamp = kCMTimeZero; + timing_info.decodeTimeStamp = kCMTimeInvalid; + + CMSampleBufferRef sample_buffer = nullptr; + cm_status = CMSampleBufferCreateReadyWithImageBuffer( + kCFAllocatorDefault, + pixel_buffer, + format_description, + &timing_info, + &sample_buffer + ); + CFRelease(format_description); + CVPixelBufferRelease(pixel_buffer); + + if (cm_status != noErr || sample_buffer == nullptr) { + BOOST_LOG(error) << "Failed to create macOS dummy sample buffer: " << cm_status; + return 1; + } - av_img->sample_buffer = new_sample_buffer; - av_img->pixel_buffer = new_pixel_buffer; - img->data = new_pixel_buffer->data(); + auto new_sample_buffer = std::make_shared(sample_buffer); + auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + CFRelease(sample_buffer); - img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); - img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); - img->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); - img->pixel_pitch = img->row_pitch / img->width; + auto old_data_retainer = std::make_shared( + av_img->sample_buffer, + av_img->pixel_buffer, + img->data + ); - old_data_retainer = nullptr; + av_img->sample_buffer = new_sample_buffer; + av_img->pixel_buffer = new_pixel_buffer; + img->data = new_pixel_buffer->data(); - // returning false here stops capture backend - return false; - }]; + img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); + img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); + img->row_pitch = CVPixelBufferIsPlanar(new_pixel_buffer->buf) ? + (int) CVPixelBufferGetBytesPerRowOfPlane(new_pixel_buffer->buf, 0) : + (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); + img->pixel_pitch = img->row_pitch / img->width; - dispatch_semaphore_wait(signal, DISPATCH_TIME_FOREVER); + old_data_retainer = nullptr; return 0; } @@ -143,11 +197,12 @@ int dummy_img(img_t *img) override { * height --> the intended capture height */ static void setResolution(void *display, int width, int height) { - [static_cast(display) setFrameWidth:width frameHeight:height]; + BOOST_LOG(info) << "ScreenCaptureKit encoder requested capture size " << width << "x" << height; + [static_cast(display) setFrameWidth:width frameHeight:height]; } static void setPixelFormat(void *display, OSType pixelFormat) { - static_cast(display).pixelFormat = pixelFormat; + static_cast(display).pixelFormat = pixelFormat; } }; @@ -163,7 +218,7 @@ static void setPixelFormat(void *display, OSType pixelFormat) { display->display_id = CGMainDisplayID(); // Print all displays available with it's name and id - auto display_array = [AVVideo displayNames]; + auto display_array = [SCKitVideo displayNames]; BOOST_LOG(info) << "Detecting displays"sv; for (NSDictionary *item in display_array) { NSNumber *display_id = item[@"id"]; @@ -177,26 +232,33 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; - display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + display->capture_backend = [[SCKitVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; - if (!display->av_capture) { + if (!display->capture_backend) { BOOST_LOG(error) << "Video setup failed."sv; return nullptr; } - display->width = display->av_capture.frameWidth; - display->height = display->av_capture.frameHeight; + display->width = display->capture_backend.frameWidth; + display->height = display->capture_backend.frameHeight; // We also need set env_width and env_height for absolute mouse coordinates display->env_width = display->width; display->env_height = display->height; + if (config.width > 0 && config.height > 0) { + BOOST_LOG(info) << "ScreenCaptureKit capture target for display " << display->display_id + << " source=" << display->width << "x" << display->height + << " client=" << config.width << "x" << config.height; + [display->capture_backend setFrameWidth:config.width frameHeight:config.height]; + } + return display; } std::vector display_names(mem_type_e hwdevice_type) { __block std::vector display_names; - auto display_array = [AVVideo displayNames]; + auto display_array = [SCKitVideo displayNames]; display_names.reserve([display_array count]); [display_array enumerateObjectsUsingBlock:^(NSDictionary *_Nonnull obj, NSUInteger idx, BOOL *_Nonnull stop) { diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index b4fb28cb736..9481eb9e7d8 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -56,7 +56,7 @@ namespace platf { } int nv12_zero_device::init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn) { - pixel_format_fn(display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); + pixel_format_fn(display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); this->display = display; this->resolution_fn = std::move(resolution_fn); diff --git a/src/platform/macos/sckit_video.h b/src/platform/macos/sckit_video.h new file mode 100644 index 00000000000..34893e92fe5 --- /dev/null +++ b/src/platform/macos/sckit_video.h @@ -0,0 +1,50 @@ +/** + * @file src/platform/macos/sckit_video.h + * @brief Declarations for ScreenCaptureKit display capture on macOS. + */ +#pragma once + +// platform includes +#import +#import +#import +#import + +static const int kMaxSCKitDisplays = 32; + +typedef bool (^SCKitFrameCallbackBlock)(CMSampleBufferRef); + +@interface SCKitVideo: NSObject + +@property(nonatomic, assign) CGDirectDisplayID displayID; +@property(nonatomic, assign) OSType pixelFormat; +@property(nonatomic, assign) int frameWidth; +@property(nonatomic, assign) int frameHeight; +@property(nonatomic, assign) int requestedFrameRate; + +@property(nonatomic, retain) SCDisplay *display; +@property(nonatomic, retain) SCStream *stream; +@property(nonatomic, copy) SCKitFrameCallbackBlock captureCallback; +@property(nonatomic, assign) dispatch_semaphore_t captureSignal; +@property(nonatomic, retain) NSError *captureError; +@property(nonatomic, assign) NSUInteger deliveredFrames; +@property(nonatomic, assign) NSUInteger completeFrames; +@property(nonatomic, assign) NSUInteger startedFrames; +@property(nonatomic, assign) NSUInteger idleFrames; +@property(nonatomic, assign) NSUInteger blankFrames; +@property(nonatomic, assign) NSUInteger suspendedFrames; +@property(nonatomic, assign) NSUInteger stoppedFrames; +@property(nonatomic, assign) NSUInteger unknownStatusFrames; +@property(nonatomic, assign) NSUInteger noImageFrames; +@property(nonatomic, assign) NSUInteger invalidFrames; +@property(nonatomic, assign) NSTimeInterval lastFrameReportTime; + ++ (NSArray *)displayNames; ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID; + +- (id)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)capture:(SCKitFrameCallbackBlock)frameCallback; + +@end diff --git a/src/platform/macos/sckit_video.mm b/src/platform/macos/sckit_video.mm new file mode 100644 index 00000000000..6af4cf6c8fd --- /dev/null +++ b/src/platform/macos/sckit_video.mm @@ -0,0 +1,303 @@ +/** + * @file src/platform/macos/sckit_video.mm + * @brief Definitions for ScreenCaptureKit display capture on macOS. + */ + +#import "sckit_video.h" + +#include "src/logging.h" + +@implementation SCKitVideo + ++ (NSArray *)displayNames { + CGDirectDisplayID displays[kMaxSCKitDisplays]; + uint32_t count; + if (CGGetActiveDisplayList(kMaxSCKitDisplays, displays, &count) != kCGErrorSuccess) { + return [NSArray array]; + } + + NSMutableArray *result = [NSMutableArray array]; + + for (uint32_t i = 0; i < count; i++) { + [result addObject:@{ + @"id": [NSNumber numberWithUnsignedInt:displays[i]], + @"name": [NSString stringWithFormat:@"%d", displays[i]], + @"displayName": [self getDisplayName:displays[i]], + }]; + } + + return [NSArray arrayWithArray:result]; +} + ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID { + for (NSScreen *screen in [NSScreen screens]) { + if ([screen.deviceDescription[@"NSScreenNumber"] isEqualToNumber:[NSNumber numberWithUnsignedInt:displayID]]) { + return screen.localizedName; + } + } + return nil; +} + ++ (SCDisplay *)screenCaptureKitDisplayForID:(CGDirectDisplayID)displayID { + __block SCShareableContent *content = nil; + __block NSError *contentError = nil; + dispatch_semaphore_t contentSignal = dispatch_semaphore_create(0); + + [SCShareableContent getShareableContentExcludingDesktopWindows:NO + onScreenWindowsOnly:YES + completionHandler:^(SCShareableContent *_Nullable shareableContent, NSError *_Nullable error) { + content = [shareableContent retain]; + contentError = [error retain]; + dispatch_semaphore_signal(contentSignal); + }]; + + dispatch_semaphore_wait(contentSignal, DISPATCH_TIME_FOREVER); + + if (contentError != nil) { + [contentError release]; + [content release]; + return nil; + } + + SCDisplay *result = nil; + for (SCDisplay *display in content.displays) { + if (display.displayID == displayID) { + result = [display retain]; + break; + } + } + + [content release]; + return [result autorelease]; +} + +- (id)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { + self = [super init]; + if (self == nil) { + return nil; + } + + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); + if (mode == nullptr) { + [self release]; + return nil; + } + + self.displayID = displayID; + self.pixelFormat = kCVPixelFormatType_32BGRA; + self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); + self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); + self.requestedFrameRate = frameRate; + + CFRelease(mode); + + self.display = [SCKitVideo screenCaptureKitDisplayForID:displayID]; + if (self.display == nil) { + [self release]; + return nil; + } + + BOOST_LOG(info) << "Using ScreenCaptureKit display capture for display " << displayID + << " (" << self.frameWidth << "x" << self.frameHeight << " pixels)" + << " requested_fps=" << frameRate; + + return self; +} + +- (void)dealloc { + [self.stream stopCaptureWithCompletionHandler:nil]; + self.stream = nil; + self.display = nil; + self.captureCallback = nil; + self.captureError = nil; + [super dealloc]; +} + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { + self.frameWidth = frameWidth; + self.frameHeight = frameHeight; +} + +- (dispatch_semaphore_t)capture:(SCKitFrameCallbackBlock)frameCallback { + @synchronized(self) { + self.captureCallback = frameCallback; + self.captureSignal = dispatch_semaphore_create(0); + self.captureError = nil; + + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:self.display excludingWindows:@[]]; + SCStreamConfiguration *configuration = [[SCStreamConfiguration alloc] init]; + configuration.width = self.frameWidth; + configuration.height = self.frameHeight; + configuration.pixelFormat = self.pixelFormat; + configuration.queueDepth = 8; + configuration.showsCursor = YES; + + // ScreenCaptureKit's default is 1/60. Use a slightly shorter interval than the + // client target to avoid SCK/WindowServer timing jitter dropping high-FPS updates. + if (self.requestedFrameRate > 0) { + CMTime targetInterval = CMTimeMake(1, self.requestedFrameRate); + configuration.minimumFrameInterval = CMTimeMultiplyByFloat64(targetInterval, 0.9); + } else { + configuration.minimumFrameInterval = kCMTimeZero; + } + + if (@available(macOS 14.0, *)) { + configuration.captureResolution = SCCaptureResolutionNominal; + } + + self.deliveredFrames = 0; + self.completeFrames = 0; + self.startedFrames = 0; + self.idleFrames = 0; + self.blankFrames = 0; + self.suspendedFrames = 0; + self.stoppedFrames = 0; + self.unknownStatusFrames = 0; + self.noImageFrames = 0; + self.invalidFrames = 0; + self.lastFrameReportTime = [[NSDate date] timeIntervalSince1970]; + + BOOST_LOG(info) << "Starting ScreenCaptureKit stream for display " << self.displayID + << " capture=" << self.frameWidth << "x" << self.frameHeight + << " pixel_format=0x" << std::hex << self.pixelFormat << std::dec + << " queue_depth=" << configuration.queueDepth + << " minimum_frame_interval=" << configuration.minimumFrameInterval.value + << "/" << configuration.minimumFrameInterval.timescale; + + self.stream = [[[SCStream alloc] initWithFilter:filter configuration:configuration delegate:nil] autorelease]; + [filter release]; + [configuration release]; + + NSError *outputError = nil; + dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, DISPATCH_QUEUE_PRIORITY_HIGH); + dispatch_queue_t recordingQueue = dispatch_queue_create("screenCaptureKitQueue", qos); + if (![self.stream addStreamOutput:self type:SCStreamOutputTypeScreen sampleHandlerQueue:recordingQueue error:&outputError]) { + self.captureError = outputError; + dispatch_semaphore_signal(self.captureSignal); + return self.captureSignal; + } + + [self.stream startCaptureWithCompletionHandler:^(NSError *_Nullable startError) { + if (startError != nil) { + @synchronized(self) { + self.captureError = startError; + BOOST_LOG(error) << "ScreenCaptureKit failed to start display " << self.displayID + << ": " << startError.localizedDescription.UTF8String; + dispatch_semaphore_signal(self.captureSignal); + } + } + }]; + + return self.captureSignal; + } +} + +- (void)reportFrameStatsIfNeeded { + NSTimeInterval now = [[NSDate date] timeIntervalSince1970]; + NSTimeInterval elapsed = now - self.lastFrameReportTime; + if (elapsed < 5.0) { + return; + } + + double deliveredFPS = elapsed > 0 ? (double) self.deliveredFrames / elapsed : 0.0; + BOOST_LOG(info) << "ScreenCaptureKit display " << self.displayID + << " delivered_fps=" << deliveredFPS + << " complete=" << self.completeFrames + << " started=" << self.startedFrames + << " idle=" << self.idleFrames + << " blank=" << self.blankFrames + << " suspended=" << self.suspendedFrames + << " stopped=" << self.stoppedFrames + << " unknown_status=" << self.unknownStatusFrames + << " no_image=" << self.noImageFrames + << " invalid=" << self.invalidFrames + << " over_seconds=" << elapsed; + + self.deliveredFrames = 0; + self.completeFrames = 0; + self.startedFrames = 0; + self.idleFrames = 0; + self.blankFrames = 0; + self.suspendedFrames = 0; + self.stoppedFrames = 0; + self.unknownStatusFrames = 0; + self.noImageFrames = 0; + self.invalidFrames = 0; + self.lastFrameReportTime = now; +} + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeScreen || !CMSampleBufferIsValid(sampleBuffer) || !CMSampleBufferDataIsReady(sampleBuffer)) { + self.invalidFrames += 1; + [self reportFrameStatsIfNeeded]; + return; + } + + SCFrameStatus frameStatus = SCFrameStatusComplete; + bool knownStatus = true; + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false); + if (attachmentsArray != nil && CFArrayGetCount(attachmentsArray) > 0) { + NSDictionary *attachments = (NSDictionary *) CFArrayGetValueAtIndex(attachmentsArray, 0); + NSNumber *statusNumber = attachments[SCStreamFrameInfoStatus]; + if (statusNumber != nil) { + frameStatus = (SCFrameStatus) statusNumber.integerValue; + } + } + + switch (frameStatus) { + case SCFrameStatusComplete: + self.completeFrames += 1; + break; + case SCFrameStatusStarted: + self.startedFrames += 1; + break; + case SCFrameStatusIdle: + self.idleFrames += 1; + break; + case SCFrameStatusBlank: + self.blankFrames += 1; + break; + case SCFrameStatusSuspended: + self.suspendedFrames += 1; + break; + case SCFrameStatusStopped: + self.stoppedFrames += 1; + break; + default: + self.unknownStatusFrames += 1; + knownStatus = false; + break; + } + + CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (imageBuffer == nil) { + self.noImageFrames += 1; + [self reportFrameStatsIfNeeded]; + return; + } + + SCKitFrameCallbackBlock callback = self.captureCallback; + if (callback == nil) { + return; + } + + self.deliveredFrames += 1; + [self reportFrameStatsIfNeeded]; + + if (!knownStatus) { + BOOST_LOG(warning) << "ScreenCaptureKit display " << self.displayID + << " delivered frame with unknown status=" << (NSInteger) frameStatus; + } + + if (!callback(sampleBuffer)) { + @synchronized(self) { + self.captureCallback = nil; + [self.stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { + BOOST_LOG(info) << "Stopped ScreenCaptureKit stream for display " << self.displayID; + dispatch_semaphore_signal(self.captureSignal); + }]; + } + } +} + +@end diff --git a/src/rtsp.cpp b/src/rtsp.cpp index 0953cd59f60..8b180189c9f 100644 --- a/src/rtsp.cpp +++ b/src/rtsp.cpp @@ -1025,6 +1025,31 @@ namespace rtsp_stream { config.monitor.width = (int) util::from_view(args.at("x-nv-video[0].clientViewportWd"sv)); config.monitor.framerate = (int) util::from_view(args.at("x-nv-video[0].maxFPS"sv)); config.monitor.framerateX100 = (int) util::from_view(args.at("x-nv-video[0].clientRefreshRateX100"sv)); + +#ifdef __APPLE__ + if (config::nvhttp.sunshine_name == "LAPTOP") { + auto old_width = config.monitor.width; + auto old_height = config.monitor.height; + auto old_framerate = config.monitor.framerate; + config.monitor.framerate = std::min(config.monitor.framerate, 60); + config.monitor.framerateX100 = 0; + BOOST_LOG(info) << "macOS stream policy [LAPTOP]: client requested " + << old_width << "x" << old_height << "@" << old_framerate + << ", using " << config.monitor.width << "x" << config.monitor.height + << "@" << config.monitor.framerate; + } else if (config::nvhttp.sunshine_name == "SAMSUNG") { + auto old_width = config.monitor.width; + auto old_height = config.monitor.height; + auto old_framerate = config.monitor.framerate; + config.monitor.framerate = 120; + config.monitor.framerateX100 = 0; + BOOST_LOG(info) << "macOS stream policy [SAMSUNG]: client requested " + << old_width << "x" << old_height << "@" << old_framerate + << ", using " << config.monitor.width << "x" << config.monitor.height + << "@" << config.monitor.framerate; + } +#endif + // Validate framerateX100 against framerate. Some clients (e.g. Moonlight Android) send the // client display's refresh rate as clientRefreshRateX100, which may differ from the requested // streaming framerate. Discard framerateX100 if the derived fps is not within 1% of framerate. diff --git a/src/stream.cpp b/src/stream.cpp index c0da8671cd5..4413f517c29 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -1293,6 +1293,10 @@ namespace stream { } auto ratecontrol_next_frame_start = std::chrono::steady_clock::now(); + auto last_video_send_report = std::chrono::steady_clock::now(); + std::uint64_t sent_video_frames = 0; + std::uint64_t sent_duplicate_video_frames = 0; + std::uint64_t sent_key_video_frames = 0; while (auto packet = packets->pop()) { if (shutdown_event->peek()) { @@ -1475,6 +1479,27 @@ namespace stream { packet->frame_timestamp = ratecontrol_next_frame_start; frame_is_dupe = true; } + sent_video_frames++; + if (frame_is_dupe) { + sent_duplicate_video_frames++; + } + if (packet->is_idr()) { + sent_key_video_frames++; + } + auto now_for_send_report = std::chrono::steady_clock::now(); + auto send_report_elapsed = now_for_send_report - last_video_send_report; + if (send_report_elapsed >= 5s) { + auto elapsed_seconds = std::chrono::duration(send_report_elapsed).count(); + BOOST_LOG(info) << "Video send rate fps=" << (sent_video_frames / elapsed_seconds) + << " frames=" << sent_video_frames + << " dupes=" << sent_duplicate_video_frames + << " keyframes=" << sent_key_video_frames + << " over_seconds=" << elapsed_seconds; + sent_video_frames = 0; + sent_duplicate_video_frames = 0; + sent_key_video_frames = 0; + last_video_send_report = now_for_send_report; + } using rtp_tick = std::chrono::duration>; uint32_t timestamp = std::chrono::round(*packet->frame_timestamp - video_epoch).count(); diff --git a/src/video.cpp b/src/video.cpp index f2281ce841a..b1272f525c8 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include // lib includes @@ -31,6 +32,11 @@ extern "C" { #include "sync.h" #include "video.h" +#ifdef __APPLE__ + #include "platform/macos/av_img_t.h" + #include +#endif + #ifdef _WIN32 extern "C" { #include @@ -305,6 +311,295 @@ namespace video { FIXED_GOP_SIZE = 1 << 12, ///< Use fixed small GOP size (encoder doesn't support on-demand IDR frames) }; +#ifdef __APPLE__ + static CFNumberRef native_vt_i32(int value) { + return CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); + } + + static void native_vt_set_i32(VTCompressionSessionRef session, CFStringRef key, int value) { + auto number = native_vt_i32(value); + if (number) { + VTSessionSetProperty(session, key, number); + CFRelease(number); + } + } + + static void native_vt_append_start_code(std::vector &out) { + out.push_back(0x00); + out.push_back(0x00); + out.push_back(0x00); + out.push_back(0x01); + } + + static void native_vt_append_parameter_sets(CMFormatDescriptionRef format, bool hevc, std::vector &out, int &nal_header_length) { + if (!format) { + return; + } + + size_t parameter_set_count = 0; + nal_header_length = 4; + + auto append_one = [&](size_t index) { + const uint8_t *parameter_set = nullptr; + size_t parameter_set_size = 0; + size_t count = 0; + int header_length = nal_header_length; + OSStatus status = hevc ? + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(format, index, ¶meter_set, ¶meter_set_size, &count, &header_length) : + CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, index, ¶meter_set, ¶meter_set_size, &count, &header_length); + if (status == noErr && parameter_set && parameter_set_size > 0) { + parameter_set_count = count; + nal_header_length = header_length > 0 ? header_length : nal_header_length; + native_vt_append_start_code(out); + out.insert(std::end(out), parameter_set, parameter_set + parameter_set_size); + } + }; + + append_one(0); + for (size_t i = 1; i < parameter_set_count; ++i) { + append_one(i); + } + } + + static std::vector native_vt_sample_to_annex_b(CMSampleBufferRef sample, bool hevc, bool keyframe) { + std::vector out; + auto format = CMSampleBufferGetFormatDescription(sample); + int nal_header_length = 4; + + if (keyframe) { + native_vt_append_parameter_sets(format, hevc, out, nal_header_length); + } + + auto block = CMSampleBufferGetDataBuffer(sample); + if (!block) { + return out; + } + + size_t block_size = CMBlockBufferGetDataLength(block); + std::vector bytes(block_size); + if (CMBlockBufferCopyDataBytes(block, 0, block_size, bytes.data()) != noErr) { + return {}; + } + + size_t offset = 0; + while (offset + nal_header_length <= block_size) { + uint32_t nal_size = 0; + for (int i = 0; i < nal_header_length; ++i) { + nal_size = (nal_size << 8) | bytes[offset + i]; + } + offset += nal_header_length; + + if (nal_size == 0 || offset + nal_size > block_size) { + break; + } + + native_vt_append_start_code(out); + out.insert(std::end(out), std::begin(bytes) + offset, std::begin(bytes) + offset + nal_size); + offset += nal_size; + } + + return out; + } + + class native_vt_encode_session_t: public encode_session_t { + public: + native_vt_encode_session_t(config_t config, sunshine_colorspace_t colorspace): + config {config}, + colorspace {colorspace}, + hevc {config.videoFormat == 1} { + auto codec = hevc ? kCMVideoCodecType_HEVC : kCMVideoCodecType_H264; + + CFMutableDictionaryRef source_attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + if (source_attrs) { + auto pixel_format = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + auto format_number = native_vt_i32((int) pixel_format); + auto width_number = native_vt_i32(config.width); + auto height_number = native_vt_i32(config.height); + CFMutableDictionaryRef io_surface = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + if (format_number) { + CFDictionarySetValue(source_attrs, kCVPixelBufferPixelFormatTypeKey, format_number); + } + if (width_number) { + CFDictionarySetValue(source_attrs, kCVPixelBufferWidthKey, width_number); + } + if (height_number) { + CFDictionarySetValue(source_attrs, kCVPixelBufferHeightKey, height_number); + } + if (io_surface) { + CFDictionarySetValue(source_attrs, kCVPixelBufferIOSurfacePropertiesKey, io_surface); + } + if (format_number) { + CFRelease(format_number); + } + if (width_number) { + CFRelease(width_number); + } + if (height_number) { + CFRelease(height_number); + } + if (io_surface) { + CFRelease(io_surface); + } + } + + auto status = VTCompressionSessionCreate( + kCFAllocatorDefault, + config.width, + config.height, + codec, + nullptr, + source_attrs, + kCFAllocatorDefault, + output_callback, + this, + &session + ); + if (source_attrs) { + CFRelease(source_attrs); + } + + if (status != noErr || !session) { + BOOST_LOG(error) << "Native VideoToolbox session create failed: " << status; + return; + } + + VTSessionSetProperty(session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); + VTSessionSetProperty(session, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse); + VTSessionSetProperty(session, kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, kCFBooleanTrue); + VTSessionSetProperty(session, kVTCompressionPropertyKey_MaximizePowerEfficiency, kCFBooleanFalse); + VTSessionSetProperty(session, kVTCompressionPropertyKey_ProfileLevel, hevc ? kVTProfileLevel_HEVC_Main_AutoLevel : kVTProfileLevel_H264_High_AutoLevel); + if (!hevc) { + VTSessionSetProperty(session, kVTCompressionPropertyKey_H264EntropyMode, kVTH264EntropyMode_CABAC); + } + native_vt_set_i32(session, kVTCompressionPropertyKey_AverageBitRate, config.bitrate * 1000); + native_vt_set_i32(session, kVTCompressionPropertyKey_ExpectedFrameRate, config.framerate); + native_vt_set_i32(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, config.framerate * 60); + native_vt_set_i32(session, kVTCompressionPropertyKey_MaxFrameDelayCount, 1); + native_vt_set_i32(session, kVTCompressionPropertyKey_ReferenceBufferCount, 1); + + status = VTCompressionSessionPrepareToEncodeFrames(session); + if (status != noErr) { + BOOST_LOG(error) << "Native VideoToolbox session prepare failed: " << status; + CFRelease(session); + session = nullptr; + return; + } + + ready = true; + BOOST_LOG(info) << "Native VideoToolbox encoder ready codec=" << (hevc ? "hevc" : "h264") + << " size=" << config.width << "x" << config.height + << " fps=" << config.framerate + << " bitrate=" << (config.bitrate * 1000); + } + + ~native_vt_encode_session_t() override { + if (session) { + VTCompressionSessionCompleteFrames(session, kCMTimeIndefinite); + VTCompressionSessionInvalidate(session); + CFRelease(session); + } + if (current_pixel) { + CVPixelBufferRelease(current_pixel); + } + } + + int convert(platf::img_t &img) override { + auto *av_img = dynamic_cast(&img); + if (!av_img || !av_img->pixel_buffer || !av_img->pixel_buffer->buf) { + return -1; + } + + auto new_pixel = (CVPixelBufferRef) CFRetain(av_img->pixel_buffer->buf); + if (current_pixel) { + CVPixelBufferRelease(current_pixel); + } + current_pixel = new_pixel; + return 0; + } + + void request_idr_frame() override { + force_idr = true; + } + + void request_normal_frame() override { + force_idr = false; + } + + void invalidate_ref_frames(int64_t first_frame, int64_t last_frame) override { + force_idr = true; + } + + bool is_ready() const { + return ready; + } + + struct frame_ref_t { + native_vt_encode_session_t *owner; + safe::mail_raw_t::queue_t packets; + void *channel_data; + int64_t frame_nr; + std::optional frame_timestamp; + }; + + static bool sample_is_keyframe(CMSampleBufferRef sample) { + CFArrayRef attachments_array = CMSampleBufferGetSampleAttachmentsArray(sample, false); + if (attachments_array && CFArrayGetCount(attachments_array) > 0) { + auto attachments = (CFDictionaryRef) CFArrayGetValueAtIndex(attachments_array, 0); + if (attachments && CFDictionaryContainsKey(attachments, kCMSampleAttachmentKey_NotSync)) { + return false; + } + } + return true; + } + + static void output_callback(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sampleBuffer) { + std::unique_ptr frame_ref {static_cast(sourceFrameRefCon)}; + auto *owner = static_cast(outputCallbackRefCon); + if (!owner || !frame_ref) { + return; + } + + if (status != noErr || !sampleBuffer || !CMSampleBufferDataIsReady(sampleBuffer)) { + owner->failed_packets++; + BOOST_LOG(error) << "Native VideoToolbox encode callback failed status=" << status << " flags=" << flags; + return; + } + + bool keyframe = sample_is_keyframe(sampleBuffer); + auto data = native_vt_sample_to_annex_b(sampleBuffer, owner->hevc, keyframe); + if (data.empty()) { + owner->failed_packets++; + BOOST_LOG(error) << "Native VideoToolbox produced an empty Annex-B frame"; + return; + } + + auto packet = std::make_unique(std::move(data), frame_ref->frame_nr, keyframe); + packet->channel_data = frame_ref->channel_data; + packet->frame_timestamp = frame_ref->frame_timestamp; + frame_ref->packets->raise(std::move(packet)); + owner->output_packets++; + if (keyframe) { + owner->key_packets++; + } + } + + config_t config; + sunshine_colorspace_t colorspace; + bool hevc; + bool ready = false; + bool force_idr = false; + VTCompressionSessionRef session = nullptr; + CVPixelBufferRef current_pixel = nullptr; + + std::chrono::steady_clock::time_point last_rate_report {std::chrono::steady_clock::now()}; + std::uint64_t input_frames = 0; + std::atomic output_packets {0}; + std::atomic key_packets {0}; + std::atomic failed_packets {0}; + std::chrono::steady_clock::duration encode_call_time {}; + }; +#endif + class avcodec_encode_session_t: public encode_session_t { public: avcodec_encode_session_t() = default; @@ -380,6 +675,20 @@ namespace video { // inject sps/vps data into idr pictures int inject; + + std::chrono::steady_clock::time_point last_rate_report {std::chrono::steady_clock::now()}; + std::uint64_t input_frames = 0; + std::uint64_t output_packets = 0; + std::uint64_t no_output_returns = 0; + std::uint64_t key_packets = 0; + std::uint64_t pts_matches = 0; + std::uint64_t pts_mismatches = 0; + std::uint64_t duplicate_bypass_packets = 0; + bool duplicate_bypass_enabled = false; + std::map> pending_frame_timestamps; + std::vector last_non_idr_packet; + std::chrono::steady_clock::duration send_frame_time {}; + std::chrono::steady_clock::duration receive_packet_time {}; }; class nvenc_encode_session_t: public encode_session_t { @@ -1520,27 +1829,96 @@ namespace video { auto &sps = session.sps; auto &vps = session.vps; + auto log_encoder_rate = [&session] { + auto now = std::chrono::steady_clock::now(); + auto elapsed = now - session.last_rate_report; + if (elapsed < 5s) { + return; + } + + auto elapsed_seconds = std::chrono::duration(elapsed).count(); + auto avg_send_ms = session.input_frames ? + std::chrono::duration(session.send_frame_time).count() / session.input_frames : + 0.0; + auto avg_receive_ms = session.output_packets ? + std::chrono::duration(session.receive_packet_time).count() / session.output_packets : + 0.0; + + BOOST_LOG(info) << "AVCodec encode rate inputs_fps=" << (session.input_frames / elapsed_seconds) + << " outputs_fps=" << (session.output_packets / elapsed_seconds) + << " inputs=" << session.input_frames + << " outputs=" << session.output_packets + << " no_output_returns=" << session.no_output_returns + << " key_packets=" << session.key_packets + << " pts_matches=" << session.pts_matches + << " pts_mismatches=" << session.pts_mismatches + << " duplicate_bypass_packets=" << session.duplicate_bypass_packets + << " duplicate_bypass_enabled=" << session.duplicate_bypass_enabled + << " pending_timestamps=" << session.pending_frame_timestamps.size() + << " avg_send_ms=" << avg_send_ms + << " avg_receive_ms=" << avg_receive_ms + << " over_seconds=" << elapsed_seconds; + + session.last_rate_report = now; + session.input_frames = 0; + session.output_packets = 0; + session.no_output_returns = 0; + session.key_packets = 0; + session.pts_matches = 0; + session.pts_mismatches = 0; + session.duplicate_bypass_packets = 0; + session.send_frame_time = {}; + session.receive_packet_time = {}; + }; + + if (session.duplicate_bypass_enabled && !frame_timestamp && !(frame->flags & AV_FRAME_FLAG_KEY) && !session.last_non_idr_packet.empty()) { + auto packet = std::make_unique( + std::vector(session.last_non_idr_packet), + frame_nr, + false + ); + packet->channel_data = channel_data; + packets->raise(std::move(packet)); + session.duplicate_bypass_packets++; + session.output_packets++; + log_encoder_rate(); + return 0; + } + // send the frame to the encoder + auto send_start = std::chrono::steady_clock::now(); auto ret = avcodec_send_frame(ctx.get(), frame); + auto send_end = std::chrono::steady_clock::now(); + session.input_frames++; + session.send_frame_time += send_end - send_start; if (ret < 0) { char err_str[AV_ERROR_MAX_STRING_SIZE] {0}; BOOST_LOG(error) << "Could not send a frame for encoding: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, ret); return -1; } + session.pending_frame_timestamps[frame_nr] = frame_timestamp; while (ret >= 0) { auto packet = std::make_unique(); auto av_packet = packet.get()->av_packet; + auto receive_start = std::chrono::steady_clock::now(); ret = avcodec_receive_packet(ctx.get(), av_packet); + auto receive_end = std::chrono::steady_clock::now(); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + session.no_output_returns++; + log_encoder_rate(); return 0; } else if (ret < 0) { return ret; } + session.output_packets++; + session.receive_packet_time += receive_end - receive_start; + if (av_packet->flags & AV_PKT_FLAG_KEY) { + session.key_packets++; BOOST_LOG(debug) << "Frame "sv << frame_nr << ": IDR Keyframe (AV_FRAME_FLAG_KEY)"sv; } @@ -1573,13 +1951,28 @@ namespace video { ); } - if (av_packet && av_packet->pts == frame_nr) { - packet->frame_timestamp = frame_timestamp; + auto timestamp = av_packet && av_packet->pts != AV_NOPTS_VALUE ? + session.pending_frame_timestamps.find(av_packet->pts) : + std::end(session.pending_frame_timestamps); + if (timestamp != std::end(session.pending_frame_timestamps)) { + session.pts_matches++; + packet->frame_timestamp = timestamp->second; + session.pending_frame_timestamps.erase(timestamp); + } else { + session.pts_mismatches++; + } + while (session.pending_frame_timestamps.size() > 256) { + session.pending_frame_timestamps.erase(std::begin(session.pending_frame_timestamps)); + } + + if (!(av_packet->flags & AV_PKT_FLAG_KEY)) { + session.last_non_idr_packet.assign(av_packet->data, av_packet->data + av_packet->size); } packet->replacements = &session.replacements; packet->channel_data = channel_data; packets->raise(std::move(packet)); + log_encoder_rate(); } return 0; @@ -1605,12 +1998,95 @@ namespace video { return 0; } +#ifdef __APPLE__ + int encode_native_vt(int64_t frame_nr, native_vt_encode_session_t &session, safe::mail_raw_t::queue_t &packets, void *channel_data, std::optional frame_timestamp) { + if (!session.session || !session.current_pixel) { + BOOST_LOG(error) << "Native VideoToolbox encode requested before session/pixel buffer is ready"; + return -1; + } + + CFMutableDictionaryRef frame_options = nullptr; + if (session.force_idr) { + frame_options = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + if (frame_options) { + CFDictionarySetValue(frame_options, kVTEncodeFrameOptionKey_ForceKeyFrame, kCFBooleanTrue); + } + } + + auto *frame_ref = new native_vt_encode_session_t::frame_ref_t { + &session, + packets, + channel_data, + frame_nr, + frame_timestamp, + }; + + auto encode_start = std::chrono::steady_clock::now(); + auto status = VTCompressionSessionEncodeFrame( + session.session, + session.current_pixel, + CMTimeMake(frame_nr, session.config.framerate), + kCMTimeInvalid, + frame_options, + frame_ref, + nullptr + ); + auto encode_end = std::chrono::steady_clock::now(); + + if (frame_options) { + CFRelease(frame_options); + } + + if (status != noErr) { + delete frame_ref; + BOOST_LOG(error) << "Native VideoToolbox encode failed status=" << status; + return -1; + } + + session.input_frames++; + session.encode_call_time += encode_end - encode_start; + session.force_idr = false; + + auto now = std::chrono::steady_clock::now(); + auto elapsed = now - session.last_rate_report; + if (elapsed >= 5s) { + auto elapsed_seconds = std::chrono::duration(elapsed).count(); + auto outputs = session.output_packets.exchange(0); + auto keys = session.key_packets.exchange(0); + auto failures = session.failed_packets.exchange(0); + auto avg_encode_ms = session.input_frames ? + std::chrono::duration(session.encode_call_time).count() / session.input_frames : + 0.0; + + BOOST_LOG(info) << "Native VideoToolbox encode rate inputs_fps=" << (session.input_frames / elapsed_seconds) + << " outputs_fps=" << (outputs / elapsed_seconds) + << " inputs=" << session.input_frames + << " outputs=" << outputs + << " key_packets=" << keys + << " failed_packets=" << failures + << " avg_encode_call_ms=" << avg_encode_ms + << " over_seconds=" << elapsed_seconds; + + session.last_rate_report = now; + session.input_frames = 0; + session.encode_call_time = {}; + } + + return 0; + } +#endif + int encode(int64_t frame_nr, encode_session_t &session, safe::mail_raw_t::queue_t &packets, void *channel_data, std::optional frame_timestamp) { if (auto avcodec_session = dynamic_cast(&session)) { return encode_avcodec(frame_nr, *avcodec_session, packets, channel_data, frame_timestamp); } else if (auto nvenc_session = dynamic_cast(&session)) { return encode_nvenc(frame_nr, *nvenc_session, packets, channel_data, frame_timestamp); } +#ifdef __APPLE__ + else if (auto native_vt_session = dynamic_cast(&session)) { + return encode_native_vt(frame_nr, *native_vt_session, packets, channel_data, frame_timestamp); + } +#endif return -1; } @@ -1995,6 +2471,10 @@ namespace video { // 0 ==> don't inject, 1 ==> inject for h264, 2 ==> inject for hevc config.videoFormat <= 1 ? (1 - (int) video_format[encoder_t::VUI_PARAMETERS]) * (1 + config.videoFormat) : 0 ); + session->duplicate_bypass_enabled = false; + BOOST_LOG(info) << "Compressed duplicate bypass " + << (session->duplicate_bypass_enabled ? "enabled" : "disabled") + << " for negotiated fps=" << config.framerate; return session; } @@ -2008,6 +2488,17 @@ namespace video { } std::unique_ptr make_encode_session(platf::display_t *disp, const encoder_t &encoder, const config_t &config, int width, int height, std::unique_ptr encode_device) { +#ifdef __APPLE__ + if (encoder.name == "videotoolbox"sv && config.videoFormat <= 1) { + auto session = std::make_unique(config, encode_device->colorspace); + if (session->is_ready()) { + BOOST_LOG(info) << "Using native VideoToolbox encoder path"; + return session; + } + BOOST_LOG(error) << "Native VideoToolbox session failed; falling back to AVCodec VideoToolbox"; + } +#endif + if (dynamic_cast(encode_device.get())) { auto avcodec_encode_device = boost::dynamic_pointer_cast(std::move(encode_device)); return make_avcodec_encode_session(disp, encoder, config, width, height, std::move(avcodec_encode_device)); @@ -2055,8 +2546,18 @@ namespace video { // set max frame time based on client-requested target framerate. double minimum_fps_target = (config::video.minimum_fps_target > 0.0) ? config::video.minimum_fps_target : (config.framerate / 2); std::chrono::duration max_frametime {1000.0 / minimum_fps_target}; + auto max_frametime_steady = std::chrono::duration_cast(max_frametime); BOOST_LOG(info) << "Minimum FPS target set to ~"sv << minimum_fps_target << "fps ("sv << max_frametime.count() << "ms)"sv; + auto next_min_frame_time = std::chrono::steady_clock::now(); + auto last_encode_loop_report = next_min_frame_time; + std::uint64_t encode_loop_attempts = 0; + std::uint64_t encode_loop_source_frames = 0; + std::uint64_t encode_loop_duplicate_frames = 0; + std::uint64_t encode_loop_idr_requests = 0; + std::chrono::steady_clock::duration encode_loop_work_time {}; + bool have_real_frame = false; + auto shutdown_event = mail->event(mail::shutdown); auto packets = mail::man->queue(mail::video_packets); auto idr_events = mail->event(mail::idr); @@ -2095,6 +2596,7 @@ namespace video { if (idr_events->peek()) { requested_idr_frame = true; + encode_loop_idr_requests++; idr_events->pop(); } @@ -2103,12 +2605,44 @@ namespace video { } std::optional frame_timestamp; + bool encoded_source_frame = false; + + // Drive output from an independent frame clock. Capture may only deliver + // ~60 Hz on macOS; a 120 Hz stream still needs an encode tick every + // 8.33 ms, reusing the last converted frame when no fresh capture exists. + auto now = std::chrono::steady_clock::now(); + if (now < next_min_frame_time) { + if (minimum_fps_target >= 60.0) { + while (std::chrono::steady_clock::now() < next_min_frame_time) { + } + } else { + std::this_thread::sleep_until(next_min_frame_time); + } + } - // Encode at a minimum FPS to avoid image quality issues with static content - if (!requested_idr_frame || images->peek()) { - if (auto img = images->pop(max_frametime)) { - frame_timestamp = img->frame_timestamp; - if (session->convert(*img)) { + // Consume the newest available captured frame without letting the capture + // queue pace the output loop. Blocking here collapses 120 Hz output back + // toward the physical display/capture refresh rate. + auto img = images->pop(0ms); + while (auto newer_img = images->pop(0ms)) { + img = std::move(newer_img); + } + if (img) { + frame_timestamp = img->frame_timestamp; + encoded_source_frame = true; + have_real_frame = true; + if (session->convert(*img)) { + BOOST_LOG(error) << "Could not convert image"sv; + return; + } + } else if (!images->running()) { + break; + } else if (!have_real_frame) { + if (auto first_img = images->pop(max_frametime_steady)) { + frame_timestamp = first_img->frame_timestamp; + encoded_source_frame = true; + have_real_frame = true; + if (session->convert(*first_img)) { BOOST_LOG(error) << "Could not convert image"sv; return; } @@ -2117,10 +2651,46 @@ namespace video { } } + auto encode_start = std::chrono::steady_clock::now(); if (encode(frame_nr++, *session, packets, channel_data, frame_timestamp)) { BOOST_LOG(error) << "Could not encode video packet"sv; return; } + auto encode_end = std::chrono::steady_clock::now(); + + encode_loop_attempts++; + encode_loop_work_time += encode_end - encode_start; + if (encoded_source_frame) { + encode_loop_source_frames++; + } else { + encode_loop_duplicate_frames++; + } + + next_min_frame_time += max_frametime_steady; + auto next_frame_lag = encode_end - next_min_frame_time; + if (next_frame_lag > max_frametime_steady) { + next_min_frame_time = encode_end + max_frametime_steady; + } + auto now_after_encode = std::chrono::steady_clock::now(); + if (now_after_encode - last_encode_loop_report >= 5s) { + auto elapsed_seconds = std::chrono::duration(now_after_encode - last_encode_loop_report).count(); + auto avg_work_ms = encode_loop_attempts ? + std::chrono::duration(encode_loop_work_time).count() / encode_loop_attempts : + 0.0; + BOOST_LOG(info) << "Encode loop rate fps=" << (encode_loop_attempts / elapsed_seconds) + << " attempts=" << encode_loop_attempts + << " source_frames=" << encode_loop_source_frames + << " duplicates=" << encode_loop_duplicate_frames + << " idr_requests=" << encode_loop_idr_requests + << " avg_encode_ms=" << avg_work_ms + << " over_seconds=" << elapsed_seconds; + encode_loop_attempts = 0; + encode_loop_source_frames = 0; + encode_loop_duplicate_frames = 0; + encode_loop_idr_requests = 0; + encode_loop_work_time = {}; + last_encode_loop_report = now_after_encode; + } session->request_normal_frame();