From 0ddbcaf9048fba33a15d7246f413162d148b6e02 Mon Sep 17 00:00:00 2001 From: szdziedzic Date: Wed, 5 Aug 2026 14:09:01 +0200 Subject: [PATCH 1/4] feat: add scale-only screenshot sizing --- .../RunnerTests+CommandExecution.swift | 12 +- .../RunnerTests+Models.swift | 1 - .../RunnerTests+ScreenRecorder.swift | 24 +- .../RecordingScripts/recording-resize.swift | 225 ------------------ apple/runner/RUNNER_PROTOCOL.md | 2 +- .../src/internal/__tests__/script.test.ts | 12 +- .../ad-script/src/internal/script-utils.ts | 3 - packages/ad-script/src/internal/script.ts | 8 - packages/contracts/src/client-capture.ts | 2 +- .../contracts/src/client-observability.ts | 1 - packages/contracts/src/client-request.ts | 1 - packages/contracts/src/screenshot.ts | 91 +++++-- scripts/integration-progress-model.ts | 2 +- scripts/layering/facade-symbols.ts | 2 + src/__tests__/cli-client-commands.test.ts | 6 +- src/__tests__/cli-config.test.ts | 24 ++ src/__tests__/client.test.ts | 23 ++ src/backend.ts | 1 - src/cli-schema/cli-config.ts | 2 +- src/cli-schema/option-schema.test.ts | 6 + .../__tests__/args-parse-interaction.test.ts | 21 +- .../__tests__/cli-help-command-usage.test.ts | 4 +- src/cli/parser/args.ts | 5 +- src/cli/parser/cli-help.ts | 6 +- src/commands/capture/index.test.ts | 6 +- src/commands/capture/runtime/screenshot.ts | 8 +- .../capture/screenshot-options.test.ts | 30 ++- src/commands/capture/screenshot.ts | 23 +- src/commands/cli-grammar/flag-types.ts | 2 +- src/commands/command-flags.ts | 1 - src/commands/command-input.ts | 33 ++- src/commands/recording/index.test.ts | 2 - src/commands/recording/index.ts | 8 +- .../recording/runtime/recording.test.ts | 2 - src/commands/recording/runtime/recording.ts | 9 - src/commands/runtime-types.ts | 2 +- src/daemon/__tests__/context.test.ts | 4 +- .../request-router-screenshot.test.ts | 12 +- src/daemon/__tests__/session-store.test.ts | 10 +- ...cord-trace-ios-simulator-recording.test.ts | 1 - .../__tests__/record-trace-ios.test.ts | 1 - .../handlers/__tests__/record-trace.test.ts | 157 ++---------- .../handlers/record-trace-android-recovery.ts | 2 - src/daemon/handlers/record-trace-android.ts | 83 +------ .../record-trace-ios-simulator-recording.ts | 21 -- src/daemon/handlers/record-trace-ios.ts | 2 - .../record-trace-recording-backends.ts | 1 - src/daemon/handlers/record-trace-recording.ts | 33 +-- src/daemon/handlers/record-trace-types.ts | 8 +- src/daemon/request-generic-dispatch.ts | 2 +- src/daemon/session-event-action.ts | 4 +- src/daemon/types.ts | 1 - .../core/__tests__/runner-client.test.ts | 2 - .../apple/core/runner/runner-contract.ts | 1 - src/recording/__tests__/overlay.test.ts | 28 +-- .../__tests__/recording-scripts.test.ts | 12 - src/recording/overlay.ts | 26 -- src/utils/__tests__/png-resize.test.ts | 30 +-- src/utils/png-resize.ts | 36 +-- src/utils/screenshot-density.ts | 4 +- src/utils/source-value.test.ts | 13 + src/utils/source-value.ts | 6 +- .../live-capture-scenario.ts | 4 +- .../live-observability-scenario.ts | 2 - .../ios-simulator-e2e/live-runner.ts | 4 +- .../integration/live-device-e2e/assertions.ts | 2 +- .../android-recording.test.ts | 24 +- .../provider-scenarios/assertions.ts | 6 + .../ios-record-trace.test.ts | 4 +- .../provider-scenarios/macos-desktop.test.ts | 32 ++- .../macos-recording.test.ts | 2 - website/docs/docs/client-api.md | 4 +- website/docs/docs/commands.md | 10 +- website/docs/docs/configuration.md | 6 +- 74 files changed, 403 insertions(+), 807 deletions(-) delete mode 100644 apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 3446c5e10c..148ec37caa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1475,24 +1475,18 @@ extension RunnerTests { if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) { return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)")) } - if let requestedMaxSize = command.maxSize, requestedMaxSize < 1 { - return Response(ok: false, error: ErrorPayload(message: "recordStart maxSize must be a positive integer")) - } do { let resolvedOutPath = resolveRecordingOutPath(requestedOutPath) let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps) - let maxSizeLabel = command.maxSize.map(String.init) ?? "native" NSLog( - "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@ maxSize=%@", + "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@", requestedOutPath, resolvedOutPath, - fpsLabel, - maxSizeLabel + fpsLabel ) let recorder = ScreenRecorder( outputPath: resolvedOutPath, - fps: command.fps.map { Int32($0) }, - maxSize: command.maxSize + fps: command.fps.map { Int32($0) } ) try recorder.start { [weak self] in return self?.captureRunnerFrame() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index d153acfb71..feb3eff78b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -136,7 +136,6 @@ struct Command: Codable { let gesturePlan: RunnerGesturePlan? let outPath: String? let fps: Int? - let maxSize: Int? let interactiveOnly: Bool? let depth: Int? let scope: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift index 668489454a..40b96ec935 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift @@ -7,7 +7,6 @@ extension RunnerTests { final class ScreenRecorder { private let outputPath: String private let fps: Int32? - private let maxSize: Int? private var effectiveFps: Int32 { max(1, fps ?? RunnerTests.defaultRecordingFps) } @@ -26,10 +25,9 @@ extension RunnerTests { private var startedSession = false private var startError: Error? - init(outputPath: String, fps: Int32?, maxSize: Int?) { + init(outputPath: String, fps: Int32?) { self.outputPath = outputPath self.fps = fps - self.maxSize = maxSize } func start(captureFrame: @escaping () -> RunnerImage?) throws { @@ -50,7 +48,7 @@ extension RunnerTests { while Date() < bootstrapDeadline { if let image = captureFrame(), let cgImage = runnerCGImage(from: image) { bootstrapImage = image - dimensions = scaledDimensions(width: cgImage.width, height: cgImage.height) + dimensions = CGSize(width: cgImage.width, height: cgImage.height) break } Thread.sleep(forTimeInterval: 0.05) @@ -261,23 +259,5 @@ extension RunnerTests { return pixelBuffer } - private func scaledDimensions(width: Int, height: Int) -> CGSize { - guard let maxSize, maxSize > 0 else { - return CGSize(width: width, height: height) - } - let longest = max(width, height) - guard longest > maxSize else { - return CGSize(width: width, height: height) - } - let scale = Double(maxSize) / Double(longest) - return CGSize( - width: scaledEvenDimension(width, scale: scale), - height: scaledEvenDimension(height, scale: scale) - ) - } - - private func scaledEvenDimension(_ value: Int, scale: Double) -> Int { - max(2, Int((Double(value) * scale / 2.0).rounded()) * 2) - } } } diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift deleted file mode 100644 index 1557617bd9..0000000000 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +++ /dev/null @@ -1,225 +0,0 @@ -import AVFoundation -import Foundation - -enum ResizeError: Error, CustomStringConvertible { - case invalidArgs(String) - case missingVideoTrack - case exportFailed(String) - - var description: String { - switch self { - case .invalidArgs(let message): - return message - case .missingVideoTrack: - return "Input video does not contain a video track." - case .exportFailed(let message): - return message - } - } -} - -do { - try run() -} catch { - fputs("recording-resize: \(error)\n", stderr) - exit(1) -} - -func run() throws { - let arguments = Array(CommandLine.arguments.dropFirst()) - let parsedArgs = try parseArguments(arguments) - let inputURL = URL(fileURLWithPath: parsedArgs.inputPath) - let outputURL = URL(fileURLWithPath: parsedArgs.outputPath) - - if FileManager.default.fileExists(atPath: outputURL.path) { - try FileManager.default.removeItem(at: outputURL) - } - - let asset = AVURLAsset(url: inputURL) - guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else { - throw ResizeError.missingVideoTrack - } - - let sourceRenderSize = resolvedRenderSize(for: sourceVideoTrack) - if max(sourceRenderSize.width, sourceRenderSize.height) <= CGFloat(parsedArgs.maxSize) { - try FileManager.default.copyItem(at: inputURL, to: outputURL) - return - } - - let renderSize = scaledRenderSize(sourceRenderSize, maxSize: parsedArgs.maxSize) - let composition = AVMutableComposition() - let fullRange = CMTimeRange(start: .zero, duration: asset.duration) - - guard let compositionVideoTrack = composition.addMutableTrack( - withMediaType: .video, - preferredTrackID: kCMPersistentTrackID_Invalid - ) else { - throw ResizeError.exportFailed("Failed to create composition video track.") - } - try compositionVideoTrack.insertTimeRange(fullRange, of: sourceVideoTrack, at: .zero) - - if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first, - let compositionAudioTrack = composition.addMutableTrack( - withMediaType: .audio, - preferredTrackID: kCMPersistentTrackID_Invalid - ) { - try? compositionAudioTrack.insertTimeRange(fullRange, of: sourceAudioTrack, at: .zero) - } - - let scale = renderSize.width / sourceRenderSize.width - let videoComposition = AVMutableVideoComposition() - videoComposition.renderSize = renderSize - videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack) - - let instruction = AVMutableVideoCompositionInstruction() - instruction.timeRange = fullRange - let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack) - // Scale the full preferred transform (including translation) to match the smaller render canvas. - let scaledTransform = scaledPreferredTransform(sourceVideoTrack.preferredTransform, scale: scale) - layerInstruction.setTransform(scaledTransform, at: .zero) - instruction.layerInstructions = [layerInstruction] - videoComposition.instructions = [instruction] - - let presetName = exportPresetName(for: parsedArgs.exportQuality, compatibleWith: composition) - guard let exporter = AVAssetExportSession(asset: composition, presetName: presetName) else { - throw ResizeError.exportFailed("Failed to create export session.") - } - - exporter.outputURL = outputURL - exporter.outputFileType = .mp4 - exporter.videoComposition = videoComposition - exporter.shouldOptimizeForNetworkUse = true - - let semaphore = DispatchSemaphore(value: 0) - exporter.exportAsynchronously { - semaphore.signal() - } - if semaphore.wait(timeout: .now() + 120) == .timedOut { - exporter.cancelExport() - throw ResizeError.exportFailed("Resize export timed out.") - } - - if exporter.status != .completed { - throw ResizeError.exportFailed(exporter.error?.localizedDescription ?? "Resize export failed.") - } -} - -enum ExportQuality: String { - case medium - case high -} - -func parseArguments( - _ arguments: [String] -) throws -> (inputPath: String, outputPath: String, maxSize: Int, exportQuality: ExportQuality) { - var inputPath: String? - var outputPath: String? - var maxSize: Int? - // Export quality defaults to medium so re-encoded recordings stay fast by default. - // Pass --quality high to opt into a slower highest-quality export. - var exportQuality: ExportQuality = .medium - var index = 0 - - while index < arguments.count { - let argument = arguments[index] - let nextIndex = index + 1 - switch argument { - case "--input": - guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--input requires a value") } - inputPath = arguments[nextIndex] - index += 2 - case "--output": - guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--output requires a value") } - outputPath = arguments[nextIndex] - index += 2 - case "--max-size": - guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--max-size requires a value") } - guard let parsed = Int(arguments[nextIndex]), parsed >= 1 else { - throw ResizeError.invalidArgs("--max-size must be a positive integer") - } - maxSize = parsed - index += 2 - case "--quality": - guard nextIndex < arguments.count else { - throw ResizeError.invalidArgs("--quality requires a value") - } - guard let parsed = ExportQuality(rawValue: arguments[nextIndex]) else { - throw ResizeError.invalidArgs("--quality must be one of: medium, high") - } - exportQuality = parsed - index += 2 - default: - throw ResizeError.invalidArgs("Unknown argument: \(argument)") - } - } - - guard let inputPath, let outputPath, let maxSize else { - throw ResizeError.invalidArgs( - "Usage: recording-resize.swift --input