From c9575ade88508fe0d9b6c6b0e86f03711abfb63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 7 Aug 2026 14:27:53 +0200 Subject: [PATCH] fix(ios): wait for hidden-keyboard synthesized text to commit before responding The synthesized-first-responder bare-type route returned ok as soon as the private XCTest event record was posted, while the target app was still committing characters. On slow CI simulators the trailing characters landed after the response, so agents (and the smoke test) observed a truncated field value through the public type path. After dispatch, poll the tapped element until its value reaches textBefore + typedText, exit immediately when the app transforms the input (formatter, mid-text caret, autocomplete), and if progress stalls as a strict prefix, re-synthesize the missing tail once. Submit-suffixed text keeps the old immediate return so a repair can never double-submit. Validated on a booted iPhone 17 Pro simulator: 5/5 passes of testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden under full-core CPU load, with the commit wait absorbing up to ~390ms of post-dispatch lag that the previous code ignored (uniform ~494ms dispatch-only before); the tail repair never had to fire, consistent with commit lag rather than true drops. --- .../RunnerTests+SynthesizedTextEntry.swift | 100 ++++++++++++++++++ .../RunnerTests+TextEntry.swift | 2 + .../RunnerTests+TextEntryPolicyTests.swift | 50 +++++++++ .../RunnerTests+TextTyping.swift | 15 +++ 4 files changed, 167 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 44ae40896..26da64df9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -159,6 +159,106 @@ extension RunnerTests { repairMode == .none && fromTapWitness && !softwareKeyboardVisible } + enum SynthesizedTextCommitProgress: Equatable { + case committed + case pending + case diverged + } + + // The private synthesize call returns once the event record is posted, not once the target + // app has committed the characters, so intermediate reads walk prefix-by-prefix toward the + // expected value. Anything off that prefix path means the app transformed the input + // (formatter, mid-text caret, autocomplete) and the runner must not second-guess it. + static func synthesizedTextCommitProgress( + observedText: String?, + expectedText: String + ) -> SynthesizedTextCommitProgress { + guard let observedText else { + return .diverged + } + if observedText == expectedText { + return .committed + } + return expectedText.hasPrefix(observedText) ? .pending : .diverged + } + + static func synthesizedTextCommitRepairTail( + observedText: String, + expectedText: String + ) -> String? { + guard expectedText.hasPrefix(observedText), observedText.count < expectedText.count else { + return nil + } + let tail = String(expectedText.dropFirst(observedText.count)) + guard !tail.contains("\n"), !tail.contains("\r") else { + return nil + } + return tail + } + + /// Blocks until the synthesized bare-type text is observable in the target field, so `type` + /// cannot report ok while trailing characters are still uncommitted (or dropped) on a slow + /// simulator. Exits fast when the app transforms the input; re-synthesizes the missing tail + /// once if commit progress stalls as a strict prefix of the expected value. + func awaitSynthesizedFirstResponderCommit( + app: XCUIApplication, + target: TextEntryTarget, + textBefore: String?, + typedText: String, + synthesizer: any TextEntrySynthesizing + ) { + guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else { + return + } + let expectedText = textBefore + typedText + var repaired = false + var lastObservedText: String? + var lastChangeAt = Date() + let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout) + while Date() < deadline { + guard let observedText = editableTextValue( + for: resolveTextEntryElement(app: app, target: target), + treatingPlaceholderAsEmpty: true + ) else { + return + } + switch Self.synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) { + case .committed, .diverged: + return + case .pending: + break + } + if lastObservedText != observedText { + lastObservedText = observedText + lastChangeAt = Date() + } else if Date().timeIntervalSince(lastChangeAt) >= TextEntryTiming.synthesizedCommitQuietWindow { + guard !repaired, + let tail = Self.synthesizedTextCommitRepairTail( + observedText: observedText, + expectedText: expectedText + ) + else { + return + } + NSLog( + "AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY route=synthesized-first-responder expectedLength=%d observedLength=%d", + expectedText.count, + observedText.count + ) + guard case .continueTyping = synthesizer.enterText( + app: app, + text: tail, + replacingExistingText: false + ) else { + return + } + repaired = true + lastChangeAt = Date() + } + sleepFor(TextEntryTiming.pollInterval) + } + } + static func shouldUseResolvedCoordinateTextEntryRoute( repairMode: TextTypingRepairMode, hasX: Bool, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index cd200ccb8..6cc987ba9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -40,6 +40,8 @@ extension RunnerTests { static let pollInterval: TimeInterval = 0.02 static let warmupValueTimeout: TimeInterval = 0.4 static let verificationStabilityWindow: TimeInterval = 0.2 + static let synthesizedCommitTimeout: TimeInterval = 3.0 + static let synthesizedCommitQuietWindow: TimeInterval = 0.6 } struct TextEntryResult { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift index 12f7a9414..71063891a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift @@ -61,6 +61,56 @@ extension RunnerTests { } } + func testSynthesizedTextCommitProgressWalksExpectedPrefixOnly() { + let expected = "hardware-keyboard" + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: "hardware-keyboard", expectedText: expected), + .committed + ) + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: "", expectedText: expected), + .pending + ) + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: "hardware-keyboa", expectedText: expected), + .pending + ) + // Transformed input (formatter, mid-text caret, autocomplete) must stop the wait. + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: "hardwarX", expectedText: expected), + .diverged + ) + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: "hardware-keyboards", expectedText: expected), + .diverged + ) + XCTAssertEqual( + Self.synthesizedTextCommitProgress(observedText: nil, expectedText: expected), + .diverged + ) + } + + func testSynthesizedTextCommitRepairTailOnlyForStrictPrefixWithoutSubmitKeys() { + XCTAssertEqual( + Self.synthesizedTextCommitRepairTail(observedText: "hardware-keyboa", expectedText: "hardware-keyboard"), + "rd" + ) + XCTAssertEqual( + Self.synthesizedTextCommitRepairTail(observedText: "", expectedText: "abc"), + "abc" + ) + XCTAssertNil( + Self.synthesizedTextCommitRepairTail(observedText: "hardware-keyboard", expectedText: "hardware-keyboard") + ) + XCTAssertNil( + Self.synthesizedTextCommitRepairTail(observedText: "hardwarX", expectedText: "hardware-keyboard") + ) + // Never re-synthesize a tail that would submit. + XCTAssertNil( + Self.synthesizedTextCommitRepairTail(observedText: "ab", expectedText: "abc\n") + ) + } + #if os(iOS) func testSynthesizedTextEntryFallsBackOnlyWhenPrivateSynthesisIsUnavailable() { XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 06c47b17a..8ac31afa0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -111,8 +111,23 @@ extension RunnerTests { { textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") + let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true) switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) { case .continueTyping: + // No refresh point: like the tap-witness target itself, the commit wait must observe + // only the element the tap selected, never rediscover a different field. + awaitSynthesizedFirstResponderCommit( + app: app, + target: TextEntryTarget( + element: currentTarget, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ), + textBefore: textBefore, + typedText: value, + synthesizer: synthesizer + ) return (currentTarget, true, nil) case .fallback: return (nil, false, .synthesisUnavailable)