diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index df07f06e84..9ef3eefefb 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -92,6 +92,11 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCachedTargetInvalidationClearsProcessBoundState \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index e9b508856d..3b5d0c2f20 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -62,6 +62,13 @@ @interface AgentDeviceRunnerViewController : UIViewController @implementation AgentDeviceRunnerViewController +- (void)agentDeviceTextEntryDidChange:(UITextField *)textField { + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-disappear-after-input"] && + textField.text.length > 0) { + [textField removeFromSuperview]; + } +} + - (void)viewDidLoad { [super viewDidLoad]; @@ -78,6 +85,27 @@ - (void)viewDidLoad { [label.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], [label.centerYAnchor constraintEqualToAnchor:self.view.centerYAnchor], ]]; + + // Keep the fixture behind a launch argument so normal runner snapshots remain unchanged. +#if TARGET_OS_IOS + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { + UITextField *textField = [[UITextField alloc] init]; + textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; + textField.borderStyle = UITextBorderStyleRoundedRect; + textField.inputView = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 1, 1)]; + [textField addTarget:self + action:@selector(agentDeviceTextEntryDidChange:) + forControlEvents:UIControlEventEditingChanged]; + textField.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:textField]; + [NSLayoutConstraint activateConstraints:@[ + [textField.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [textField.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24], + [textField.widthAnchor constraintEqualToConstant:240], + [textField.heightAnchor constraintEqualToConstant:44], + ]]; + } +#endif } @end diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index dd054956a2..3446c5e10c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -340,6 +340,105 @@ extension RunnerTests { } #endif +#if os(iOS) + func testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText() throws { + let command = try runnerCommandFixture( + #"{"command":"type","commandId":"type-without-focus","text":"hello"}"# + ) + + let response = executeTypeCommand( + activeApp: XCUIApplication(bundleIdentifier: "com.example.agentdevice.missing-input"), + command: command + ) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertEqual( + response.error?.hint, + "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing." + ) + } + + func testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden() throws { + // The fixture uses a real text responder with an empty input view to model hardware-keyboard input. + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let frame = textField.frame + XCTAssertFalse(frame.isEmpty) + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-hardware-keyboard-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + XCTAssertFalse( + isKeyboardVisible(app: app), + "the test must exercise a focused responder with the software keyboard hidden" + ) + + let failureCountBefore = currentXCTestFailureCount() + let typeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard","text":"hardware-keyboard"}"# + ) + let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + + XCTAssertTrue(typeResponse.ok, String(describing: typeResponse.error)) + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertEqual(String(describing: textField.value ?? ""), "hardware-keyboard") + + let secondFailureCountBefore = currentXCTestFailureCount() + let secondTypeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard-again","text":"-again"}"# + ) + let secondTypeResponse = executeTypeCommand(activeApp: app, command: secondTypeCommand) + + XCTAssertFalse(secondTypeResponse.ok) + XCTAssertEqual(secondTypeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(didRecordXCTestFailure(since: secondFailureCountBefore)) + XCTAssertEqual(String(describing: textField.value ?? ""), "hardware-keyboard") + } + + func testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand() throws { + app.launchArguments = [ + "--agent-device-text-entry-regression", + "--agent-device-text-entry-disappear-after-input", + ] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-disappearing-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + + let failureCountBefore = currentXCTestFailureCount() + let typeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-disappearing-input","text":"ab","delayMs":50}"# + ) + let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(typeResponse.ok) + XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(textField.exists) + } +#endif + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) let response = Response(ok: true, data: DataPayload(message: "tapped")) @@ -1528,6 +1627,9 @@ extension RunnerTests { alertDeadline: Date? = nil ) throws -> Response { var activeApp = activeApp + if command.command != .tap && command.command != .type && !isReadOnlyCommand(command) { + clearRememberedTextEntryTap() + } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime: return Response( @@ -1556,6 +1658,7 @@ extension RunnerTests { expectedPoint: expectedPoint ) if match.isAmbiguous { + clearRememberedTextEntryTap() return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) } if let element = match.element { @@ -1574,6 +1677,7 @@ extension RunnerTests { elementFrame: frame, windowFrame: onScreenWindowFrame(app: activeApp) ) { + clearRememberedTextEntryTap() return Response(ok: false, error: ErrorPayload( code: "ELEMENT_OFFSCREEN", message: "element resolved off-screen at (\(Int(frame.midX)), \(Int(frame.midY)))")) @@ -1609,6 +1713,7 @@ extension RunnerTests { if isTextEntry { waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } + rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" @@ -1639,11 +1744,13 @@ extension RunnerTests { return activateElement(app: activeApp, element: element, action: "tap by selector") } if let response = unsupportedResponse(for: outcome) { + clearRememberedTextEntryTap() return response } if isTextEntry { waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } + rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped", timing: timing, @@ -1655,9 +1762,11 @@ extension RunnerTests { : nil ) } + clearRememberedTextEntryTap() return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found")) } if let x = command.x, let y = command.y { + let textInput = textInputAt(app: activeApp, x: x, y: y) var fallback: GestureFallback? if command.synthesized == true { let policyKind = SynthesizedGesturePolicyKind.coordinateTap @@ -1670,6 +1779,7 @@ extension RunnerTests { } if case .performed = outcome { logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) + rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) } logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) @@ -1678,8 +1788,10 @@ extension RunnerTests { let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } if let response = unsupportedResponse(for: outcome) { + clearRememberedTextEntryTap() return response } + rememberTextEntryTap(textInput) return gestureResponse( message: "tapped", timing: timing, @@ -1687,6 +1799,7 @@ extension RunnerTests { fallback: fallback ) } + clearRememberedTextEntryTap() return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y")) case .mouseClick: guard let x = command.x, let y = command.y else { @@ -2578,6 +2691,12 @@ extension RunnerTests { synthesizer: PrivateXCTestTextEntrySynthesizer(), commandId: command.commandId ) + if let failure = textResult.failure { + return Response( + ok: false, + error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) + ) + } if textResult.verified == false { let expected = textResult.expectedText ?? "" let observed = textResult.observedText ?? "" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 972facbcec..389e3300aa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -86,6 +86,7 @@ extension RunnerTests { currentApp = app currentBundleId = nil currentAppProcessIdentifier = nil + clearRememberedTextEntryTap() snapshotXCTestPenaltyWarmupExemptionPending = false } @@ -96,6 +97,7 @@ extension RunnerTests { currentApp = nil currentBundleId = nil currentAppProcessIdentifier = nil + clearRememberedTextEntryTap() snapshotXCTestPenaltyWarmupExemptionPending = false } @@ -121,6 +123,7 @@ extension RunnerTests { ) currentApp = candidate currentAppProcessIdentifier = observedProcessIdentifier + clearRememberedTextEntryTap() clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") clearPrivateAXAcceptedDepth(reason: "target_process_changed") snapshotXCTestPenaltyWarmupExemptionPending = true @@ -195,6 +198,7 @@ extension RunnerTests { currentApp = target currentBundleId = bundleId currentAppProcessIdentifier = Self.processIdentifier(of: target) + clearRememberedTextEntryTap() snapshotXCTestPenaltyWarmupExemptionPending = false needsFirstInteractionDelay = true return target diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift index e46d670423..2c2aee314b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift @@ -49,8 +49,14 @@ extension RunnerTests { } RunnerTargetActivationSpy.state = .runningForeground + textEntryTapWitness = TextEntryTapWitness( + element: app, + bundleId: "com.example.previous", + processIdentifier: 41 + ) _ = activateTarget(bundleId: "com.example.foreground", reason: "unit_test") XCTAssertEqual(RunnerTargetActivationSpy.activationCount, 0) + XCTAssertNil(textEntryTapWitness) RunnerTargetActivationSpy.state = .runningBackground _ = activateTarget(bundleId: "com.example.background", reason: "unit_test") @@ -110,6 +116,18 @@ extension RunnerTests { XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) } + func testTextEntryTapWitnessIsBoundToTargetIdentity() { + let witness = TextEntryTapWitness( + element: app, + bundleId: "com.example.app", + processIdentifier: 42 + ) + + XCTAssertTrue(witness.matches(bundleId: "com.example.app", processIdentifier: 42)) + XCTAssertFalse(witness.matches(bundleId: "com.example.other", processIdentifier: 42)) + XCTAssertFalse(witness.matches(bundleId: "com.example.app", processIdentifier: 43)) + } + func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() { currentApp = app currentBundleId = "com.example.app" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index e27431a63d..07670f177b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -4,6 +4,24 @@ import XCTest // pipeline, readiness polling, and field clearing. Behavior-preserving extraction from // RunnerTests+Interaction.swift (no logic changes) to keep that file navigable. extension RunnerTests { + enum TextEntryFailure: String { + case notFocused = "TEXT_INPUT_NOT_FOCUSED" + + var message: String { + switch self { + case .notFocused: + return "No focused text input was available for typing." + } + } + + var hint: String { + switch self { + case .notFocused: + return "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing." + } + } + } + enum TextTypingRepairMode { case none case append @@ -25,6 +43,7 @@ extension RunnerTests { let expectedText: String? let observedText: String? var textEntryRoute: String? = nil + var failure: TextEntryFailure? = nil } struct TextEntryTarget { @@ -51,6 +70,16 @@ extension RunnerTests { let focusConfirmed: Bool } + struct TextEntryTapWitness { + let element: XCUIElement + let bundleId: String? + let processIdentifier: Int? + + func matches(bundleId: String?, processIdentifier: Int?) -> Bool { + self.bundleId == bundleId && self.processIdentifier == processIdentifier + } + } + func clearTextInput(_ element: XCUIElement) { // Skip the clear (delete burst + moveCaretToEnd edge-tap) ONLY when we can confirm the // field is empty. Why skip: the edge-tap computes a point from the element frame, which can @@ -96,6 +125,49 @@ extension RunnerTests { #endif } + func rememberTextEntryTap(_ element: XCUIElement?) { + guard let element, isTextEntryElement(element) else { + clearRememberedTextEntryTap() + return + } + textEntryTapWitness = TextEntryTapWitness( + element: element, + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) + } + + func clearRememberedTextEntryTap() { + textEntryTapWitness = nil + } + + private func rememberedTextEntryTarget() -> TextEntryTarget? { + guard let witness = textEntryTapWitness else { + return nil + } + // The tap is proof for one immediately-following bare type only. Consume it before checking + // the element so a failed or interrupted type cannot reuse stale focus evidence. + clearRememberedTextEntryTap() + guard witness.matches( + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) else { + return nil + } + let element = witness.element + // XCUIElement is query-backed rather than a stable node identity. A same-identifier field + // introduced by app-side navigation between tap and this immediate type can therefore + // re-resolve here; keep the witness one-shot and fail closed on every observable identity + // boundary instead of using frame equality, which would reject legitimate layout changes. + guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { + return nil + } + // Keep the target scoped to the element that the preceding tap actually selected. Do not + // attach a refresh point: if that element disappeared, bare type must fail closed rather + // than rediscovering a different field or dispatching unscoped app.typeText. + return TextEntryTarget(element: element, refreshPoint: nil, prefersFocusedElement: false) + } + func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, @@ -124,11 +196,15 @@ extension RunnerTests { func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { guard let x, let y else { + let softwareKeyboardVisible = isKeyboardVisible(app: app) + if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() { + return rememberedTarget + } // Bare `type` targets the current first responder. On iOS we intentionally do not trust // `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that // app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger // target because there is no selector/coordinate focus move to validate. - if isKeyboardVisible(app: app) { + if softwareKeyboardVisible { return TextEntryTarget( element: focusedTextInput(app: app), refreshPoint: nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 7aff2bbb52..9fb65e7f4d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -60,6 +60,26 @@ extension RunnerTests { mode: repairMode ) + // Dispatching text through XCTest without a resolved target or evidence of a focused + // responder records a test failure. That tears down the long-lived runner and turns a + // single invalid request into a restart cascade, so fail before entering that channel. + guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else { + logTextEntryPhase( + commandId: commandId, + phase: "total", + startedAt: totalStartedAt, + chars: text.count, + mode: repairMode + ) + return TextEntryResult( + verified: nil, + repaired: false, + expectedText: expectedText, + observedText: nil, + failure: .notFocused + ) + } + if repairMode == .replacement { guard let replacementTarget = initialTarget else { logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode) @@ -79,12 +99,12 @@ extension RunnerTests { } } - func typeIntoCurrentTarget(_ value: String) -> XCUIElement? { + func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool) { if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) { textEntryRoute = "xctest-element" currentTarget.typeText(value) - return currentTarget - } else { + return (currentTarget, true) + } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") @@ -108,8 +128,20 @@ extension RunnerTests { #else app.typeText(value) #endif - return resolveTextEntryElement(app: app, target: activeTarget) + return (resolveTextEntryElement(app: app, target: activeTarget), true) } + return (nil, false) + } + + func dispatchFailureResult() -> TextEntryResult { + TextEntryResult( + verified: nil, + repaired: false, + expectedText: expectedText, + observedText: nil, + textEntryRoute: textEntryRoute, + failure: .notFocused + ) } func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) { @@ -131,7 +163,11 @@ extension RunnerTests { var typedTarget: XCUIElement? let delayedTypeStartedAt = Date() for (index, character) in characters.enumerated() { - typedTarget = typeIntoCurrentTarget(String(character)) ?? typedTarget + let dispatch = typeIntoCurrentTarget(String(character)) + guard dispatch.dispatched else { + return dispatchFailureResult() + } + typedTarget = dispatch.element ?? typedTarget if index + 1 < characters.count { sleepFor(delaySeconds) } @@ -176,7 +212,11 @@ extension RunnerTests { if repairMode != .none && characters.count > 1 { let firstCharacter = String(characters[0]) let firstStartedAt = Date() - var firstTypedTarget = typeIntoCurrentTarget(firstCharacter) + let firstDispatch = typeIntoCurrentTarget(firstCharacter) + guard firstDispatch.dispatched else { + return dispatchFailureResult() + } + var firstTypedTarget = firstDispatch.element logTextEntryPhase( commandId: commandId, phase: "type-first", @@ -201,7 +241,11 @@ extension RunnerTests { ) let remainingText = String(characters.dropFirst()) let remainingStartedAt = Date() - firstTypedTarget = typeIntoCurrentTarget(remainingText) ?? firstTypedTarget + let remainingDispatch = typeIntoCurrentTarget(remainingText) + guard remainingDispatch.dispatched else { + return dispatchFailureResult() + } + firstTypedTarget = remainingDispatch.element ?? firstTypedTarget logTextEntryPhase( commandId: commandId, phase: "type-remaining", @@ -212,7 +256,11 @@ extension RunnerTests { typedTarget = firstTypedTarget } else { let typeStartedAt = Date() - typedTarget = typeIntoCurrentTarget(text) + let dispatch = typeIntoCurrentTarget(text) + guard dispatch.dispatched else { + return dispatchFailureResult() + } + typedTarget = dispatch.element logTextEntryPhase( commandId: commandId, phase: "type-all", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 9fff84eec0..56eeaf5526 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -48,6 +48,11 @@ final class RunnerTests: XCTestCase { var currentApp: XCUIApplication? var currentBundleId: String? var currentAppProcessIdentifier: Int? + // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when + // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text + // input is a scoped witness for the immediately-following bare type; lifecycle and non-text + // interactions clear it before it can become stale. + var textEntryTapWitness: TextEntryTapWitness? let maxRequestBytes = 2 * 1024 * 1024 let mainThreadExecutionTimeout: TimeInterval = 30 let appExistenceTimeout: TimeInterval = 30 diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 752d757031..20fa7a1b34 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -383,6 +383,7 @@ agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and `fill` clears then types. `type` does not clear. `type` accepts text only. Do not pass `@ref` to `type`; use `fill @ref "text"` to target a field directly, or `press @ref` then `type "text"` to append in the focused field. +If `type` reports `TEXT_INPUT_NOT_FOCUSED`, focus a visible text input and retry; when accessibility does not expose the input, use a coordinate focus command before typing. Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates. Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update. On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field.