diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 2d80bbc69eb8..b8780bf626be 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -408,6 +408,13 @@ strip = "symbols" # See https://github.com/openai/codex/issues/1411 for details. codegen-units = 1 +# Fast local release builds: no LTO + parallel codegen. +# Use with: cargo build --profile local-release +[profile.local-release] +inherits = "release" +lto = false +codegen-units = 8 + [profile.ci-test] debug = 1 # Reduce debug symbol size inherits = "test" diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index fc36adaa550f..ed7b580c9ddd 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -388,6 +388,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -2871,6 +2879,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 444c8af7c250..ffd9ffe6c385 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5492,6 +5492,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -12837,6 +12845,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/v2/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index e604157ad6e5..ab4fcb1c4c74 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -715,6 +715,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -10692,6 +10700,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 39641078658f..8ef1be917a72 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -816,6 +824,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index abb8aee5dc81..5a145ce6de40 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -816,6 +824,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 2e0c3605e7d4..17edb99d3e81 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -959,6 +967,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 5ae0c5a122b5..4f3d6f2d15dc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -66,6 +66,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1466,6 +1474,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 126d78603a11..b81978907aae 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index dfdab228df3f..08190ac5ed2b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 8f48dee4b735..422a6a02b985 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index edccc337da29..441e134c5338 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -66,6 +66,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1466,6 +1474,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index cc41aac27f51..4c83fd8297c1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index c3b50fee3b01..1a8735e7ae45 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -66,6 +66,14 @@ } ] }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1466,6 +1474,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 2240150394df..45820563030c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 41b0d2d409c3..a97691108f1b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -4,6 +4,14 @@ "AgentPath": { "type": "string" }, + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -1224,6 +1232,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 770cc920cfb8..823ec5946bb4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -959,6 +967,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 7f1c3e4948dd..79bf729cd508 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -959,6 +967,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 761ddc9a624c..4d32ab81926b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1,6 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BuiltinToolCallStatus": { + "enum": [ + "inProgress", + "completed", + "failed" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { @@ -959,6 +967,53 @@ "title": "McpToolCallThreadItem", "type": "object" }, + { + "properties": { + "arguments": true, + "callId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "output": true, + "status": { + "$ref": "#/definitions/BuiltinToolCallStatus" + }, + "success": { + "type": [ + "boolean", + "null" + ] + }, + "tool": { + "type": "string" + }, + "type": { + "enum": [ + "builtinToolCall" + ], + "title": "BuiltinToolCallThreadItemType", + "type": "string" + } + }, + "required": [ + "arguments", + "callId", + "id", + "status", + "tool", + "type" + ], + "title": "BuiltinToolCallThreadItem", + "type": "object" + }, { "properties": { "arguments": true, diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BuiltinToolCallStatus.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BuiltinToolCallStatus.ts new file mode 100644 index 000000000000..2a0b3dd4f13c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BuiltinToolCallStatus.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BuiltinToolCallStatus = "inProgress" | "completed" | "failed"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index 9202f3728f05..dd04db5b22f7 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -4,6 +4,7 @@ import type { MessagePhase } from "../MessagePhase"; import type { ReasoningEffort } from "../ReasoningEffort"; import type { JsonValue } from "../serde_json/JsonValue"; +import type { BuiltinToolCallStatus } from "./BuiltinToolCallStatus"; import type { CollabAgentState } from "./CollabAgentState"; import type { CollabAgentTool } from "./CollabAgentTool"; import type { CollabAgentToolCallStatus } from "./CollabAgentToolCallStatus"; @@ -56,7 +57,7 @@ durationMs: number | null, } | { "type": "fileChange", id: string, changes: Arra /** * The duration of the MCP tool call in milliseconds. */ -durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array | null, success: boolean | null, +durationMs: number | null, } | { "type": "builtinToolCall", id: string, callId: string, tool: string, namespace?: string, arguments: JsonValue, output?: JsonValue, success?: boolean, status: BuiltinToolCallStatus, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array | null, success: boolean | null, /** * The duration of the dynamic tool call in milliseconds. */ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadResumeParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadResumeParams.ts index 770344de8ede..e59d62537592 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadResumeParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadResumeParams.ts @@ -37,6 +37,10 @@ model?: string | null, modelProvider?: string | null, serviceTier?: ServiceTier * and subsequent turns. */ approvalsReviewer?: ApprovalsReviewer | null, sandbox?: SandboxMode | null, config?: { [key in string]?: JsonValue } | null, baseInstructions?: string | null, developerInstructions?: string | null, personality?: Personality | null, /** + * If true, opt into emitting raw Responses API items on the event stream + * after resuming this thread. + */ +experimentalRawEvents: boolean, /** * If true, persist additional rollout EventMsg variants required to * reconstruct a richer thread history on subsequent resume/fork/read. */ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 495572fd7831..49c73e0f3d12 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -24,6 +24,7 @@ export type { AppsDefaultConfig } from "./AppsDefaultConfig"; export type { AppsListParams } from "./AppsListParams"; export type { AppsListResponse } from "./AppsListResponse"; export type { AskForApproval } from "./AskForApproval"; +export type { BuiltinToolCallStatus } from "./BuiltinToolCallStatus"; export type { ByteRange } from "./ByteRange"; export type { CancelLoginAccountParams } from "./CancelLoginAccountParams"; export type { CancelLoginAccountResponse } from "./CancelLoginAccountResponse"; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 48fa56d687c8..fe981ddb74bf 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -325,6 +325,12 @@ impl ThreadHistoryBuilder { ThreadItem::from(payload.item.clone()), ); } + codex_protocol::items::TurnItem::BuiltinToolCall(_) => { + self.upsert_item_in_turn_id( + &payload.turn_id, + ThreadItem::from(payload.item.clone()), + ); + } codex_protocol::items::TurnItem::UserMessage(_) | codex_protocol::items::TurnItem::HookPrompt(_) | codex_protocol::items::TurnItem::AgentMessage(_) @@ -346,6 +352,12 @@ impl ThreadHistoryBuilder { ThreadItem::from(payload.item.clone()), ); } + codex_protocol::items::TurnItem::BuiltinToolCall(_) => { + self.upsert_item_in_turn_id( + &payload.turn_id, + ThreadItem::from(payload.item.clone()), + ); + } codex_protocol::items::TurnItem::UserMessage(_) | codex_protocol::items::TurnItem::HookPrompt(_) | codex_protocol::items::TurnItem::AgentMessage(_) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index a6a4aa5403ee..80614b83e3b3 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -27,6 +27,7 @@ use codex_protocol::config_types::Verbosity; use codex_protocol::config_types::WebSearchMode; use codex_protocol::config_types::WebSearchToolConfig; use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; +use codex_protocol::items::BuiltinToolCallStatus as CoreBuiltinToolCallStatus; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::mcp::Resource as McpResource; use codex_protocol::mcp::ResourceTemplate as McpResourceTemplate; @@ -2695,6 +2696,11 @@ pub struct ThreadResumeParams { pub developer_instructions: Option, #[ts(optional = nullable)] pub personality: Option, + /// If true, opt into emitting raw Responses API items on the event stream + /// after resuming this thread. + #[experimental("thread/resume.experimentalRawEvents")] + #[serde(default)] + pub experimental_raw_events: bool, /// If true, persist additional rollout EventMsg variants required to /// reconstruct a richer thread history on subsequent resume/fork/read. #[experimental("thread/resume.persistFullHistory")] @@ -4311,6 +4317,24 @@ pub enum ThreadItem { }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] + BuiltinToolCall { + id: String, + call_id: String, + tool: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + namespace: Option, + arguments: JsonValue, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + success: Option, + status: BuiltinToolCallStatus, + }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] DynamicToolCall { id: String, tool: String, @@ -4396,6 +4420,7 @@ impl ThreadItem { | ThreadItem::CommandExecution { id, .. } | ThreadItem::FileChange { id, .. } | ThreadItem::McpToolCall { id, .. } + | ThreadItem::BuiltinToolCall { id, .. } | ThreadItem::DynamicToolCall { id, .. } | ThreadItem::CollabAgentToolCall { id, .. } | ThreadItem::WebSearch { id, .. } @@ -4754,6 +4779,16 @@ impl From for ThreadItem { summary: reasoning.summary_text, content: reasoning.raw_content, }, + CoreTurnItem::BuiltinToolCall(item) => ThreadItem::BuiltinToolCall { + id: item.id, + call_id: item.call_id, + tool: item.tool, + namespace: item.namespace, + arguments: item.arguments, + output: item.output, + success: item.success, + status: BuiltinToolCallStatus::from(item.status), + }, CoreTurnItem::WebSearch(search) => ThreadItem::WebSearch { id: search.id, query: search.query, @@ -4884,6 +4919,25 @@ pub enum McpToolCallStatus { Failed, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum BuiltinToolCallStatus { + InProgress, + Completed, + Failed, +} + +impl From for BuiltinToolCallStatus { + fn from(value: CoreBuiltinToolCallStatus) -> Self { + match value { + CoreBuiltinToolCallStatus::InProgress => Self::InProgress, + CoreBuiltinToolCallStatus::Completed => Self::Completed, + CoreBuiltinToolCallStatus::Failed => Self::Failed, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -7972,6 +8026,32 @@ mod tests { } ); + let builtin_tool_call = + TurnItem::BuiltinToolCall(codex_protocol::items::BuiltinToolCallItem { + id: "builtin-1".to_string(), + call_id: "builtin-1".to_string(), + tool: "gui_click".to_string(), + namespace: Some("builtin".to_string()), + arguments: serde_json::json!({"target": "Search"}), + output: Some(serde_json::json!("{\"status\":\"clicked\"}")), + success: Some(true), + status: codex_protocol::items::BuiltinToolCallStatus::Completed, + }); + + assert_eq!( + ThreadItem::from(builtin_tool_call), + ThreadItem::BuiltinToolCall { + id: "builtin-1".to_string(), + call_id: "builtin-1".to_string(), + tool: "gui_click".to_string(), + namespace: Some("builtin".to_string()), + arguments: serde_json::json!({"target": "Search"}), + output: Some(serde_json::json!("{\"status\":\"clicked\"}")), + success: Some(true), + status: BuiltinToolCallStatus::Completed, + } + ); + let search_item = TurnItem::WebSearch(WebSearchItem { id: "search-1".to_string(), query: "docs".to_string(), diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 2b9aa12ab420..bf73d64abcce 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -3666,6 +3666,7 @@ impl CodexMessageProcessor { base_instructions, developer_instructions, personality, + experimental_raw_events: _experimental_raw_events, persist_extended_history, } = params; @@ -3764,7 +3765,7 @@ impl CodexMessageProcessor { self.ensure_conversation_listener( thread_id, request_id.connection_id, - /*raw_events_enabled*/ false, + /*experimental_raw_events*/ false, ApiVersion::V2, ) .await, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index a8f3229b5415..673e739f0fd6 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -398,6 +398,9 @@ "guardian_approval": { "type": "boolean" }, + "gui_tools": { + "type": "boolean" + }, "image_detail_original": { "type": "boolean" }, @@ -700,6 +703,17 @@ ], "type": "object" }, + "GuiToolsToml": { + "additionalProperties": false, + "properties": { + "coordinate_targeting": { + "default": null, + "description": "When `true`, `gui_click` and `gui_drag` may accept direct coordinates so the main model can act from a previously observed screenshot instead of delegating target resolution to the internal grounding round-trip.", + "type": "boolean" + } + }, + "type": "object" + }, "History": { "additionalProperties": false, "description": "Settings that govern if and what will be written to `~/.codex/history.jsonl`.", @@ -1684,6 +1698,15 @@ "ToolsToml": { "additionalProperties": false, "properties": { + "gui": { + "allOf": [ + { + "$ref": "#/definitions/GuiToolsToml" + } + ], + "default": null, + "description": "GUI tool behavior settings." + }, "view_image": { "default": null, "description": "Enable the `view_image` tool that lets the agent attach local images.", @@ -2096,6 +2119,9 @@ "guardian_approval": { "type": "boolean" }, + "gui_tools": { + "type": "boolean" + }, "image_detail_original": { "type": "boolean" }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 92d75390d422..ee520231597d 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -23,6 +23,7 @@ use crate::compact_remote::run_inline_remote_auto_compact_task; use crate::config::ManagedFeatures; use crate::connectors; use crate::exec_policy::ExecPolicyManager; +use crate::gui_instructions::render_gui_tools_section; #[cfg(test)] use crate::models_manager::collaboration_mode_presets::CollaborationModesConfig; use crate::models_manager::manager::ModelsManager; @@ -44,6 +45,7 @@ use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; +use crate::stream_events_utils::response_input_to_response_item; use crate::turn_metadata::TurnMetadataState; use crate::util::error_or_panic; use async_channel::Receiver; @@ -92,12 +94,15 @@ use codex_protocol::config_types::Settings; use codex_protocol::config_types::WebSearchMode; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; +use codex_protocol::items::BuiltinToolCallItem; +use codex_protocol::items::BuiltinToolCallStatus; use codex_protocol::items::PlanItem; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::build_hook_prompt_message; use codex_protocol::mcp::CallToolResult; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::PermissionProfile; use codex_protocol::models::format_allow_prefixes; use codex_protocol::openai_models::ModelInfo; @@ -196,6 +201,25 @@ mod rollout_reconstruction; #[cfg(test)] mod rollout_reconstruction_tests; +fn is_visible_builtin_tool_call(tool_name: &str) -> bool { + tool_name.starts_with("gui_") +} + +fn parse_builtin_tool_call_arguments(arguments: &str) -> Value { + serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.to_string())) +} + +fn parse_builtin_tool_call_payload_arguments(payload: &ToolPayload) -> Option { + match payload { + ToolPayload::Function { arguments } => Some(parse_builtin_tool_call_arguments(arguments)), + _ => None, + } +} + +fn serialize_builtin_tool_output(output: &FunctionCallOutputPayload) -> Value { + serde_json::to_value(&output.body).unwrap_or(Value::Null) +} + #[derive(Debug, PartialEq)] pub enum SteerInputError { NoActiveTurn(Vec), @@ -301,12 +325,14 @@ use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; +use crate::tools::context::ToolPayload; use crate::tools::js_repl::JsReplHandle; use crate::tools::js_repl::resolve_compatible_node; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::network_approval::build_blocked_request_observer; use crate::tools::network_approval::build_network_policy_decider; use crate::tools::parallel::ToolCallRuntime; +use crate::tools::router::ToolCall; use crate::tools::router::ToolRouterParams; use crate::tools::sandboxing::ApprovalStore; use crate::turn_diff_tracker::TurnDiffTracker; @@ -945,6 +971,10 @@ impl TurnContext { }) .with_unified_exec_shell_mode(self.tools_config.unified_exec_shell_mode.clone()) .with_web_search_config(self.tools_config.web_search_config.clone()) + .with_gui_coordinate_targeting(config.gui_coordinate_targeting) + .with_gui_batch_enabled(config.gui_batch_enabled) + .with_gui_batch_grounding_strategy(config.gui_batch_grounding_strategy.clone()) + .with_gui_batch_action_delay_ms(config.gui_batch_action_delay_ms) .with_allow_login_shell(self.tools_config.allow_login_shell) .with_agent_type_description(crate::agent::role::spawn_tool_spec::build( &config.agent_roles, @@ -1405,6 +1435,7 @@ impl Session { main_execve_wrapper_exe, ) .with_web_search_config(per_turn_config.web_search_config.clone()) + .with_gui_coordinate_targeting(per_turn_config.gui_coordinate_targeting) .with_allow_login_shell(per_turn_config.permissions.allow_login_shell) .with_agent_type_description(crate::agent::role::spawn_tool_spec::build( &per_turn_config.agent_roles, @@ -3665,6 +3696,13 @@ impl Session { { developer_sections.push(plugin_section); } + if let Some(gui_tools_section) = render_gui_tools_section( + turn_context.features.enabled(Feature::GuiTools), + turn_context.config.gui_coordinate_targeting, + turn_context.config.gui_batch_enabled, + ) { + developer_sections.push(gui_tools_section); + } if turn_context.features.enabled(Feature::CodexGitCommit) && let Some(commit_message_instruction) = commit_message_trailer_instruction( turn_context.config.commit_attribution.as_deref(), @@ -3895,6 +3933,24 @@ impl Session { self.record_conversation_items(turn_context, std::slice::from_ref(&response_item)) .await; + if let Some(item) = self + .visible_builtin_tool_call_started_item(&response_item) + .await + { + self.emit_turn_item_started(turn_context, &TurnItem::BuiltinToolCall(item)) + .await; + return; + } + + if let Some(item) = self + .visible_builtin_tool_call_completed_item(&response_item) + .await + { + self.emit_turn_item_completed(turn_context, TurnItem::BuiltinToolCall(item)) + .await; + return; + } + // Derive a turn item and emit lifecycle events if applicable. if let Some(item) = parse_turn_item(&response_item) { self.emit_turn_item_started(turn_context, &item).await; @@ -3902,6 +3958,145 @@ impl Session { } } + pub(crate) async fn maybe_emit_visible_builtin_tool_call_started( + &self, + turn_context: &TurnContext, + call: &ToolCall, + ) { + let Some(item) = self + .visible_builtin_tool_call_started_item_from_call(call) + .await + else { + return; + }; + + self.emit_turn_item_started(turn_context, &TurnItem::BuiltinToolCall(item)) + .await; + } + + pub(crate) async fn maybe_emit_visible_builtin_tool_call_completed( + &self, + turn_context: &TurnContext, + response_item: &ResponseItem, + ) { + let Some(item) = self + .visible_builtin_tool_call_completed_item(response_item) + .await + else { + return; + }; + + self.emit_turn_item_completed(turn_context, TurnItem::BuiltinToolCall(item)) + .await; + } + + async fn visible_builtin_tool_call_started_item_from_call( + &self, + call: &ToolCall, + ) -> Option { + if !is_visible_builtin_tool_call(&call.tool_name) { + return None; + } + + let arguments = parse_builtin_tool_call_payload_arguments(&call.payload)?; + let item = BuiltinToolCallItem { + id: call.call_id.clone(), + call_id: call.call_id.clone(), + tool: call.tool_name.clone(), + namespace: call.tool_namespace.clone(), + arguments, + output: None, + success: None, + status: BuiltinToolCallStatus::InProgress, + }; + + self.state + .lock() + .await + .record_visible_builtin_tool_call(item.clone()); + + Some(item) + } + + async fn visible_builtin_tool_call_started_item( + &self, + response_item: &ResponseItem, + ) -> Option { + let ResponseItem::FunctionCall { + name, + namespace, + arguments, + call_id, + .. + } = response_item + else { + return None; + }; + + if !is_visible_builtin_tool_call(name) { + return None; + } + + let item = BuiltinToolCallItem { + id: call_id.clone(), + call_id: call_id.clone(), + tool: name.clone(), + namespace: namespace.clone(), + arguments: parse_builtin_tool_call_arguments(arguments), + output: None, + success: None, + status: BuiltinToolCallStatus::InProgress, + }; + + self.state + .lock() + .await + .record_visible_builtin_tool_call(item.clone()); + + Some(item) + } + + async fn visible_builtin_tool_call_completed_item( + &self, + response_item: &ResponseItem, + ) -> Option { + match response_item { + ResponseItem::FunctionCallOutput { call_id, output } => { + let mut item = self + .state + .lock() + .await + .take_visible_builtin_tool_call(call_id)?; + item.output = Some(serialize_builtin_tool_output(output)); + item.success = output.success; + item.status = if output.success == Some(false) { + BuiltinToolCallStatus::Failed + } else { + BuiltinToolCallStatus::Completed + }; + Some(item) + } + ResponseItem::CustomToolCallOutput { + call_id, output, .. + } => { + let mut item = self + .state + .lock() + .await + .take_visible_builtin_tool_call(call_id)?; + item.output = Some(serialize_builtin_tool_output(output)); + item.success = output.success; + item.status = if output.success == Some(false) { + BuiltinToolCallStatus::Failed + } else { + BuiltinToolCallStatus::Completed + }; + Some(item) + } + _ => None, + } + } + pub(crate) async fn record_user_prompt_and_emit_turn_item( &self, turn_context: &TurnContext, @@ -5477,6 +5672,10 @@ async fn spawn_review_thread( sess.services.main_execve_wrapper_exe.as_ref(), ) .with_web_search_config(/*web_search_config*/ None) + .with_gui_coordinate_targeting(config.gui_coordinate_targeting) + .with_gui_batch_enabled(config.gui_batch_enabled) + .with_gui_batch_grounding_strategy(config.gui_batch_grounding_strategy.clone()) + .with_gui_batch_action_delay_ms(config.gui_batch_action_delay_ms) .with_allow_login_shell(config.permissions.allow_login_shell) .with_agent_type_description(crate::agent::role::spawn_tool_spec::build( &config.agent_roles, @@ -7231,8 +7430,18 @@ async fn drain_in_flight( while let Some(res) = in_flight.next().await { match res { Ok(response_input) => { - sess.record_conversation_items(&turn_context, &[response_input.into()]) + if let Some(response_item) = response_input_to_response_item(&response_input) { + sess.record_conversation_items( + &turn_context, + std::slice::from_ref(&response_item), + ) .await; + sess.maybe_emit_visible_builtin_tool_call_completed( + &turn_context, + &response_item, + ) + .await; + } } Err(err) => { error_or_panic(format!("in-flight tool future failed during drain: {err}")); diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 8f1bba647c45..7b7d7c8de304 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -46,6 +46,7 @@ use crate::tools::context::ToolPayload; use crate::tools::handlers::ShellHandler; use crate::tools::handlers::UnifiedExecHandler; use crate::tools::registry::ToolHandler; +use crate::tools::router::ToolCall; use crate::tools::router::ToolCallSource; use crate::turn_diff_tracker::TurnDiffTracker; use codex_app_server_protocol::AppInfo; @@ -3962,6 +3963,25 @@ async fn build_initial_context_omits_default_image_save_location_without_image_h ); } +#[tokio::test] +async fn build_initial_context_includes_gui_workflow_instructions_when_feature_enabled() { + let (session, mut turn_context) = make_session_and_context().await; + turn_context + .features + .enable(Feature::GuiTools) + .expect("enable gui tools"); + + let initial_context = session.build_initial_context(&turn_context).await; + let developer_texts = developer_input_texts(&initial_context); + + assert!( + developer_texts + .iter() + .any(|text| text.contains("## Native GUI Tools") && text.contains("`gui_wait`")), + "expected initial context to include gui workflow instructions, got {developer_texts:?}" + ); +} + #[tokio::test] async fn handle_output_item_done_records_image_save_history_message() { let (session, turn_context) = make_session_and_context().await; @@ -4579,6 +4599,71 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() )); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn visible_gui_builtin_tool_calls_emit_turn_item_lifecycle_events() { + let (sess, tc, rx) = make_session_and_context_with_rx().await; + let call = ToolCall { + tool_name: "gui_observe".to_string(), + tool_namespace: None, + call_id: "call_gui_visible".to_string(), + payload: ToolPayload::Function { + arguments: "{\"return_image\":false}".to_string(), + }, + }; + + sess.maybe_emit_visible_builtin_tool_call_started(tc.as_ref(), &call) + .await; + + let output_item = ResponseItem::FunctionCallOutput { + call_id: call.call_id.clone(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }; + sess.record_conversation_items(tc.as_ref(), std::slice::from_ref(&output_item)) + .await; + sess.maybe_emit_visible_builtin_tool_call_completed(tc.as_ref(), &output_item) + .await; + + let first = tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("expected builtin item started event") + .expect("channel open"); + assert!(matches!( + first.msg, + EventMsg::ItemStarted(ItemStartedEvent { + item: TurnItem::BuiltinToolCall(codex_protocol::items::BuiltinToolCallItem { + call_id, + tool, + status: codex_protocol::items::BuiltinToolCallStatus::InProgress, + .. + }), + .. + }) if call_id == "call_gui_visible" && tool == "gui_observe" + )); + + let second = tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("expected raw response item event") + .expect("channel open"); + assert!(matches!(second.msg, EventMsg::RawResponseItem(_))); + + let third = tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("expected builtin item completed event") + .expect("channel open"); + assert!(matches!( + third.msg, + EventMsg::ItemCompleted(ItemCompletedEvent { + item: TurnItem::BuiltinToolCall(codex_protocol::items::BuiltinToolCallItem { + call_id, + tool, + status: codex_protocol::items::BuiltinToolCallStatus::Completed, + .. + }), + .. + }) if call_id == "call_gui_visible" && tool == "gui_observe" + )); +} + #[tokio::test] async fn steer_input_requires_active_turn() { let (sess, _tc, _rx) = make_session_and_context_with_rx().await; diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 02f9582561b2..a8e9fb69f064 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -187,6 +187,26 @@ consolidation_model = "gpt-5" ); } +#[test] +fn gui_coordinate_targeting_reads_from_tools_config() -> std::io::Result<()> { + let cfg: ConfigToml = toml::from_str( + r#" +[tools.gui] +coordinate_targeting = true +"#, + ) + .expect("TOML deserialization should succeed"); + + let config = Config::load_from_base_config_with_overrides( + cfg, + ConfigOverrides::default(), + tempdir()?.keep(), + )?; + + assert!(config.gui_coordinate_targeting); + Ok(()) +} + #[test] fn parses_bundled_skills_config() { let cfg: ConfigToml = toml::from_str( @@ -221,6 +241,7 @@ web_search = true Some(ToolsToml { web_search: None, view_image: None, + gui: None, }) ); } @@ -240,6 +261,7 @@ web_search = false Some(ToolsToml { web_search: None, view_image: None, + gui: None, }) ); } @@ -4519,6 +4541,10 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { model_availability_nux: ModelAvailabilityNuxConfig::default(), analytics_enabled: Some(true), feedback_enabled: true, + gui_coordinate_targeting: false, + gui_batch_enabled: true, + gui_batch_grounding_strategy: "parallel".to_string(), + gui_batch_action_delay_ms: 0, tool_suggest: ToolSuggestConfig::default(), tui_alternate_screen: AltScreenMode::Auto, tui_status_line: None, @@ -4661,6 +4687,10 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { model_availability_nux: ModelAvailabilityNuxConfig::default(), analytics_enabled: Some(true), feedback_enabled: true, + gui_coordinate_targeting: false, + gui_batch_enabled: true, + gui_batch_grounding_strategy: "parallel".to_string(), + gui_batch_action_delay_ms: 0, tool_suggest: ToolSuggestConfig::default(), tui_alternate_screen: AltScreenMode::Auto, tui_status_line: None, @@ -4801,6 +4831,10 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { model_availability_nux: ModelAvailabilityNuxConfig::default(), analytics_enabled: Some(false), feedback_enabled: true, + gui_coordinate_targeting: false, + gui_batch_enabled: true, + gui_batch_grounding_strategy: "parallel".to_string(), + gui_batch_action_delay_ms: 0, tool_suggest: ToolSuggestConfig::default(), tui_alternate_screen: AltScreenMode::Auto, tui_status_line: None, @@ -4927,6 +4961,10 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { model_availability_nux: ModelAvailabilityNuxConfig::default(), analytics_enabled: Some(true), feedback_enabled: true, + gui_coordinate_targeting: false, + gui_batch_enabled: true, + gui_batch_grounding_strategy: "parallel".to_string(), + gui_batch_action_delay_ms: 0, tool_suggest: ToolSuggestConfig::default(), tui_alternate_screen: AltScreenMode::Auto, tui_status_line: None, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index f22258dfb9eb..e6ece40ad562 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -228,6 +228,20 @@ pub struct Config { /// Effective service tier preference for new turns (`fast` or `flex`). pub service_tier: Option, + /// When enabled, GUI tools may expose direct coordinate arguments so the + /// main model can act from the latest observed screenshot without using a + /// separate grounding call. + pub gui_coordinate_targeting: bool, + + /// Whether the `gui_batch` tool is enabled (registered in the tool plan). + pub gui_batch_enabled: bool, + + /// Default grounding strategy for `gui_batch`: `"parallel"` or `"unified"`. + pub gui_batch_grounding_strategy: String, + + /// Delay in milliseconds between batch action steps. Defaults to 0 (no delay). + pub gui_batch_action_delay_ms: u64, + /// Model used specifically for review sessions. pub review_model: Option, @@ -1519,6 +1533,31 @@ pub struct ToolsToml { /// Enable the `view_image` tool that lets the agent attach local images. #[serde(default)] pub view_image: Option, + + /// GUI tool behavior settings. + #[serde(default)] + pub gui: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct GuiToolsToml { + /// When `true`, `gui_click` and `gui_drag` may accept direct coordinates so + /// the main model can act from a previously observed screenshot instead of + /// delegating target resolution to the internal grounding round-trip. + #[serde(default)] + pub coordinate_targeting: Option, + /// Whether the `gui_batch` tool is enabled. Defaults to `true`. + #[serde(default)] + pub batch: Option, + /// Grounding strategy for `gui_batch`: `"parallel"` (default) runs N + /// independent grounding calls in parallel; `"unified"` sends all targets + /// in a single multi-target call with validation rounds. + #[serde(default)] + pub batch_grounding_strategy: Option, + /// Delay in milliseconds between batch action steps. Defaults to 0 (no delay). + #[serde(default)] + pub batch_action_delay_ms: Option, } #[derive(Deserialize)] @@ -1961,6 +2000,42 @@ fn resolve_web_search_config( } } +fn resolve_gui_coordinate_targeting(config_toml: &ConfigToml) -> bool { + config_toml + .tools + .as_ref() + .and_then(|tools| tools.gui.as_ref()) + .and_then(|gui| gui.coordinate_targeting) + .unwrap_or(false) +} + +fn resolve_gui_batch_enabled(config_toml: &ConfigToml) -> bool { + config_toml + .tools + .as_ref() + .and_then(|tools| tools.gui.as_ref()) + .and_then(|gui| gui.batch) + .unwrap_or(true) +} + +fn resolve_gui_batch_grounding_strategy(config_toml: &ConfigToml) -> String { + config_toml + .tools + .as_ref() + .and_then(|tools| tools.gui.as_ref()) + .and_then(|gui| gui.batch_grounding_strategy.clone()) + .unwrap_or_else(|| "parallel".to_string()) +} + +fn resolve_gui_batch_action_delay_ms(config_toml: &ConfigToml) -> u64 { + config_toml + .tools + .as_ref() + .and_then(|tools| tools.gui.as_ref()) + .and_then(|gui| gui.batch_action_delay_ms) + .unwrap_or(0) +} + pub(crate) fn resolve_web_search_mode_for_turn( web_search_mode: &Constrained, sandbox_policy: &SandboxPolicy, @@ -2276,6 +2351,10 @@ impl Config { let web_search_mode = resolve_web_search_mode(&cfg, &config_profile, &features) .unwrap_or(WebSearchMode::Cached); let web_search_config = resolve_web_search_config(&cfg, &config_profile); + let gui_coordinate_targeting = resolve_gui_coordinate_targeting(&cfg); + let gui_batch_enabled = resolve_gui_batch_enabled(&cfg); + let gui_batch_grounding_strategy = resolve_gui_batch_grounding_strategy(&cfg); + let gui_batch_action_delay_ms = resolve_gui_batch_action_delay_ms(&cfg); let agent_roles = agent_roles::load_agent_roles(&cfg, &config_layer_stack, &mut startup_warnings)?; @@ -2586,6 +2665,10 @@ impl Config { let config = Self { model, service_tier, + gui_coordinate_targeting, + gui_batch_enabled, + gui_batch_grounding_strategy, + gui_batch_action_delay_ms, review_model, model_context_window: cfg.model_context_window, model_auto_compact_token_limit: cfg.model_auto_compact_token_limit, diff --git a/codex-rs/core/src/gui_instructions.rs b/codex-rs/core/src/gui_instructions.rs new file mode 100644 index 000000000000..0502fb5c4d0b --- /dev/null +++ b/codex-rs/core/src/gui_instructions.rs @@ -0,0 +1,87 @@ +pub(crate) fn render_gui_tools_section( + gui_tools_enabled: bool, + gui_coordinate_targeting: bool, + gui_batch_enabled: bool, +) -> Option { + if !gui_tools_enabled { + return None; + } + + let coordinate_guidance = if gui_coordinate_targeting { + "\n- Coordinate-based `gui_click` and `gui_drag` may appear when `[tools.gui] coordinate_targeting = true`, but that path is currently an experimental placeholder and is disabled by default.\n- Prefer semantic grounding for real work; the direct-coordinate path is reserved for future experiments after its benchmark quality improves." + } else { + "" + }; + + let batch_guidance = if gui_batch_enabled { + "\n- Use `gui_batch` to execute multiple independent GUI actions (click, type, key, scroll, drag) in a single tool call for faster task completion. All steps share one screenshot and one grounding call. Only include steps that do NOT depend on the visual effects of earlier steps in the same batch. For dependent actions, use individual gui_* tools with `gui_wait` or `gui_observe` between them. Prefer `gui_batch` when you have 2 or more consecutive independent actions." + } else { + "" + }; + + Some(format!( + "\ +## Native GUI Tools +When the experimental `gui_tools` feature is enabled, the built-in macOS GUI tools are available as normal native functions at the same level as other built-in tools. +Use them as a tight observe-act-verify loop: +- Start with `gui_observe` to inspect the current GUI surface and optionally ground a semantic control without acting yet. +- Prefer semantic targeting first when it is clear: `gui_observe.target`, `gui_click.target`, `gui_drag.from_target` plus `gui_drag.to_target`, `gui_type.target`, `gui_scroll.target`, and `gui_wait.target` resolve visible GUI controls by label or meaning, optionally refined with `location_hint`, `scope`, and `grounding_mode`. +- Write GUI `target` descriptions using visible screenshot evidence: prefer the exact on-screen text, icon, state, nearby context, and coarse location that make the control unique. +- Name the actionable or editable surface itself in `target`, not surrounding whitespace, generic container chrome, or descriptive text next to the real control. +- When several similar controls are visible, add nearby text, state, relative position, `scope`, or `window_title` so the runtime can disambiguate them. +- `gui_key.key` must be a plain literal key name like `Enter`, `Escape`, or a single printable character. Do not wrap the key value in markdown backticks or extra punctuation. +- After a state-changing GUI action, prefer `gui_wait` or a fresh `gui_observe` before the next action so you verify what changed instead of assuming success. +- If a GUI action misses, ground again from a fresh screenshot instead of reusing stale assumptions about the old surface. +- After one failed attempt on the same visible target, revise the `target` and `scope` using the latest screenshot evidence and prefer `grounding_mode: \"complex\"` for the retry. +- Reuse `capture_mode`, `window_title`, or `window_selector` across related GUI steps to keep the tool focused on the same surface. +- Use `gui_type.secret_env_var` or `gui_type.secret_command_env_var` for sensitive values. +- Prefer `capture_mode: \"window\"` for in-app work and `capture_mode: \"display\"` for desktop-wide UI such as the Dock, menu bar, permission prompts, or cross-window drags. +- Remember that GUI actions now run with native safety guards: avoid overlapping risky actions, and stop to re-observe when the UI looks different than expected.{batch_guidance}{coordinate_guidance}" + )) +} + +#[cfg(test)] +mod tests { + use super::render_gui_tools_section; + + #[test] + fn omits_gui_tools_section_when_disabled() { + assert_eq!(render_gui_tools_section(false, false, false), None); + } + + #[test] + fn renders_gui_tools_section_when_enabled() { + let rendered = render_gui_tools_section(true, false, false).expect("gui instructions"); + + assert!(rendered.contains("## Native GUI Tools")); + assert!(rendered.contains("`gui_observe.target`")); + assert!(rendered.contains("`gui_wait`")); + assert!(rendered.contains("`gui_drag.from_target`")); + assert!(rendered.contains("`scope`")); + assert!(rendered.contains("observe-act-verify")); + assert!(rendered.contains("semantic targeting")); + assert!(rendered.contains("visible screenshot evidence")); + assert!(rendered.contains("grounding_mode: \"complex\"")); + assert!(!rendered.contains("Coordinate-based `gui_click`")); + assert!(!rendered.contains("`gui_batch`")); + } + + #[test] + fn renders_coordinate_guidance_when_enabled() { + let rendered = render_gui_tools_section(true, true, false).expect("gui instructions"); + + assert!(rendered.contains("Coordinate-based `gui_click` and `gui_drag`")); + assert!(rendered.contains("experimental placeholder")); + assert!(rendered.contains("disabled by default")); + assert!(rendered.contains("Prefer semantic grounding")); + } + + #[test] + fn renders_batch_guidance_when_enabled() { + let rendered = render_gui_tools_section(true, false, true).expect("gui instructions"); + + assert!(rendered.contains("`gui_batch`")); + assert!(rendered.contains("independent GUI actions")); + assert!(!rendered.contains("grounding_strategy")); + } +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 8ffc7acfb781..b1dbea7477f5 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -40,6 +40,7 @@ mod flags; #[cfg(test)] mod git_info_tests; mod guardian; +mod gui_instructions; mod hook_runtime; pub mod instructions; pub mod landlock; diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 206f75060c70..6b8bcf16db07 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -1,5 +1,6 @@ //! Session-wide mutable state. +use codex_protocol::items::BuiltinToolCallItem; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; use codex_sandboxing::policy_transforms::merge_permission_profiles; @@ -33,6 +34,7 @@ pub(crate) struct SessionState { pub(crate) active_connector_selection: HashSet, pub(crate) pending_session_start_source: Option, granted_permissions: Option, + visible_builtin_tool_calls: HashMap, } impl SessionState { @@ -51,6 +53,7 @@ impl SessionState { active_connector_selection: HashSet::new(), pending_session_start_source: None, granted_permissions: None, + visible_builtin_tool_calls: HashMap::new(), } } @@ -214,6 +217,21 @@ impl SessionState { pub(crate) fn granted_permissions(&self) -> Option { self.granted_permissions.clone() } + + pub(crate) fn record_visible_builtin_tool_call( + &mut self, + item: BuiltinToolCallItem, + ) -> Option { + self.visible_builtin_tool_calls + .insert(item.call_id.clone(), item) + } + + pub(crate) fn take_visible_builtin_tool_call( + &mut self, + call_id: &str, + ) -> Option { + self.visible_builtin_tool_calls.remove(call_id) + } } // Sometimes new snapshots don't include credits or plan information. diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index 5cc9574db3df..1b8e0afb7685 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -215,6 +215,9 @@ pub(crate) async fn handle_output_item_done( record_completed_response_item(ctx.sess.as_ref(), ctx.turn_context.as_ref(), &item) .await; + ctx.sess + .maybe_emit_visible_builtin_tool_call_started(ctx.turn_context.as_ref(), &call) + .await; let cancellation_token = ctx.cancellation_token.child_token(); let tool_future: InFlightFuture<'static> = Box::pin( @@ -461,7 +464,13 @@ pub(crate) fn response_input_to_response_item(input: &ResponseInputItem) -> Opti execution: execution.clone(), tools: tools.clone(), }), - _ => None, + other => { + tracing::debug!( + "response_input_to_response_item: unhandled variant {:?}", + std::mem::discriminant(other) + ); + None + } } } diff --git a/codex-rs/core/src/tools/handlers/gui.rs b/codex-rs/core/src/tools/handlers/gui.rs new file mode 100644 index 000000000000..7faf14a32a23 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui.rs @@ -0,0 +1,3811 @@ +use async_trait::async_trait; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ResponseInputItem; +use codex_protocol::openai_models::InputModality; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value as JsonValue; +use std::collections::HashMap; +#[cfg(test)] +use std::path::PathBuf; +use std::process::Command; +use tokio::sync::Mutex; +use tokio::time::Duration; +use tokio::time::Instant; +use tokio::time::sleep; +use tokio::time::timeout; + +use crate::function_tool::FunctionCallError; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolOutput; +use crate::tools::context::ToolPayload; +use crate::tools::handlers::parse_arguments; +use crate::tools::registry::ToolHandler; +use crate::tools::registry::ToolKind; + +const SCREENSHOT_JPEG_QUALITY: u8 = 75; + +#[path = "gui/grounding.rs"] +mod grounding; +#[path = "gui/platform.rs"] +mod platform; +#[path = "gui/provider.rs"] +mod provider; +#[path = "gui/readiness.rs"] +mod readiness; +#[path = "gui/session.rs"] +mod session; +#[cfg(test)] +#[path = "gui/tests/mod.rs"] +mod tests; + +use platform::PlatformObservation; +use platform::default_gui_platform; +use provider::GuiGroundingProvider; +use provider::default_gui_grounding_provider; +use readiness::enforce_gui_tool_capability; + +const PLATFORM_NAME: &str = if cfg!(target_os = "macos") { + "macOS" +} else if cfg!(target_os = "windows") { + "Windows" +} else { + "this platform" +}; +const GUI_UNSUPPORTED_MESSAGE: &str = "Native GUI tools are not supported on this platform yet."; +const GUI_IMAGE_UNSUPPORTED_MESSAGE: &str = + "Native GUI screenshot tools are not allowed because you do not support image inputs"; +const DEFAULT_DRAG_DURATION_MS: i64 = 450; +const DEFAULT_DRAG_STEPS: i64 = 24; +const DEFAULT_HOVER_SETTLE_MS: i64 = 200; +const DEFAULT_CLICK_AND_HOLD_MS: i64 = 650; +const DEFAULT_GUI_WAIT_TIMEOUT_MS: i64 = 8000; +const DEFAULT_GUI_WAIT_INTERVAL_MS: i64 = 350; +const WAIT_CONFIRMATION_COUNT: i64 = 2; +const DEFAULT_POST_ACTION_SETTLE_MS: i64 = 3000; +const DEFAULT_POST_TYPE_SETTLE_MS: i64 = 500; +const DEFAULT_TYPE_FOCUS_SETTLE_MS: i64 = 180; +const DEFAULT_TARGETED_SCROLL_DISTANCE: &str = "medium"; +const DEFAULT_TARGETLESS_SCROLL_DISTANCE: &str = "page"; +const GUI_DIRECT_COORDINATE_PLACEHOLDER_MESSAGE: &str = "Direct coordinate GUI targeting is currently kept as an experimental placeholder only. Semantic grounding remains the supported path because the direct-coordinate benchmark accuracy is still poor."; +const MAX_OBSERVE_STATE_ENTRIES: usize = 64; + +#[derive(Default)] +pub struct GuiHandler { + observe_state: Mutex>, +} + +#[derive(Clone, Debug)] +struct ObserveState { + capture: CaptureArtifact, + app_name: Option, +} + +#[derive(Clone, Debug, Default)] +pub(super) struct HostCaptureExclusionState { + applied: bool, + frontmost_excluded: bool, + adjusted: bool, + frontmost_app_name: Option, + frontmost_bundle_id: Option, + redaction_count: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum CaptureMode { + Window, + Display, +} + +impl CaptureMode { + fn as_str(self) -> &'static str { + match self { + CaptureMode::Window => "window", + CaptureMode::Display => "display", + } + } +} + +impl std::fmt::Display for CaptureMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Serialize for CaptureMode { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +#[derive(Clone, Debug)] +pub(super) struct CaptureArtifact { + pub(super) origin_x: f64, + pub(super) origin_y: f64, + pub(super) width: u32, + pub(super) height: u32, + pub(super) image_width: u32, + pub(super) image_height: u32, + pub(super) display_index: i64, + pub(super) capture_mode: CaptureMode, + pub(super) window_title: Option, + pub(super) window_count: Option, + pub(super) window_capture_strategy: Option, + pub(super) host_exclusion: HostCaptureExclusionState, +} + +impl CaptureArtifact { + pub(super) fn scale_x(&self) -> f64 { + if self.width > 0 { + self.image_width as f64 / self.width as f64 + } else { + 1.0 + } + } + + pub(super) fn scale_y(&self) -> f64 { + if self.height > 0 { + self.image_height as f64 / self.height as f64 + } else { + 1.0 + } + } +} + +impl ObserveState { + fn capture_bounds(&self) -> HelperRect { + HelperRect { + x: self.capture.origin_x, + y: self.capture.origin_y, + width: self.capture.width as f64, + height: self.capture.height as f64, + } + } +} + +#[derive(Clone, Debug, Deserialize)] +struct WindowSelector { + title: Option, + title_contains: Option, + index: Option, +} + +#[derive(Debug, Deserialize)] +struct ObserveArgs { + app: Option, + target: Option, + location_hint: Option, + scope: Option, + grounding_mode: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + return_image: Option, +} + +#[derive(Debug, Deserialize)] +struct ClickArgs { + target: Option, + x: Option, + y: Option, + coordinate_space: Option, + location_hint: Option, + scope: Option, + grounding_mode: Option, + button: Option, + clicks: Option, + hold_ms: Option, + settle_ms: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct WaitArgs { + target: String, + location_hint: Option, + scope: Option, + grounding_mode: Option, + state: Option, + timeout_ms: Option, + interval_ms: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct DragArgs { + from_target: Option, + from_x: Option, + from_y: Option, + from_location_hint: Option, + from_scope: Option, + to_target: Option, + to_x: Option, + to_y: Option, + to_location_hint: Option, + to_scope: Option, + coordinate_space: Option, + grounding_mode: Option, + duration_ms: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct ScrollArgs { + direction: Option, + distance: Option, + amount: Option, + target: Option, + location_hint: Option, + scope: Option, + grounding_mode: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct TypeArgs { + value: Option, + secret_env_var: Option, + secret_command_env_var: Option, + target: Option, + location_hint: Option, + scope: Option, + grounding_mode: Option, + replace: Option, + submit: Option, + type_strategy: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct KeyArgs { + key: String, + modifiers: Option>, + repeat: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct MoveArgs { + x: f64, + y: f64, + app: Option, +} + +#[derive(Debug, Deserialize)] +struct BatchArgs { + steps: Vec, + app: Option, + capture_mode: Option, + window_title: Option, + window_selector: Option, +} + +#[derive(Debug, Deserialize)] +struct BatchStep { + action: String, + // Semantic targeting (click, type, scroll) + target: Option, + location_hint: Option, + scope: Option, + // Click-specific + button: Option, + clicks: Option, + hold_ms: Option, + settle_ms: Option, + // Type-specific + value: Option, + secret_env_var: Option, + secret_command_env_var: Option, + replace: Option, + submit: Option, + type_strategy: Option, + // Key-specific + key: Option, + modifiers: Option>, + repeat: Option, + // Scroll-specific + direction: Option, + distance: Option, + amount: Option, + // Drag-specific + from_target: Option, + from_location_hint: Option, + from_scope: Option, + to_target: Option, + to_location_hint: Option, + to_scope: Option, + duration_ms: Option, +} + +const MAX_BATCH_STEPS: usize = 10; + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct HelperCaptureContext { + app_name: Option, + cursor: HelperPoint, + display: HelperDisplayDescriptor, + window_id: Option, + window_title: Option, + window_bounds: Option, + window_count: Option, + window_capture_strategy: Option, + host_self_exclude_applied: Option, + host_frontmost_excluded: Option, + host_frontmost_app_name: Option, + host_frontmost_bundle_id: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct HelperPoint { + x: f64, + y: f64, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct HelperDisplayDescriptor { + index: i64, + bounds: HelperRect, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct HelperRect { + x: f64, + y: f64, + width: f64, + height: f64, +} + +#[derive(Clone, Debug)] +struct ActionEvidence { + image_url: Option, + state: ObserveState, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct GroundingModelResponse { + status: String, + found: bool, + confidence: Option, + reason: Option, + coordinate_space: Option, + click_point: Option, + bbox: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GroundingBoundingBox { + x1: f64, + y1: f64, + x2: f64, + y2: f64, +} + +#[derive(Clone, Debug)] +struct ResolvedTarget { + window_title: Option, + provider: String, + confidence: f64, + reason: Option, + grounding_mode_requested: String, + grounding_mode_effective: String, + scope: Option, + point: HelperPoint, + bounds: HelperRect, + local_point: Option, + local_bounds: Option, + raw: Option, + capture_state: ObserveState, +} + +#[derive(Clone, Debug)] +struct TargetProbe { + capture_state: ObserveState, + target: Option, + timed_out: bool, +} + +#[derive(Clone, Debug)] +struct GroundedGuiTarget { + grounding_method: &'static str, + resolved: ResolvedTarget, +} + +#[derive(Clone, Debug)] +struct GuiTargetProbeResult { + matched: bool, + attempts: i64, + grounded: Option, + state: ObserveState, + image_url: Option, +} + +#[derive(Clone, Copy, Debug)] +struct GuiTargetRequest<'a> { + app: Option<&'a str>, + capture_mode: Option<&'a str>, + window_selection: Option<&'a WindowSelector>, + target: &'a str, + location_hint: Option<&'a str>, + scope: Option<&'a str>, + grounding_mode: Option<&'a str>, + action: &'static str, + related_target: Option<&'a str>, + related_scope: Option<&'a str>, + related_location_hint: Option<&'a str>, + related_point: Option<&'a HelperPoint>, +} + +#[derive(Clone, Copy, Debug)] +enum DragEndpoint<'a> { + Target { + target: &'a str, + location_hint: Option<&'a str>, + scope: Option<&'a str>, + }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GuiCoordinateSpace { + ImagePixels, + DisplayPoints, +} + +#[derive(Clone, Copy, Debug)] +enum ScrollDirection { + Up, + Down, + Left, + Right, +} + +#[derive(Clone, Copy, Debug)] +struct ResolvedGuiScrollPlan { + amount: i64, + distance_preset: &'static str, + unit: &'static str, + viewport_dimension: Option, + viewport_source: Option<&'static str>, + travel_fraction: Option, +} + +pub struct GuiToolOutput { + body: Vec, + code_result: JsonValue, + success: bool, +} + +impl GuiToolOutput {} + +impl ToolOutput for GuiToolOutput { + fn log_preview(&self) -> String { + self.body + .iter() + .find_map(|item| match item { + FunctionCallOutputContentItem::InputText { text } => Some(text.clone()), + _ => None, + }) + .unwrap_or_default() + } + + fn success_for_logging(&self) -> bool { + self.success + } + + fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem { + ResponseInputItem::FunctionCallOutput { + call_id: call_id.to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(self.body.clone()), + success: Some(self.success), + }, + } + } + + fn code_mode_result(&self, _payload: &ToolPayload) -> JsonValue { + self.code_result.clone() + } +} + +#[async_trait] +impl ToolHandler for GuiHandler { + type Output = GuiToolOutput; + + fn kind(&self) -> ToolKind { + ToolKind::Function + } + + async fn handle(&self, invocation: ToolInvocation) -> Result { + match invocation.tool_name.as_str() { + "gui_observe" => self.handle_observe(invocation).await, + "gui_wait" => self.handle_wait(invocation).await, + "gui_click" => self.handle_click(invocation).await, + "gui_drag" => self.handle_drag(invocation).await, + "gui_scroll" => self.handle_scroll(invocation).await, + "gui_type" => self.handle_type(invocation).await, + "gui_key" => self.handle_key(invocation).await, + "gui_move" => self.handle_move(invocation).await, + "gui_batch" => self.handle_batch(invocation).await, + name => Err(FunctionCallError::RespondToModel(format!( + "unsupported gui tool `{name}`" + ))), + } + } +} + +impl GuiHandler { + async fn set_observe_state(&self, conversation_id: &str, state: ObserveState) { + let mut map = self.observe_state.lock().await; + map.insert(conversation_id.to_string(), state); + if map.len() > MAX_OBSERVE_STATE_ENTRIES { + // Evict arbitrary excess entries to prevent unbounded growth. + let keys_to_remove: Vec = map + .keys() + .filter(|k| k.as_str() != conversation_id) + .take(map.len() - MAX_OBSERVE_STATE_ENTRIES) + .cloned() + .collect(); + for key in keys_to_remove { + map.remove(&key); + } + } + } + + async fn handle_observe( + &self, + invocation: ToolInvocation, + ) -> Result { + let args = parse_function_args::(&invocation.payload)?; + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let semantic_target = normalize_optional_string(args.target.as_deref()); + let location_hint = normalize_optional_string(args.location_hint.as_deref()); + let scope = normalize_optional_string(args.scope.as_deref()); + let attach_image = + prepare_gui_observe_request(&invocation, semantic_target.is_some(), args.return_image) + .await?; + let observation = observe_platform( + args.app.as_deref(), + /*activate_app*/ true, + args.capture_mode.as_deref(), + window_selection.as_ref(), + args.app.as_deref().is_some(), + ) + .await?; + let state = observation.state; + self.set_observe_state( + &invocation.session.conversation_id.to_string(), + state.clone(), + ) + .await; + let image_output = attach_image.then(|| screenshot_data_url(&observation.image_bytes)); + let app_label = state + .app_name + .as_ref() + .map(|app| format!(" for app `{app}`")) + .unwrap_or_default(); + let subject = if state.capture.capture_mode == CaptureMode::Window { + state + .capture + .window_title + .as_ref() + .map(|title| format!("window `{title}`")) + .unwrap_or_else(|| "window".to_string()) + } else { + format!("display {}", state.capture.display_index) + }; + if let Some(target) = semantic_target.as_deref() { + let grounded = self + .resolve_gui_target( + &invocation, + GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: location_hint.as_deref(), + scope: scope.as_deref(), + grounding_mode: args.grounding_mode.as_deref(), + action: "observe", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + ) + .await?; + let Some(grounded) = grounded else { + let summary = format!( + "Captured {platform} {subject}{app_label}, but could not resolve semantic GUI target `{target}` in the observed surface.", + platform = PLATFORM_NAME + ); + return Ok(self.build_gui_output( + summary, + state, + image_output, + false, + Some(serde_json::json!({ + "error": format!("No confident semantic GUI target `{target}` was found."), + "target": target, + "grounding_method": "grounding", + "grounding_mode_requested": normalize_grounding_mode(args.grounding_mode.as_deref(), "observe")?, + "grounding_mode_effective": normalize_grounding_mode(args.grounding_mode.as_deref(), "observe")?, + "scope": scope, + "confidence": 0.0, + })), + )); + }; + let summary = format!( + "Captured {platform} {subject}{app_label} and resolved semantic GUI target `{target}` in the observed surface.", + platform = PLATFORM_NAME + ); + let mut extra = serde_json::Map::new(); + extra.insert( + "target_resolution".to_string(), + build_target_resolution_details(target, &grounded), + ); + return Ok(self.build_gui_output( + summary, + state, + image_output, + true, + Some(JsonValue::Object(extra)), + )); + } + + let summary = format!( + "Captured {platform} {subject}{app_label} at origin ({origin_x}, {origin_y}) with size {width}x{height} for visual inspection and follow-up GUI grounding.", + platform = PLATFORM_NAME, + origin_x = state.capture.origin_x.round(), + origin_y = state.capture.origin_y.round(), + width = state.capture.width, + height = state.capture.height + ); + Ok(self.build_gui_output(summary, state, image_output, true, None)) + } + + async fn handle_wait( + &self, + invocation: ToolInvocation, + ) -> Result { + let args = parse_function_args::(&invocation.payload)?; + let mut window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let mut app = normalize_optional_string(args.app.as_deref()); + let mut capture_mode = normalize_optional_string(args.capture_mode.as_deref()); + let target = normalize_optional_string(Some(args.target.as_str())).ok_or_else(|| { + FunctionCallError::RespondToModel("gui_wait requires a semantic `target`.".to_string()) + })?; + let location_hint = normalize_optional_string(args.location_hint.as_deref()); + let scope = normalize_optional_string(args.scope.as_deref()); + enforce_gui_tool_capability(&invocation, "gui_wait", true).await?; + if app.is_none() && capture_mode.is_none() && window_selection.is_none() { + if let Some(previous_state) = self.get_observe_state(&invocation).await { + app = previous_state.app_name.clone(); + capture_mode = Some(previous_state.capture.capture_mode.to_string()); + if previous_state.capture.capture_mode == CaptureMode::Window { + window_selection = + previous_state + .capture + .window_title + .as_ref() + .map(|title| WindowSelector { + title: Some(title.clone()), + title_contains: None, + index: None, + }); + } + } + } + + let target_state = normalize_wait_target_state(args.state.as_deref())?; + let timeout_ms = args.timeout_ms.unwrap_or(DEFAULT_GUI_WAIT_TIMEOUT_MS); + let interval_ms = args.interval_ms.unwrap_or(DEFAULT_GUI_WAIT_INTERVAL_MS); + if timeout_ms <= 0 { + return Err(FunctionCallError::RespondToModel( + "gui_wait.timeout_ms must be a positive integer".to_string(), + )); + } + if interval_ms <= 0 { + return Err(FunctionCallError::RespondToModel( + "gui_wait.interval_ms must be a positive integer".to_string(), + )); + } + + let probe = self + .probe_for_target( + &invocation, + GuiTargetRequest { + app: app.as_deref(), + capture_mode: capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target: &target, + location_hint: location_hint.as_deref(), + scope: scope.as_deref(), + grounding_mode: args.grounding_mode.as_deref(), + action: "wait", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + target_state, + timeout_ms, + interval_ms, + ) + .await?; + + let summary = if probe.matched { + match (target_state, probe.grounded.as_ref()) { + ("appear", Some(resolved)) => format!( + "Confirmed GUI target `{target}` appeared after {} visual checks and {} consecutive confirmations at global ({}, {}).{}", + probe.attempts, + WAIT_CONFIRMATION_COUNT, + resolved.resolved.point.x.round(), + resolved.resolved.point.y.round(), + if probe.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ), + ("disappear", _) => format!( + "Confirmed GUI target `{target}` disappeared after {} visual checks and {} consecutive confirmations.{}", + probe.attempts, + WAIT_CONFIRMATION_COUNT, + if probe.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ), + _ => unreachable!("validated wait target state"), + } + } else { + format!( + "Timed out after {timeout_ms}ms waiting for GUI target `{target}` to {target_state}.{}", + if probe.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ) + }; + + Ok(self.build_gui_output( + summary, + probe.state, + probe.image_url, + probe.matched, + Some(serde_json::json!({ + "timeout_ms": timeout_ms, + "interval_ms": interval_ms, + "target": target, + "target_state": target_state, + "attempts": probe.attempts, + "wait_confirmations_required": WAIT_CONFIRMATION_COUNT, + "target_found": probe.grounded.is_some(), + "grounding_method": probe.grounded.as_ref().map(|grounded| grounded.grounding_method), + "target_resolution": probe.grounded.as_ref().map(|grounded| build_target_resolution_details(&target, grounded)), + })), + )) + } + + async fn handle_click( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_click", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let semantic_target = normalize_optional_string(args.target.as_deref()); + let location_hint = normalize_optional_string(args.location_hint.as_deref()); + let scope = normalize_optional_string(args.scope.as_deref()); + let coordinate_point = normalize_optional_coordinate_point(args.x, args.y, "x", "y")?; + if coordinate_point.is_none() + && normalize_optional_string(args.coordinate_space.as_deref()).is_some() + { + return Err(FunctionCallError::RespondToModel( + "gui_click.coordinate_space requires both `x` and `y`.".to_string(), + )); + } + if semantic_target.is_some() && coordinate_point.is_some() { + return Err(FunctionCallError::RespondToModel( + "gui_click accepts either semantic `target` fields or direct coordinates, not both in the same call." + .to_string(), + )); + } + if coordinate_point.is_some() { + normalize_coordinate_space(args.coordinate_space.as_deref())?; + + if !invocation.turn.tools_config.gui_coordinate_targeting { + return Err(FunctionCallError::RespondToModel( + "Direct coordinate GUI clicking is disabled by default. Enable `[tools.gui] coordinate_targeting = true` only if you intentionally want to keep the experimental placeholder path visible." + .to_string(), + )); + } + return Err(FunctionCallError::RespondToModel( + GUI_DIRECT_COORDINATE_PLACEHOLDER_MESSAGE.to_string(), + )); + } + let target = semantic_target.as_deref().ok_or_else(|| { + FunctionCallError::RespondToModel("gui_click requires a semantic `target`.".to_string()) + })?; + enforce_gui_tool_capability(&invocation, "gui_click", true).await?; + let grounded = self + .resolve_gui_target( + &invocation, + GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: location_hint.as_deref(), + scope: scope.as_deref(), + grounding_mode: args.grounding_mode.as_deref(), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + ) + .await?; + let grounded = grounded.ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "Could not resolve semantic GUI target `{target}`." + )) + })?; + let target_details = build_target_resolution_details(target, &grounded); + let resolved = grounded.resolved; + let coordinate_summary = resolved + .local_point + .as_ref() + .map(|point| { + format!( + "target `{target}` at image coordinate ({}, {})", + point.x.round(), + point.y.round() + ) + }) + .unwrap_or_else(|| { + format!( + "target `{target}` at global coordinate ({}, {})", + resolved.point.x.round(), + resolved.point.y.round() + ) + }); + let global_x = resolved.point.x; + let global_y = resolved.point.y; + let state = resolved.capture_state; + let target_details = Some(target_details); + let button = args.button.as_deref().unwrap_or("left"); + let clicks = args.clicks.unwrap_or(1); + let hold_ms = args.hold_ms.unwrap_or(DEFAULT_CLICK_AND_HOLD_MS).max(1); + let settle_ms = args.settle_ms.unwrap_or(DEFAULT_HOVER_SETTLE_MS).max(1); + let event_mode = match (button, clicks, args.hold_ms) { + ("none", 1, None) => "move_cursor", + ("left", 1, None) => "click", + ("left", 1, Some(_)) => "click_and_hold", + ("left", 2, None) => "double_click", + ("right", 1, None) => "right_click", + ("none", _, Some(_)) => { + return Err(FunctionCallError::RespondToModel( + "gui_click cannot combine `button: none` with `hold_ms`".to_string(), + )); + } + ("none", other, None) => { + return Err(FunctionCallError::RespondToModel(format!( + "gui_click with `button: none` only supports a single hover action, got `{other}`" + ))); + } + ("left", 2, Some(_)) => { + return Err(FunctionCallError::RespondToModel( + "gui_click cannot combine `clicks: 2` with `hold_ms`".to_string(), + )); + } + ("left", other, _) => { + return Err(FunctionCallError::RespondToModel(format!( + "gui_click only supports 1 or 2 left clicks, got `{other}`" + ))); + } + ("right", other, _) => { + return Err(FunctionCallError::RespondToModel(format!( + "gui_click only supports a single right click, got `{other}`" + ))); + } + (other, _, _) => { + return Err(FunctionCallError::RespondToModel(format!( + "gui_click.button only supports `left`, `right`, or `none`, got `{other}`" + ))); + } + }; + action_session.throw_if_emergency_stopped()?; + + run_gui_event( + event_mode, + args.app.as_deref(), + &[("CODEX_GUI_X", global_x), ("CODEX_GUI_Y", global_y)], + &[ + ("CODEX_GUI_HOLD_MS", hold_ms.to_string()), + ("CODEX_GUI_SETTLE_MS", settle_ms.to_string()), + ], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + DEFAULT_POST_ACTION_SETTLE_MS, + ) + .await?; + + let summary = format!( + "{action} at {coordinate_summary} on {platform} {mode} {subject} (global {gx}, {gy}).{evidence_note} Use gui_wait or gui_observe to verify the resulting UI state before the next risky action.", + action = describe_click_action(button, clicks, args.hold_ms.is_some()), + mode = state.capture.capture_mode.as_str(), + platform = PLATFORM_NAME, + subject = describe_capture_subject(&state), + gx = global_x.round(), + gy = global_y.round(), + evidence_note = if evidence.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + let mut extra_details = serde_json::Map::new(); + extra_details.insert( + "action_kind".to_string(), + JsonValue::String(event_mode.to_string()), + ); + extra_details.insert("executed_point".to_string(), point_json(global_x, global_y)); + if let Some(target_details) = target_details { + extend_object_fields(&mut extra_details, target_details); + } + extra_details.insert( + "pre_action_capture".to_string(), + build_capture_details_from_state(&state), + ); + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + Some(JsonValue::Object(extra_details)), + )) + } + + async fn handle_drag( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_drag", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let from_target = normalize_optional_string(args.from_target.as_deref()); + let from_point = + normalize_optional_coordinate_point(args.from_x, args.from_y, "from_x", "from_y")?; + let from_location_hint = normalize_optional_string(args.from_location_hint.as_deref()); + let from_scope = normalize_optional_string(args.from_scope.as_deref()); + let to_target = normalize_optional_string(args.to_target.as_deref()); + let to_point = normalize_optional_coordinate_point(args.to_x, args.to_y, "to_x", "to_y")?; + let to_location_hint = normalize_optional_string(args.to_location_hint.as_deref()); + let to_scope = normalize_optional_string(args.to_scope.as_deref()); + if from_target.is_some() && from_point.is_some() { + return Err(FunctionCallError::RespondToModel( + "gui_drag source accepts either `from_target` fields or `from_x`/`from_y`, not both in the same call." + .to_string(), + )); + } + if to_target.is_some() && to_point.is_some() { + return Err(FunctionCallError::RespondToModel( + "gui_drag destination accepts either `to_target` fields or `to_x`/`to_y`, not both in the same call." + .to_string(), + )); + } + let uses_direct_coordinates = from_point.is_some() || to_point.is_some(); + if !uses_direct_coordinates + && normalize_optional_string(args.coordinate_space.as_deref()).is_some() + { + return Err(FunctionCallError::RespondToModel( + "gui_drag.coordinate_space requires at least one coordinate endpoint.".to_string(), + )); + } + if uses_direct_coordinates { + normalize_coordinate_space(args.coordinate_space.as_deref())?; + + if !invocation.turn.tools_config.gui_coordinate_targeting { + return Err(FunctionCallError::RespondToModel( + "Direct coordinate GUI dragging is disabled by default. Enable `[tools.gui] coordinate_targeting = true` only if you intentionally want to keep the experimental placeholder path visible." + .to_string(), + )); + } + return Err(FunctionCallError::RespondToModel( + GUI_DIRECT_COORDINATE_PLACEHOLDER_MESSAGE.to_string(), + )); + } + enforce_gui_tool_capability(&invocation, "gui_drag", true).await?; + let source_endpoint = normalize_drag_endpoint( + "source", + "from_target", + from_target.as_deref(), + from_location_hint.as_deref(), + from_scope.as_deref(), + )?; + let destination_endpoint = normalize_drag_endpoint( + "destination", + "to_target", + to_target.as_deref(), + to_location_hint.as_deref(), + to_scope.as_deref(), + )?; + // Ground both drag endpoints in parallel for ~2x speed. + let source_req = match &source_endpoint { + DragEndpoint::Target { + target, + location_hint, + scope, + } => GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: *location_hint, + scope: *scope, + grounding_mode: args.grounding_mode.as_deref(), + action: "drag_source", + related_target: to_target.as_deref(), + related_scope: to_scope.as_deref(), + related_location_hint: to_location_hint.as_deref(), + related_point: None, + }, + }; + let dest_req = match &destination_endpoint { + DragEndpoint::Target { + target, + location_hint, + scope, + } => GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: *location_hint, + scope: *scope, + grounding_mode: args.grounding_mode.as_deref(), + action: "drag_destination", + related_target: from_target.as_deref(), + related_scope: from_scope.as_deref(), + related_location_hint: from_location_hint.as_deref(), + related_point: None, + }, + }; + let (source_result, dest_result) = tokio::join!( + self.resolve_gui_target(&invocation, source_req), + self.resolve_gui_target(&invocation, dest_req), + ); + let (from_global_x, from_global_y, state, from_summary, source_target_details) = { + let DragEndpoint::Target { target, .. } = source_endpoint; + let grounded = source_result?.ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "Could not resolve semantic GUI drag source `{target}`." + )) + })?; + let target_details = build_target_resolution_details(target, &grounded); + let resolved = grounded.resolved; + let summary = resolved + .local_point + .as_ref() + .map(|point| { + format!( + "target `{target}` at image coordinate ({}, {})", + point.x.round(), + point.y.round() + ) + }) + .unwrap_or_else(|| { + format!( + "target `{target}` at global coordinate ({}, {})", + resolved.point.x.round(), + resolved.point.y.round() + ) + }); + ( + resolved.point.x, + resolved.point.y, + resolved.capture_state, + summary, + Some(target_details), + ) + }; + let (to_global_x, to_global_y, to_summary, destination_target_details) = { + let DragEndpoint::Target { target, .. } = destination_endpoint; + let grounded = dest_result?.ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "Could not resolve semantic GUI drag destination `{target}`." + )) + })?; + let target_details = build_target_resolution_details(target, &grounded); + let resolved = grounded.resolved; + let summary = resolved + .local_point + .as_ref() + .map(|point| { + format!( + "target `{target}` at image coordinate ({}, {})", + point.x.round(), + point.y.round() + ) + }) + .unwrap_or_else(|| { + format!( + "target `{target}` at global coordinate ({}, {})", + resolved.point.x.round(), + resolved.point.y.round() + ) + }); + ( + resolved.point.x, + resolved.point.y, + summary, + Some(target_details), + ) + }; + let duration_ms = args.duration_ms.unwrap_or(DEFAULT_DRAG_DURATION_MS).max(1); + let steps = DEFAULT_DRAG_STEPS; + action_session.throw_if_emergency_stopped()?; + + run_gui_event( + "drag", + args.app.as_deref(), + &[ + ("CODEX_GUI_FROM_X", from_global_x), + ("CODEX_GUI_FROM_Y", from_global_y), + ("CODEX_GUI_TO_X", to_global_x), + ("CODEX_GUI_TO_Y", to_global_y), + ], + &[ + ("CODEX_GUI_DURATION_MS", duration_ms.to_string()), + ("CODEX_GUI_STEPS", steps.to_string()), + ], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + DEFAULT_POST_ACTION_SETTLE_MS, + ) + .await?; + let summary = format!( + "Dragged from {from_summary} to {to_summary} on {platform} {mode} {subject} (global {fx}, {fy} -> {tx}, {ty}).{evidence_note} Use gui_wait or gui_observe to confirm the drop landed where you expected.", + mode = state.capture.capture_mode.as_str(), + platform = PLATFORM_NAME, + subject = describe_capture_subject(&state), + fx = from_global_x.round(), + fy = from_global_y.round(), + tx = to_global_x.round(), + ty = to_global_y.round(), + evidence_note = if evidence.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + let mut extra_details = serde_json::Map::new(); + extra_details.insert( + "action_kind".to_string(), + JsonValue::String("drag".to_string()), + ); + extra_details.insert( + "executed_from_point".to_string(), + point_json(from_global_x, from_global_y), + ); + extra_details.insert( + "executed_to_point".to_string(), + point_json(to_global_x, to_global_y), + ); + extra_details.insert( + "pre_action_capture".to_string(), + build_capture_details_from_state(&state), + ); + if let Some(source_target_details) = source_target_details { + extend_object_fields(&mut extra_details, source_target_details); + } + if let Some(destination_target_details) = destination_target_details { + extra_details.insert( + "destination_target_resolution".to_string(), + destination_target_details, + ); + } + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + (!extra_details.is_empty()).then_some(JsonValue::Object(extra_details)), + )) + } + + async fn handle_scroll( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_scroll", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let direction = normalize_scroll_direction(args.direction.as_deref())?; + let distance = normalize_scroll_distance(args.distance.as_deref())?; + let semantic_target = normalize_optional_string(args.target.as_deref()); + let location_hint = normalize_optional_string(args.location_hint.as_deref()); + let scope = normalize_optional_string(args.scope.as_deref()); + enforce_gui_tool_capability(&invocation, "gui_scroll", semantic_target.is_some()).await?; + + let mut float_env = Vec::new(); + let mut state_for_summary = None; + let mut target_details = None; + let mut executed_point = None; + let mut target_bounds = None; + if let Some(target) = semantic_target.as_deref() { + let grounded = self + .resolve_gui_target( + &invocation, + GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: location_hint.as_deref(), + scope: scope.as_deref(), + grounding_mode: args.grounding_mode.as_deref(), + action: "scroll", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + ) + .await?; + let grounded = grounded.ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "Could not resolve semantic GUI target `{target}` for scrolling." + )) + })?; + let details = build_target_resolution_details(target, &grounded); + let resolved = grounded.resolved; + float_env.push(("CODEX_GUI_X", resolved.point.x)); + float_env.push(("CODEX_GUI_Y", resolved.point.y)); + executed_point = Some((resolved.point.x, resolved.point.y)); + target_bounds = Some(resolved.bounds.clone()); + state_for_summary = Some(resolved.capture_state); + target_details = Some(details); + } else if args.app.is_some() || args.capture_mode.is_some() || window_selection.is_some() { + let context = + capture_context(args.app.as_deref(), false, window_selection.as_ref()).await?; + let capture = resolve_capture_target( + &context, + args.capture_mode.as_deref(), + window_selection.is_some(), + args.app.as_deref().is_some(), + )?; + state_for_summary = Some(ObserveState { + capture: CaptureArtifact { + origin_x: capture.bounds.x, + origin_y: capture.bounds.y, + width: capture.width, + height: capture.height, + image_width: capture.width, + image_height: capture.height, + display_index: context.display.index, + capture_mode: capture.mode, + window_title: capture.window_title, + window_count: capture.window_count, + window_capture_strategy: capture.window_capture_strategy, + host_exclusion: HostCaptureExclusionState { + applied: context.host_self_exclude_applied.unwrap_or(false), + frontmost_excluded: context.host_frontmost_excluded.unwrap_or(false), + adjusted: capture.host_self_exclude_adjusted, + frontmost_app_name: context.host_frontmost_app_name.clone(), + frontmost_bundle_id: context.host_frontmost_bundle_id.clone(), + redaction_count: 0, + }, + }, + app_name: context.app_name, + }); + } else if let Some(previous_state) = self.get_observe_state(&invocation).await { + state_for_summary = Some(previous_state); + } + + let capture_bounds = state_for_summary.as_ref().map(ObserveState::capture_bounds); + let scroll_plan = resolve_scroll_plan( + args.amount, + distance, + semantic_target.is_some(), + direction, + target_bounds.as_ref(), + capture_bounds.as_ref(), + ); + let (delta_x, delta_y) = scroll_delta_components(direction, scroll_plan.amount); + action_session.throw_if_emergency_stopped()?; + + run_gui_event( + "scroll", + args.app.as_deref(), + &float_env, + &[ + ("CODEX_GUI_SCROLL_X", delta_x.to_string()), + ("CODEX_GUI_SCROLL_Y", delta_y.to_string()), + ("CODEX_GUI_SCROLL_UNIT", scroll_plan.unit.to_string()), + ], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + DEFAULT_POST_ACTION_SETTLE_MS, + ) + .await?; + let summary = format!( + "Scrolled {platform} GUI {dir} using `{preset}` distance ({amount} {unit}).{context}.{evidence_note} Refresh with gui_wait or gui_observe before grounding the next GUI action.", + platform = PLATFORM_NAME, + dir = scroll_direction_label(direction), + preset = scroll_plan.distance_preset, + amount = scroll_plan.amount, + unit = scroll_plan.unit, + context = state_for_summary + .as_ref() + .map(|state| format!( + " on {} {}", + state.capture.capture_mode, + describe_capture_subject(state) + )) + .unwrap_or_default(), + evidence_note = if evidence.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + let mut extra_details = serde_json::Map::new(); + extra_details.insert( + "action_kind".to_string(), + JsonValue::String("scroll".to_string()), + ); + extra_details.insert( + "scroll_direction".to_string(), + JsonValue::String(scroll_direction_label(direction).to_string()), + ); + extra_details.insert( + "scroll_distance".to_string(), + JsonValue::String(scroll_plan.distance_preset.to_string()), + ); + extra_details.insert( + "scroll_amount".to_string(), + JsonValue::from(scroll_plan.amount), + ); + extra_details.insert( + "scroll_unit".to_string(), + JsonValue::String(scroll_plan.unit.to_string()), + ); + if let Some(viewport_dimension) = scroll_plan.viewport_dimension { + extra_details.insert( + "scroll_viewport_dimension".to_string(), + JsonValue::from(viewport_dimension), + ); + } + if let Some(viewport_source) = scroll_plan.viewport_source { + extra_details.insert( + "scroll_viewport_source".to_string(), + JsonValue::String(viewport_source.to_string()), + ); + } + if let Some(travel_fraction) = scroll_plan.travel_fraction { + extra_details.insert( + "scroll_travel_fraction".to_string(), + JsonValue::from(travel_fraction), + ); + } + if let Some((x, y)) = executed_point { + extra_details.insert("executed_point".to_string(), point_json(x, y)); + } + if let Some(target_details) = target_details { + extend_object_fields(&mut extra_details, target_details); + if let Some(state) = state_for_summary.as_ref() { + extra_details.insert( + "pre_action_capture".to_string(), + build_capture_details_from_state(state), + ); + } + } else { + extra_details.insert( + "grounding_method".to_string(), + JsonValue::String("targetless".to_string()), + ); + extra_details.insert("confidence".to_string(), JsonValue::from(1.0)); + if let Some(state) = state_for_summary.as_ref() { + extra_details.insert( + "pre_action_capture".to_string(), + build_capture_details_from_state(state), + ); + } + } + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + Some(JsonValue::Object(extra_details)), + )) + } + + async fn handle_type( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_type", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let text = resolve_type_value(&args)?; + let semantic_target = normalize_optional_string(args.target.as_deref()); + let location_hint = normalize_optional_string(args.location_hint.as_deref()); + let scope = normalize_optional_string(args.scope.as_deref()); + enforce_gui_tool_capability(&invocation, "gui_type", semantic_target.is_some()).await?; + let strategy = normalize_optional_string(args.type_strategy.as_deref()); + if let Some(strategy) = strategy.as_deref() + && !matches!( + strategy, + "clipboard_paste" + | "physical_keys" + | "system_events_paste" + | "system_events_keystroke" + | "system_events_keystroke_chars" + ) + { + return Err(FunctionCallError::RespondToModel(format!( + "gui_type.type_strategy only supports `clipboard_paste`, `physical_keys`, `system_events_paste`, `system_events_keystroke`, or `system_events_keystroke_chars`, got `{strategy}`" + ))); + } + action_session.throw_if_emergency_stopped()?; + + let mut target_details = None; + let mut executed_point = None; + let mut pre_action_capture = None; + if let Some(target) = semantic_target.as_deref() { + let grounded = self + .resolve_gui_target( + &invocation, + GuiTargetRequest { + app: args.app.as_deref(), + capture_mode: args.capture_mode.as_deref(), + window_selection: window_selection.as_ref(), + target, + location_hint: location_hint.as_deref(), + scope: scope.as_deref(), + grounding_mode: args.grounding_mode.as_deref(), + action: "type", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + ) + .await?; + let grounded = grounded.ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "Could not resolve semantic input target `{target}`." + )) + })?; + let details = build_target_resolution_details(target, &grounded); + let resolved = grounded.resolved; + let focus_point = targeted_type_focus_point(&resolved); + run_gui_event( + "click", + args.app.as_deref(), + &[ + ("CODEX_GUI_X", focus_point.x), + ("CODEX_GUI_Y", focus_point.y), + ], + &[], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + sleep(Duration::from_millis(DEFAULT_TYPE_FOCUS_SETTLE_MS as u64)).await; + action_session.throw_if_emergency_stopped()?; + executed_point = Some((focus_point.x, focus_point.y)); + pre_action_capture = Some(build_capture_details_from_state(&resolved.capture_state)); + target_details = Some(details); + } else { + prepare_targeted_gui_action( + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + ) + .await?; + action_session.throw_if_emergency_stopped()?; + } + + let replace = args.replace.unwrap_or(true); + let submit = args.submit.unwrap_or(false); + let effective_strategy = if matches!( + strategy.as_deref(), + Some("system_events_paste") + | Some("system_events_keystroke") + | Some("system_events_keystroke_chars") + ) { + run_system_events_type( + args.app.as_deref(), + window_selection.as_ref(), + &text, + replace, + submit, + strategy + .as_deref() + .expect("system events strategy should be present"), + ) + .await?; + strategy.clone() + } else if let Some(native_strategy) = strategy.as_deref() { + run_gui_event( + "type_text", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_TEXT", text.clone()), + ( + "CODEX_GUI_REPLACE", + if replace { "1" } else { "0" }.to_string(), + ), + ( + "CODEX_GUI_SUBMIT", + if submit { "1" } else { "0" }.to_string(), + ), + ("CODEX_GUI_TYPE_STRATEGY", native_strategy.to_string()), + ], + ) + .await?; + Some(native_strategy.to_string()) + } else { + // Prefer the paste-based macOS typing path first, then fall back + // to native unicode injection if System Events is unavailable. + if run_system_events_type( + args.app.as_deref(), + window_selection.as_ref(), + &text, + replace, + submit, + "system_events_paste", + ) + .await + .is_err() + { + let native_strategy = "unicode"; + run_gui_event( + "type_text", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_TEXT", text.clone()), + ( + "CODEX_GUI_REPLACE", + if replace { "1" } else { "0" }.to_string(), + ), + ( + "CODEX_GUI_SUBMIT", + if submit { "1" } else { "0" }.to_string(), + ), + ("CODEX_GUI_TYPE_STRATEGY", native_strategy.to_string()), + ], + ) + .await?; + Some("unicode".to_string()) + } else { + Some("system_events_paste".to_string()) + } + }; + action_session.throw_if_emergency_stopped()?; + + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + DEFAULT_POST_TYPE_SETTLE_MS, + ) + .await?; + let summary = format!( + "Typed {} character(s){}{}.{} Use gui_wait or gui_observe to verify the field contents and any follow-on UI changes.", + text.chars().count(), + strategy + .as_ref() + .map(|value| format!(" with strategy `{value}`")) + .unwrap_or_default(), + semantic_target + .as_ref() + .map(|target| format!(" into semantic target `{target}`")) + .unwrap_or_default(), + if evidence.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + let mut extra_details = serde_json::Map::new(); + extra_details.insert( + "action_kind".to_string(), + JsonValue::String("type_text".to_string()), + ); + if let Some(strategy) = strategy { + extra_details.insert( + "type_strategy_requested".to_string(), + JsonValue::String(strategy), + ); + } + if let Some(effective_strategy) = effective_strategy { + extra_details.insert( + "type_strategy_effective".to_string(), + JsonValue::String(effective_strategy), + ); + } + if let Some((x, y)) = executed_point { + extra_details.insert("executed_point".to_string(), point_json(x, y)); + } + if let Some(target_details) = target_details { + extend_object_fields(&mut extra_details, target_details); + if let Some(pre_action_capture) = pre_action_capture { + extra_details.insert("pre_action_capture".to_string(), pre_action_capture); + } + } else { + extra_details.insert( + "grounding_method".to_string(), + JsonValue::String("targetless".to_string()), + ); + extra_details.insert("confidence".to_string(), JsonValue::from(1.0)); + } + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + Some(JsonValue::Object(extra_details)), + )) + } + + async fn handle_key( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_key", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + let repeat = args.repeat.unwrap_or(1).max(1); + let mut modifiers = args.modifiers.unwrap_or_default(); + enforce_gui_tool_capability(&invocation, "gui_key", false).await?; + let key_code = resolve_key_code(&args.key, &mut modifiers)?; + let modifiers_env = modifiers.join(","); + prepare_targeted_gui_action( + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + ) + .await?; + action_session.throw_if_emergency_stopped()?; + + // Escape keycode 53: tell the emergency stop monitor to treat the + // upcoming Escape detection as a programmatic event, not a user abort. + if key_code == 53 { + action_session.expect_escape(); + } + + run_gui_event( + "key_press", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_KEY_CODE", key_code.to_string()), + ("CODEX_GUI_REPEAT", repeat.to_string()), + ("CODEX_GUI_MODIFIERS", modifiers_env.clone()), + ], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + DEFAULT_POST_TYPE_SETTLE_MS, + ) + .await?; + let summary = format!( + "Pressed key `{}`{} {} time(s).{} Use gui_wait or gui_observe if this shortcut should change the visible UI.", + args.key, + if modifiers_env.is_empty() { + String::new() + } else { + format!(" with modifiers [{}]", modifiers_env) + }, + repeat, + if evidence.image_url.is_some() { + " Attached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + Some(serde_json::json!({ + "action_kind": "key_press", + "grounding_method": "targetless", + "confidence": 1.0, + "repeat": repeat, + })), + )) + } + + async fn handle_move( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_move", true)?; + let args = parse_function_args::(&invocation.payload)?; + action_session.hide_other_apps(args.app.as_deref()); + enforce_gui_tool_capability(&invocation, "gui_move", false).await?; + action_session.throw_if_emergency_stopped()?; + run_gui_event( + "move_cursor", + args.app.as_deref(), + &[("CODEX_GUI_X", args.x), ("CODEX_GUI_Y", args.y)], + &[("CODEX_GUI_SETTLE_MS", DEFAULT_HOVER_SETTLE_MS.to_string())], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + + let summary = format!( + "Moved the {platform} pointer to absolute display coordinate ({x}, {y}).", + platform = PLATFORM_NAME, + x = args.x.round(), + y = args.y.round() + ); + Ok(GuiToolOutput { + body: vec![FunctionCallOutputContentItem::InputText { + text: summary.clone(), + }], + code_result: serde_json::json!({ + "message": summary, + "action_kind": "move_cursor", + "grounding_method": "absolute_coordinates", + "confidence": 1.0, + "executed_point": { + "x": args.x, + "y": args.y, + }, + "app": args.app, + }), + success: true, + }) + } + + async fn handle_batch( + &self, + invocation: ToolInvocation, + ) -> Result { + let mut action_session = + session::begin_gui_action_session(&invocation, "gui_batch", true)?; + let args = parse_function_args::(&invocation.payload)?; + if args.steps.is_empty() { + return Err(FunctionCallError::RespondToModel( + "gui_batch requires at least one step.".to_string(), + )); + } + if args.steps.len() > MAX_BATCH_STEPS { + return Err(FunctionCallError::RespondToModel(format!( + "gui_batch supports at most {MAX_BATCH_STEPS} steps, got {}.", + args.steps.len() + ))); + } + for (i, step) in args.steps.iter().enumerate() { + if !matches!( + step.action.as_str(), + "click" | "type" | "key" | "scroll" | "drag" + ) { + return Err(FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: unsupported action `{}`. Supported: click, type, key, scroll, drag.", + step.action + ))); + } + } + + action_session.hide_other_apps(args.app.as_deref()); + let window_selection = normalize_window_selection( + args.window_title.as_deref(), + args.window_selector.as_ref(), + )?; + enforce_gui_tool_capability(&invocation, "gui_batch", true).await?; + + // ── Step 1: Collect grounding targets ────────────────────────── + let mut grounding_targets: Vec = Vec::new(); + for (i, step) in args.steps.iter().enumerate() { + if step.action == "drag" { + // Drag produces two grounding targets: source + destination. + if let Some(from) = normalize_optional_string(step.from_target.as_deref()) { + grounding_targets.push(grounding::BatchGroundingTarget { + step_index: i, + role: grounding::BatchGroundingRole::Primary, + target: from, + action: "drag_source".to_string(), + location_hint: normalize_optional_string(step.from_location_hint.as_deref()), + scope: normalize_optional_string(step.from_scope.as_deref()), + }); + } + if let Some(to) = normalize_optional_string(step.to_target.as_deref()) { + grounding_targets.push(grounding::BatchGroundingTarget { + step_index: i, + role: grounding::BatchGroundingRole::DragDestination, + target: to, + action: "drag_destination".to_string(), + location_hint: normalize_optional_string(step.to_location_hint.as_deref()), + scope: normalize_optional_string(step.to_scope.as_deref()), + }); + } + } else { + let semantic_target = normalize_optional_string(step.target.as_deref()); + if let Some(target) = semantic_target { + grounding_targets.push(grounding::BatchGroundingTarget { + step_index: i, + role: grounding::BatchGroundingRole::Primary, + target, + action: step.action.clone(), + location_hint: normalize_optional_string(step.location_hint.as_deref()), + scope: normalize_optional_string(step.scope.as_deref()), + }); + } + } + } + + // ── Step 2 & 3: Screenshot + grounding (only if needed) ─────── + let mut batch_capture_state: Option = None; + let grounded_results = if !grounding_targets.is_empty() { + // Take ONE screenshot for all grounding targets. + let observation = observe_platform( + args.app.as_deref(), + /*activate_app*/ true, + args.capture_mode.as_deref(), + window_selection.as_ref(), + args.app.as_deref().is_some(), + ) + .await?; + let capture_state = observation.state; + self.set_observe_state( + &invocation.session.conversation_id.to_string(), + capture_state.clone(), + ) + .await; + + let image_bytes = if observation.image_bytes.is_empty() { + let bounds = capture_state.capture_bounds(); + capture_region( + &bounds, + capture_state.capture.image_width, + capture_state.capture.image_height, + ) + .await? + } else { + observation.image_bytes + }; + // Dispatch grounding based on configured strategy. + let strategy = &invocation.turn.tools_config.gui_batch_grounding_strategy; + let results = if strategy == "unified" { + grounding::resolve_batch_grounded_targets_unified( + &invocation, + &grounding_targets, + &capture_state, + &image_bytes, + ) + .await? + } else { + grounding::resolve_batch_grounded_targets( + &invocation, + &grounding_targets, + &capture_state, + &image_bytes, + ) + .await? + }; + batch_capture_state = Some(capture_state); + results + } else { + // No semantic targets — just activate the app without screenshotting. + prepare_targeted_gui_action( + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + ) + .await?; + Vec::new() + }; + + // Build a map from (step_index, role) → resolved target. + let mut resolved_map: HashMap< + (usize, grounding::BatchGroundingRole), + ResolvedTarget, + > = HashMap::new(); + for (i, result) in grounded_results.into_iter().enumerate() { + if let Some(resolved) = result { + let gt = &grounding_targets[i]; + resolved_map.insert((gt.step_index, gt.role.clone()), resolved); + } else { + let gt = &grounding_targets[i]; + return Err(FunctionCallError::RespondToModel(format!( + "gui_batch: could not resolve target `{}` for step {} ({} action).", + gt.target, gt.step_index, gt.action + ))); + } + } + + // ── Step 4: Execute each action sequentially ─────────────────── + let mut step_summaries: Vec = Vec::new(); + let mut step_details: Vec = Vec::new(); + + let batch_action_delay_ms = invocation.turn.tools_config.gui_batch_action_delay_ms; + for (i, step) in args.steps.iter().enumerate() { + action_session.throw_if_emergency_stopped()?; + + // Delay between steps to let the UI settle. + if i > 0 && batch_action_delay_ms > 0 { + sleep(Duration::from_millis(batch_action_delay_ms)).await; + } + + match step.action.as_str() { + "click" => { + let resolved = resolved_map.get(&(i, grounding::BatchGroundingRole::Primary)).ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: click requires a `target`." + )) + })?; + let button = step.button.as_deref().unwrap_or("left"); + let clicks = step.clicks.unwrap_or(1); + let hold_ms = step.hold_ms.unwrap_or(DEFAULT_CLICK_AND_HOLD_MS).max(1); + let settle_ms = step.settle_ms.unwrap_or(DEFAULT_HOVER_SETTLE_MS).max(1); + let event_mode = match (button, clicks, step.hold_ms) { + ("none", 1, None) => "move_cursor", + ("left", 1, None) => "click", + ("left", 1, Some(_)) => "click_and_hold", + ("left", 2, None) => "double_click", + ("right", 1, None) => "right_click", + _ => { + return Err(FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: unsupported click variant (button={button}, clicks={clicks})" + ))); + } + }; + run_gui_event( + event_mode, + args.app.as_deref(), + &[ + ("CODEX_GUI_X", resolved.point.x), + ("CODEX_GUI_Y", resolved.point.y), + ], + &[ + ("CODEX_GUI_HOLD_MS", hold_ms.to_string()), + ("CODEX_GUI_SETTLE_MS", settle_ms.to_string()), + ], + ) + .await?; + let target_label = step.target.as_deref().unwrap_or("unknown"); + step_summaries.push(format!( + "step {i}: {action} `{target_label}` at ({x}, {y})", + action = describe_click_action(button, clicks, step.hold_ms.is_some()), + x = resolved.point.x.round(), + y = resolved.point.y.round(), + )); + step_details.push(serde_json::json!({ + "step": i, + "action": "click", + "event_mode": event_mode, + "target": target_label, + "point": { "x": resolved.point.x.round(), "y": resolved.point.y.round() }, + "confidence": resolved.confidence, + })); + } + "type" => { + // Resolve text value + let text = if let Some(value) = &step.value { + value.clone() + } else if let Some(env_var) = &step.secret_env_var { + std::env::var(env_var).map_err(|_| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: secret_env_var `{env_var}` is not set." + )) + })? + } else if let Some(cmd_var) = &step.secret_command_env_var { + std::env::var(cmd_var).map_err(|_| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: secret_command_env_var `{cmd_var}` is not set." + )) + })? + } else { + return Err(FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: type action requires `value`, `secret_env_var`, or `secret_command_env_var`." + ))); + }; + + // If target was specified, click to focus first + if let Some(resolved) = resolved_map.get(&(i, grounding::BatchGroundingRole::Primary)) { + let focus_point = targeted_type_focus_point(resolved); + run_gui_event( + "click", + args.app.as_deref(), + &[ + ("CODEX_GUI_X", focus_point.x), + ("CODEX_GUI_Y", focus_point.y), + ], + &[], + ) + .await?; + action_session.throw_if_emergency_stopped()?; + sleep(Duration::from_millis(DEFAULT_TYPE_FOCUS_SETTLE_MS as u64)).await; + action_session.throw_if_emergency_stopped()?; + } + + let replace = step.replace.unwrap_or(true); + let submit = step.submit.unwrap_or(false); + let strategy = normalize_optional_string(step.type_strategy.as_deref()); + + if matches!( + strategy.as_deref(), + Some("system_events_paste") + | Some("system_events_keystroke") + | Some("system_events_keystroke_chars") + ) { + run_system_events_type( + args.app.as_deref(), + window_selection.as_ref(), + &text, + replace, + submit, + strategy.as_deref().unwrap(), + ) + .await?; + } else if let Some(native_strategy) = strategy.as_deref() { + run_gui_event( + "type_text", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_TEXT", text.clone()), + ("CODEX_GUI_REPLACE", if replace { "1" } else { "0" }.to_string()), + ("CODEX_GUI_SUBMIT", if submit { "1" } else { "0" }.to_string()), + ("CODEX_GUI_TYPE_STRATEGY", native_strategy.to_string()), + ], + ) + .await?; + } else { + // Default: try system_events_paste, fallback to unicode + if run_system_events_type( + args.app.as_deref(), + window_selection.as_ref(), + &text, + replace, + submit, + "system_events_paste", + ) + .await + .is_err() + { + run_gui_event( + "type_text", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_TEXT", text.clone()), + ("CODEX_GUI_REPLACE", if replace { "1" } else { "0" }.to_string()), + ("CODEX_GUI_SUBMIT", if submit { "1" } else { "0" }.to_string()), + ("CODEX_GUI_TYPE_STRATEGY", "unicode".to_string()), + ], + ) + .await?; + } + } + + let target_label = step.target.as_deref().unwrap_or("focused field"); + step_summaries.push(format!( + "step {i}: typed {} chars into `{target_label}`", + text.chars().count(), + )); + step_details.push(serde_json::json!({ + "step": i, + "action": "type", + "chars_typed": text.chars().count(), + "target": target_label, + })); + } + "key" => { + let key = step.key.as_deref().ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: key action requires `key`." + )) + })?; + let mut modifiers = step.modifiers.clone().unwrap_or_default(); + let key_code = resolve_key_code(key, &mut modifiers)?; + let repeat = step.repeat.unwrap_or(1).max(1); + let modifiers_env = modifiers.join(","); + + if key_code == 53 { + action_session.expect_escape(); + } + + run_gui_event( + "key_press", + args.app.as_deref(), + &[], + &[ + ("CODEX_GUI_KEY_CODE", key_code.to_string()), + ("CODEX_GUI_REPEAT", repeat.to_string()), + ("CODEX_GUI_MODIFIERS", modifiers_env.clone()), + ], + ) + .await?; + + step_summaries.push(format!( + "step {i}: pressed key `{key}`{}", + if modifiers_env.is_empty() { + String::new() + } else { + format!(" with [{modifiers_env}]") + }, + )); + step_details.push(serde_json::json!({ + "step": i, + "action": "key", + "key": key, + "repeat": repeat, + })); + } + "scroll" => { + let direction = normalize_scroll_direction(step.direction.as_deref())?; + let distance = normalize_scroll_distance(step.distance.as_deref())?; + + let mut float_env: Vec<(&str, f64)> = Vec::new(); + let mut target_bounds = None; + if let Some(resolved) = resolved_map.get(&(i, grounding::BatchGroundingRole::Primary)) { + float_env.push(("CODEX_GUI_X", resolved.point.x)); + float_env.push(("CODEX_GUI_Y", resolved.point.y)); + target_bounds = Some(resolved.bounds.clone()); + } + + let capture_bounds = + batch_capture_state.as_ref().map(ObserveState::capture_bounds); + let scroll_plan = resolve_scroll_plan( + step.amount, + distance, + resolved_map.contains_key(&(i, grounding::BatchGroundingRole::Primary)), + direction, + target_bounds.as_ref(), + capture_bounds.as_ref(), + ); + let (delta_x, delta_y) = scroll_delta_components(direction, scroll_plan.amount); + + run_gui_event( + "scroll", + args.app.as_deref(), + &float_env, + &[ + ("CODEX_GUI_SCROLL_X", delta_x.to_string()), + ("CODEX_GUI_SCROLL_Y", delta_y.to_string()), + ("CODEX_GUI_SCROLL_UNIT", scroll_plan.unit.to_string()), + ], + ) + .await?; + + let target_label = step.target.as_deref().unwrap_or("current surface"); + step_summaries.push(format!( + "step {i}: scrolled {dir} on `{target_label}`", + dir = scroll_direction_label(direction), + )); + step_details.push(serde_json::json!({ + "step": i, + "action": "scroll", + "direction": scroll_direction_label(direction), + "target": target_label, + })); + } + "drag" => { + let from_target_str = step.from_target.as_deref().ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: drag requires `from_target`." + )) + })?; + let to_target_str = step.to_target.as_deref().ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: drag requires `to_target`." + )) + })?; + let from_resolved = resolved_map + .get(&(i, grounding::BatchGroundingRole::Primary)) + .ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: could not resolve drag source `{from_target_str}`." + )) + })?; + let to_resolved = resolved_map + .get(&(i, grounding::BatchGroundingRole::DragDestination)) + .ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "gui_batch step {i}: could not resolve drag destination `{to_target_str}`." + )) + })?; + + let duration_ms = step.duration_ms.unwrap_or(DEFAULT_DRAG_DURATION_MS).max(1); + run_gui_event( + "drag", + args.app.as_deref(), + &[ + ("CODEX_GUI_FROM_X", from_resolved.point.x), + ("CODEX_GUI_FROM_Y", from_resolved.point.y), + ("CODEX_GUI_TO_X", to_resolved.point.x), + ("CODEX_GUI_TO_Y", to_resolved.point.y), + ], + &[ + ("CODEX_GUI_DURATION_MS", duration_ms.to_string()), + ("CODEX_GUI_STEPS", DEFAULT_DRAG_STEPS.to_string()), + ], + ) + .await?; + + step_summaries.push(format!( + "step {i}: dragged `{from_target_str}` → `{to_target_str}`", + )); + step_details.push(serde_json::json!({ + "step": i, + "action": "drag", + "from_target": from_target_str, + "to_target": to_target_str, + "from_point": { "x": from_resolved.point.x.round(), "y": from_resolved.point.y.round() }, + "to_point": { "x": to_resolved.point.x.round(), "y": to_resolved.point.y.round() }, + })); + } + _ => unreachable!("action validated above"), + } + } + + action_session.throw_if_emergency_stopped()?; + + // ── Step 5: ONE evidence screenshot ──────────────────────────── + // Use shorter settle for batches that only contain typing and key + // presses; use the full settle for click/scroll actions that may + // trigger larger UI changes. + let has_click_or_scroll = args + .steps + .iter() + .any(|s| s.action == "click" || s.action == "scroll" || s.action == "drag"); + let settle_ms = if has_click_or_scroll { + DEFAULT_POST_ACTION_SETTLE_MS + } else { + DEFAULT_POST_TYPE_SETTLE_MS + }; + let evidence = self + .capture_evidence_image( + &invocation, + args.app.as_deref(), + args.capture_mode.as_deref(), + window_selection.as_ref(), + settle_ms, + ) + .await?; + + // ── Step 6: Build combined result ────────────────────────────── + let summary = format!( + "Executed {} GUI actions in batch on {platform}:\n{steps}{evidence_note}\nUse gui_wait or gui_observe to verify the resulting UI state.", + args.steps.len(), + platform = PLATFORM_NAME, + steps = step_summaries + .iter() + .map(|s| format!(" - {s}")) + .collect::>() + .join("\n"), + evidence_note = if evidence.image_url.is_some() { + "\nAttached a refreshed GUI evidence screenshot." + } else { + "" + } + ); + + let extra_details = serde_json::json!({ + "action_kind": "batch", + "steps_count": args.steps.len(), + "grounding_targets_count": grounding_targets.len(), + "grounding_method": "batch_grounding", + "steps": step_details, + }); + + Ok(self.build_gui_output( + summary, + evidence.state, + evidence.image_url, + true, + Some(extra_details), + )) + } + + async fn get_observe_state(&self, invocation: &ToolInvocation) -> Option { + self.observe_state + .lock() + .await + .get(&invocation.session.conversation_id.to_string()) + .cloned() + } + + async fn capture_post_action_evidence( + &self, + invocation: &ToolInvocation, + app: Option<&str>, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, + default_settle_ms: i64, + ) -> Result { + let attach_image = supports_image_input(invocation); + let mut app = normalize_optional_string(app); + let mut capture_mode = normalize_optional_string(capture_mode); + let mut window_selection = window_selection.cloned(); + + if app.is_none() && capture_mode.is_none() && window_selection.is_none() { + if let Some(previous_state) = self.get_observe_state(invocation).await { + app = previous_state.app_name.clone(); + capture_mode = Some(previous_state.capture.capture_mode.to_string()); + if previous_state.capture.capture_mode == CaptureMode::Window { + window_selection = + previous_state + .capture + .window_title + .as_ref() + .map(|title| WindowSelector { + title: Some(title.clone()), + title_contains: None, + index: None, + }); + } + } + } + + sleep(Duration::from_millis(default_settle_ms.max(0) as u64)).await; + + // Do not re-activate the app when capturing post-action evidence. + // The action (click/drag/type/etc.) already targeted the active app, + // and re-activation can reset transient UI state such as selected + // items, hover highlights, or chess piece selection. + let observation = observe_platform( + app.as_deref(), + /*activate_app*/ false, + capture_mode.as_deref(), + window_selection.as_ref(), + app.as_deref().is_some(), + ) + .await?; + let image_bytes = if attach_image { + Some(observation.image_bytes.clone()) + } else { + None + }; + let image_url = image_bytes.as_deref().map(screenshot_data_url); + let state = observation.state; + self.set_observe_state( + &invocation.session.conversation_id.to_string(), + state.clone(), + ) + .await; + + Ok(ActionEvidence { image_url, state }) + } + + fn build_gui_output( + &self, + summary: String, + state: ObserveState, + image_url: Option, + success: bool, + extra_details: Option, + ) -> GuiToolOutput { + let mut body = vec![FunctionCallOutputContentItem::InputText { + text: summary.clone(), + }]; + if let Some(image_url) = &image_url { + body.push(FunctionCallOutputContentItem::InputImage { + image_url: image_url.clone(), + detail: None, + }); + } + + let mut code_result = serde_json::json!({ + "message": summary, + "image_url": image_url, + "display_index": state.capture.display_index, + "capture_mode": state.capture.capture_mode, + "origin_x": state.capture.origin_x, + "origin_y": state.capture.origin_y, + "width": state.capture.width, + "height": state.capture.height, + "image_width": state.capture.image_width, + "image_height": state.capture.image_height, + "capture_scale_x": state.capture.scale_x(), + "capture_scale_y": state.capture.scale_y(), + "app": state.app_name, + "window_title": state.capture.window_title, + "window_count": state.capture.window_count, + "window_capture_strategy": state.capture.window_capture_strategy, + "capture_host_self_exclude_applied": state.capture.host_exclusion.applied, + "capture_host_frontmost_excluded": state.capture.host_exclusion.frontmost_excluded, + "capture_host_self_exclude_adjusted": state.capture.host_exclusion.adjusted, + "capture_host_frontmost_app": state.capture.host_exclusion.frontmost_app_name, + "capture_host_frontmost_bundle_id": state.capture.host_exclusion.frontmost_bundle_id, + "capture_host_self_exclude_redaction_count": state.capture.host_exclusion.redaction_count, + }); + if let Some(JsonValue::Object(extra)) = extra_details + && let Some(base) = code_result.as_object_mut() + { + for (key, value) in extra { + base.insert(key, value); + } + } + + GuiToolOutput { + body, + code_result, + success, + } + } + + async fn probe_semantic_target( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + ) -> Result { + let observation = observe_platform( + request.app, + true, + request.capture_mode, + request.window_selection, + request.app.is_some(), + ) + .await?; + let capture_state = observation.state; + let target = self + .ground_target( + invocation, + request, + &capture_state, + &observation.image_bytes, + ) + .await?; + Ok(TargetProbe { + capture_state, + target, + timed_out: false, + }) + } + + async fn probe_semantic_target_before_deadline( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + deadline: Instant, + ) -> Result { + // Do not activate the app during polling probes. The caller + // (gui_wait) may be checking whether a transient state persists + // and re-activation could dismiss it. + let observation = observe_platform( + request.app, + /*activate_app*/ false, + request.capture_mode, + request.window_selection, + request.app.is_some(), + ) + .await?; + let capture_state = observation.state; + let Some(remaining) = remaining_wait_budget_duration(deadline) else { + return Ok(TargetProbe { + capture_state, + target: None, + timed_out: true, + }); + }; + let target = match timeout( + remaining, + self.ground_target( + invocation, + request, + &capture_state, + &observation.image_bytes, + ), + ) + .await + { + Ok(result) => result?, + Err(_) => { + return Ok(TargetProbe { + capture_state, + target: None, + timed_out: true, + }); + } + }; + Ok(TargetProbe { + capture_state, + target, + timed_out: false, + }) + } + + async fn resolve_gui_target( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + ) -> Result, FunctionCallError> { + let initial_probe = self + .probe_semantic_target( + invocation, + GuiTargetRequest { + capture_mode: fallback_probe_capture_mode( + request.capture_mode, + /*attempt*/ 1, + request.app, + ), + ..request + }, + ) + .await?; + if let Some(resolved) = initial_probe.target { + return Ok(Some(GroundedGuiTarget { + grounding_method: "grounding", + resolved, + })); + } + + let should_retry_with_display = request.capture_mode.is_none() + && request.app.is_some() + && initial_probe.capture_state.capture.capture_mode == CaptureMode::Window; + if !should_retry_with_display { + return Ok(None); + } + + let fallback_probe = self + .probe_semantic_target( + invocation, + GuiTargetRequest { + capture_mode: fallback_probe_capture_mode( + request.capture_mode, + /*attempt*/ 2, + request.app, + ), + ..request + }, + ) + .await?; + Ok(fallback_probe.target.map(|resolved| GroundedGuiTarget { + grounding_method: "grounding_display_fallback", + resolved, + })) + } + + async fn ground_target( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + image_bytes: &[u8], + ) -> Result, FunctionCallError> { + if !supports_image_input(invocation) { + return Err(FunctionCallError::RespondToModel( + GUI_IMAGE_UNSUPPORTED_MESSAGE.to_string(), + )); + } + + let image_bytes = if image_bytes.is_empty() { + let bounds = capture_state.capture_bounds(); + capture_region( + &bounds, + capture_state.capture.image_width, + capture_state.capture.image_height, + ) + .await? + } else { + image_bytes.to_vec() + }; + default_gui_grounding_provider() + .ground(invocation, request, capture_state, &image_bytes) + .await + } + + async fn capture_evidence_image( + &self, + invocation: &ToolInvocation, + app: Option<&str>, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, + default_settle_ms: i64, + ) -> Result { + self.capture_post_action_evidence( + invocation, + app, + capture_mode, + window_selection, + default_settle_ms, + ) + .await + } + + async fn probe_for_target( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + state: &'static str, + timeout_ms: i64, + interval_ms: i64, + ) -> Result { + let attach_image = supports_image_input(invocation); + let deadline = Instant::now() + Duration::from_millis(timeout_ms.max(0) as u64); + let mut attempts = 0; + let initial_probe = self + .probe_semantic_target_before_deadline( + invocation, + GuiTargetRequest { + capture_mode: fallback_probe_capture_mode(request.capture_mode, 1, request.app), + ..request + }, + deadline, + ) + .await?; + let mut last_grounded = initial_probe.target.map(|resolved| GroundedGuiTarget { + grounding_method: "grounding", + resolved, + }); + attempts += 1; + let mut current_state = initial_probe.capture_state; + let mut budget_exhausted = initial_probe.timed_out; + let initial_satisfied = match state { + "appear" => last_grounded.is_some(), + "disappear" => last_grounded.is_none(), + _ => false, + }; + let mut consecutive_satisfied = if initial_satisfied { 1 } else { 0 }; + let mut matched = consecutive_satisfied >= WAIT_CONFIRMATION_COUNT; + + while !matched && !budget_exhausted { + let Some(remaining_ms) = remaining_wait_budget_ms(deadline) else { + break; + }; + let sleep_ms = remaining_ms.min(interval_ms as u64); + sleep(Duration::from_millis(sleep_ms)).await; + let Some(_) = remaining_wait_budget_ms(deadline) else { + break; + }; + let probe = self + .probe_semantic_target_before_deadline( + invocation, + GuiTargetRequest { + capture_mode: fallback_probe_capture_mode( + request.capture_mode, + attempts + 1, + request.app, + ), + ..request + }, + deadline, + ) + .await?; + current_state = probe.capture_state.clone(); + last_grounded = probe.target.map(|resolved| GroundedGuiTarget { + grounding_method: "grounding", + resolved, + }); + attempts += 1; + budget_exhausted = probe.timed_out; + if budget_exhausted { + break; + } + let satisfied = match state { + "appear" => last_grounded.is_some(), + "disappear" => last_grounded.is_none(), + _ => false, + }; + consecutive_satisfied = if satisfied { + consecutive_satisfied + 1 + } else { + 0 + }; + matched = consecutive_satisfied >= WAIT_CONFIRMATION_COUNT; + } + + self.set_observe_state( + &invocation.session.conversation_id.to_string(), + current_state.clone(), + ) + .await; + + let image_url = if attach_image { + Some(capture_image_url_for_state(¤t_state).await?) + } else { + None + }; + + Ok(GuiTargetProbeResult { + matched, + attempts, + grounded: last_grounded, + state: current_state, + image_url, + }) + } +} + +fn parse_function_args(payload: &ToolPayload) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let ToolPayload::Function { arguments } = payload else { + return Err(FunctionCallError::RespondToModel( + "gui handler received unsupported payload".to_string(), + )); + }; + parse_arguments(arguments) +} + +fn supports_image_input(invocation: &ToolInvocation) -> bool { + invocation + .turn + .model_info + .input_modalities + .contains(&InputModality::Image) +} + +async fn prepare_gui_observe_request( + invocation: &ToolInvocation, + targeted: bool, + return_image: Option, +) -> Result { + enforce_gui_tool_capability(invocation, "gui_observe", targeted).await?; + Ok(return_image.unwrap_or(true) && supports_image_input(invocation)) +} + +fn normalize_wait_target_state(state: Option<&str>) -> Result<&'static str, FunctionCallError> { + match state.map(str::trim).filter(|value| !value.is_empty()) { + None => Ok("appear"), + Some("appear") => Ok("appear"), + Some("disappear") => Ok("disappear"), + Some(other) => Err(FunctionCallError::RespondToModel(format!( + "gui_wait.state only supports `appear` or `disappear`, got `{other}`" + ))), + } +} + +fn normalize_grounding_mode( + grounding_mode: Option<&str>, + action: &str, +) -> Result<&'static str, FunctionCallError> { + match grounding_mode + .map(str::trim) + .filter(|value| !value.is_empty()) + { + None => Ok(default_grounding_mode_for_action(action)), + Some("single") => Ok("single"), + Some("complex") => Ok("complex"), + Some(other) => Err(FunctionCallError::RespondToModel(format!( + "{action}.grounding_mode only supports `single` or `complex`, got `{other}`" + ))), + } +} + +fn remaining_wait_budget_ms(deadline: Instant) -> Option { + remaining_wait_budget_duration(deadline).map(|duration| duration.as_millis() as u64) +} + +fn remaining_wait_budget_duration(deadline: Instant) -> Option { + let now = Instant::now(); + if now >= deadline { + None + } else { + Some(deadline.duration_since(now)) + } +} + +fn default_grounding_mode_for_action(action: &str) -> &'static str { + match action { + "type" | "drag_source" | "drag_destination" => "complex", + _ => "single", + } +} + +fn normalize_scroll_direction( + direction: Option<&str>, +) -> Result { + match direction + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("down") + { + "up" => Ok(ScrollDirection::Up), + "down" => Ok(ScrollDirection::Down), + "left" => Ok(ScrollDirection::Left), + "right" => Ok(ScrollDirection::Right), + other => Err(FunctionCallError::RespondToModel(format!( + "gui_scroll.direction only supports `up`, `down`, `left`, or `right`, got `{other}`" + ))), + } +} + +fn normalize_scroll_distance( + distance: Option<&str>, +) -> Result, FunctionCallError> { + match distance.map(str::trim).filter(|value| !value.is_empty()) { + None => Ok(None), + Some("small") => Ok(Some("small")), + Some("medium") => Ok(Some("medium")), + Some("page") => Ok(Some("page")), + Some(other) => Err(FunctionCallError::RespondToModel(format!( + "gui_scroll.distance only supports `small`, `medium`, or `page`, got `{other}`" + ))), + } +} + +fn scroll_direction_uses_horizontal_axis(direction: ScrollDirection) -> bool { + matches!(direction, ScrollDirection::Left | ScrollDirection::Right) +} + +fn scroll_viewport_dimension_for_direction(rect: &HelperRect, direction: ScrollDirection) -> i64 { + if scroll_direction_uses_horizontal_axis(direction) { + rect.width.round().max(1.0) as i64 + } else { + rect.height.round().max(1.0) as i64 + } +} + +fn scroll_distance_fraction(distance: &str) -> f64 { + match distance { + "small" => 0.25, + "medium" => 0.5, + "page" => 0.75, + _ => 0.5, + } +} + +fn scroll_distance_line_amount(distance: &str) -> i64 { + match distance { + "small" => 3, + "medium" => 5, + "page" => 12, + _ => 5, + } +} + +fn resolve_scroll_plan( + amount: Option, + distance: Option<&'static str>, + has_target: bool, + direction: ScrollDirection, + target_bounds: Option<&HelperRect>, + capture_bounds: Option<&HelperRect>, +) -> ResolvedGuiScrollPlan { + if let Some(amount) = amount { + return ResolvedGuiScrollPlan { + amount: amount.clamp(1, 50), + distance_preset: "custom", + unit: "line", + viewport_dimension: None, + viewport_source: None, + travel_fraction: None, + }; + } + + let distance_preset = distance.unwrap_or(if has_target { + DEFAULT_TARGETED_SCROLL_DISTANCE + } else { + DEFAULT_TARGETLESS_SCROLL_DISTANCE + }); + + if let Some(bounds) = target_bounds { + let viewport_dimension = scroll_viewport_dimension_for_direction(bounds, direction); + let travel_fraction = scroll_distance_fraction(distance_preset); + return ResolvedGuiScrollPlan { + amount: (viewport_dimension as f64 * travel_fraction) + .round() + .clamp(1.0, 4000.0) as i64, + distance_preset, + unit: "pixel", + viewport_dimension: Some(viewport_dimension), + viewport_source: Some("target_box"), + travel_fraction: Some(travel_fraction), + }; + } + + if let Some(bounds) = capture_bounds { + let viewport_dimension = scroll_viewport_dimension_for_direction(bounds, direction); + let travel_fraction = scroll_distance_fraction(distance_preset); + return ResolvedGuiScrollPlan { + amount: (viewport_dimension as f64 * travel_fraction) + .round() + .clamp(1.0, 4000.0) as i64, + distance_preset, + unit: "pixel", + viewport_dimension: Some(viewport_dimension), + viewport_source: Some("capture_rect"), + travel_fraction: Some(travel_fraction), + }; + } + + ResolvedGuiScrollPlan { + amount: scroll_distance_line_amount(distance_preset), + distance_preset, + unit: "line", + viewport_dimension: None, + viewport_source: None, + travel_fraction: None, + } +} + +fn scroll_delta_components(direction: ScrollDirection, amount: i64) -> (i64, i64) { + match direction { + ScrollDirection::Up => (0, amount), + ScrollDirection::Down => (0, -amount), + ScrollDirection::Left => (-amount, 0), + ScrollDirection::Right => (amount, 0), + } +} + +fn targeted_type_focus_point(resolved: &ResolvedTarget) -> HelperPoint { + let bounds = &resolved.bounds; + if bounds.width.is_finite() + && bounds.height.is_finite() + && bounds.width > 0.0 + && bounds.height > 0.0 + { + return HelperPoint { + x: bounds.x + (bounds.width / 2.0), + y: bounds.y + (bounds.height / 2.0), + }; + } + + resolved.point.clone() +} + +fn scroll_direction_label(direction: ScrollDirection) -> &'static str { + match direction { + ScrollDirection::Up => "up", + ScrollDirection::Down => "down", + ScrollDirection::Left => "left", + ScrollDirection::Right => "right", + } +} + +fn normalize_drag_endpoint<'a>( + endpoint_label: &str, + target_field: &str, + target: Option<&'a str>, + location_hint: Option<&'a str>, + scope: Option<&'a str>, +) -> Result, FunctionCallError> { + let Some(target) = target else { + return Err(FunctionCallError::RespondToModel(format!( + "gui_drag requires `{target_field}` for the {endpoint_label}." + ))); + }; + Ok(DragEndpoint::Target { + target, + location_hint, + scope, + }) +} + +fn build_target_resolution_details(target: &str, grounded: &GroundedGuiTarget) -> JsonValue { + let resolved = &grounded.resolved; + serde_json::json!({ + "target": target, + "grounding_method": grounded.grounding_method, + "grounding_provider": resolved.provider, + "grounding_mode_requested": resolved.grounding_mode_requested, + "grounding_mode_effective": resolved.grounding_mode_effective, + "grounding_coordinate_space": "image_pixels", + "confidence": resolved.confidence, + "reason": resolved.reason, + "scope": resolved.scope, + "target_window_title": resolved.window_title, + "grounding_display_point": { + "x": resolved.point.x, + "y": resolved.point.y, + }, + "grounding_display_box": { + "x": resolved.bounds.x, + "y": resolved.bounds.y, + "width": resolved.bounds.width, + "height": resolved.bounds.height, + }, + "grounding_image_box": resolved.local_bounds.as_ref().map(|bounds| serde_json::json!({ + "x": bounds.x, + "y": bounds.y, + "width": bounds.width, + "height": bounds.height, + })), + "target_global_point": { + "x": resolved.point.x, + "y": resolved.point.y, + }, + "target_image_point": resolved.local_point.as_ref().map(|point| serde_json::json!({ + "x": point.x, + "y": point.y, + })), + "target_bounds": { + "x": resolved.bounds.x, + "y": resolved.bounds.y, + "width": resolved.bounds.width, + "height": resolved.bounds.height, + }, + "grounding_diagnostics": build_grounding_diagnostics(resolved.raw.as_ref()), + "raw_grounding": resolved.raw.clone(), + }) +} + +fn build_grounding_diagnostics(raw_grounding: Option<&JsonValue>) -> Option { + let JsonValue::Object(raw) = raw_grounding? else { + return None; + }; + + Some(serde_json::json!({ + "selected_attempt": raw.get("selected_attempt"), + "rounds_attempted": raw.get("grounding_rounds_attempted"), + "validation_triggered": raw.get("grounding_validation_triggered"), + "model_image": raw.get("grounding_model_image"), + "validation": raw.get("validation"), + "round_artifacts": raw.get("grounding_round_artifacts"), + })) +} + +fn build_capture_details_from_state(state: &ObserveState) -> JsonValue { + serde_json::json!({ + "capture_mode": state.capture.capture_mode, + "origin_x": state.capture.origin_x, + "origin_y": state.capture.origin_y, + "width": state.capture.width, + "height": state.capture.height, + "image_width": state.capture.image_width, + "image_height": state.capture.image_height, + "capture_scale_x": state.capture.scale_x(), + "capture_scale_y": state.capture.scale_y(), + "app": state.app_name, + "window_title": state.capture.window_title, + "window_count": state.capture.window_count, + "window_capture_strategy": state.capture.window_capture_strategy, + "capture_host_self_exclude_applied": state.capture.host_exclusion.applied, + "capture_host_frontmost_excluded": state.capture.host_exclusion.frontmost_excluded, + "capture_host_self_exclude_adjusted": state.capture.host_exclusion.adjusted, + "capture_host_frontmost_app": state.capture.host_exclusion.frontmost_app_name, + "capture_host_frontmost_bundle_id": state.capture.host_exclusion.frontmost_bundle_id, + "capture_host_self_exclude_redaction_count": state.capture.host_exclusion.redaction_count, + }) +} + +fn point_json(x: f64, y: f64) -> JsonValue { + serde_json::json!({ + "x": x, + "y": y, + }) +} + +fn extend_object_fields(target: &mut serde_json::Map, value: JsonValue) { + if let JsonValue::Object(fields) = value { + for (key, value) in fields { + target.insert(key, value); + } + } +} + +#[cfg(test)] +fn local_point_within_state(state: &ObserveState, point: &HelperPoint) -> Option { + let local_x = point.x - state.capture.origin_x; + let local_y = point.y - state.capture.origin_y; + if local_x >= 0.0 + && local_y >= 0.0 + && local_x < state.capture.width as f64 + && local_y < state.capture.height as f64 + { + Some(HelperPoint { + x: local_x, + y: local_y, + }) + } else { + None + } +} + +fn image_point_within_capture(state: &ObserveState, point: &HelperPoint) -> Option { + if point.x >= 0.0 + && point.y >= 0.0 + && point.x < state.capture.image_width as f64 + && point.y < state.capture.image_height as f64 + { + Some(point.clone()) + } else { + None + } +} + +fn local_rect_within_state(state: &ObserveState, rect: &HelperRect) -> Option { + if rect.x >= 0.0 + && rect.y >= 0.0 + && rect.width > 0.0 + && rect.height > 0.0 + && rect.x < state.capture.image_width as f64 + && rect.y < state.capture.image_height as f64 + { + // Clamp the rect to image bounds so controls at window edges are + // still usable instead of being discarded entirely. + Some(HelperRect { + x: rect.x, + y: rect.y, + width: rect.width.min(state.capture.image_width as f64 - rect.x), + height: rect.height.min(state.capture.image_height as f64 - rect.y), + }) + } else { + None + } +} + +async fn capture_image_url_for_state(state: &ObserveState) -> Result { + let window_selection = if state.capture.capture_mode == CaptureMode::Window { + state + .capture + .window_title + .as_ref() + .map(|title| WindowSelector { + title: Some(title.clone()), + title_contains: None, + index: None, + }) + } else { + None + }; + let observation = observe_platform( + state.app_name.as_deref(), + false, + Some(state.capture.capture_mode.as_str()), + window_selection.as_ref(), + false, + ) + .await?; + Ok(screenshot_data_url(&observation.image_bytes)) +} + +fn rounded_dimension(value: f64, label: &str) -> Result { + let rounded = value.round(); + if !rounded.is_finite() || rounded <= 0.0 || rounded > u32::MAX as f64 { + return Err(FunctionCallError::RespondToModel(format!( + "invalid {label} from native GUI runtime: {value}" + ))); + } + Ok(rounded as u32) +} + +#[derive(Clone, Debug)] +struct CaptureTarget { + mode: CaptureMode, + bounds: HelperRect, + width: u32, + height: u32, + host_self_exclude_adjusted: bool, + window_title: Option, + window_count: Option, + window_capture_strategy: Option, +} + +fn normalize_optional_string(value: Option<&str>) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + +fn normalize_coordinate_space( + coordinate_space: Option<&str>, +) -> Result { + match coordinate_space + .map(str::trim) + .filter(|value| !value.is_empty()) + { + None | Some("image_pixels") => Ok(GuiCoordinateSpace::ImagePixels), + Some("display_points") => Ok(GuiCoordinateSpace::DisplayPoints), + Some(other) => Err(FunctionCallError::RespondToModel(format!( + "gui coordinate_space only supports `image_pixels` or `display_points`, got `{other}`" + ))), + } +} + +fn normalize_optional_coordinate_point( + x: Option, + y: Option, + x_field: &str, + y_field: &str, +) -> Result, FunctionCallError> { + match (x, y) { + (None, None) => Ok(None), + (Some(x), Some(y)) => Ok(Some(HelperPoint { x, y })), + _ => Err(FunctionCallError::RespondToModel(format!( + "gui coordinate targeting requires both `{x_field}` and `{y_field}`" + ))), + } +} + +fn normalize_window_selection( + window_title: Option<&str>, + selector: Option<&WindowSelector>, +) -> Result, FunctionCallError> { + let title = normalize_optional_string(window_title).or_else(|| { + selector + .and_then(|selector| selector.title.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + }); + let title_contains = selector + .and_then(|selector| selector.title_contains.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned); + let index = selector.and_then(|selector| selector.index); + if let Some(index) = index + && index <= 0 + { + return Err(FunctionCallError::RespondToModel( + "gui window_selector.index must be a positive integer".to_string(), + )); + } + if title.is_none() && title_contains.is_none() && index.is_none() { + return Ok(None); + } + Ok(Some(WindowSelector { + title, + title_contains, + index, + })) +} + +fn normalize_capture_mode( + capture_mode: Option<&str>, +) -> Result, FunctionCallError> { + match capture_mode + .map(str::trim) + .filter(|value| !value.is_empty()) + { + None => Ok(None), + Some("display") => Ok(Some(CaptureMode::Display)), + Some("window") => Ok(Some(CaptureMode::Window)), + Some(other) => Err(FunctionCallError::RespondToModel(format!( + "gui capture_mode only supports `display` or `window`, got `{other}`" + ))), + } +} + +fn fallback_probe_capture_mode<'a>( + requested_capture_mode: Option<&'a str>, + attempt: i64, + app: Option<&str>, +) -> Option<&'a str> { + if requested_capture_mode.is_some() { + return requested_capture_mode; + } + if attempt > 1 && app.is_some() { + Some("display") + } else { + None + } +} + +fn resolve_capture_target( + context: &HelperCaptureContext, + capture_mode: Option<&str>, + window_selection_requested: bool, + prefer_window_when_available: bool, +) -> Result { + let requested_mode = normalize_capture_mode(capture_mode)?; + if window_selection_requested && context.window_bounds.is_none() { + return Err(FunctionCallError::RespondToModel( + "requested window could not be found; check `window_title`/`window_selector` or switch to `capture_mode: \"display\"`" + .to_string(), + )); + } + + let host_self_exclude_adjusted = requested_mode.is_none() + && !window_selection_requested + && !prefer_window_when_available + && context.host_self_exclude_applied.unwrap_or(false) + && context.host_frontmost_excluded.unwrap_or(false) + && context.window_bounds.is_some(); + + let use_window = match requested_mode { + Some(CaptureMode::Window) => context.window_bounds.is_some(), + Some(CaptureMode::Display) => false, + None => { + window_selection_requested + || host_self_exclude_adjusted + || (prefer_window_when_available && context.window_bounds.is_some()) + } + }; + + let (mode, bounds) = if use_window { + let Some(bounds) = context.window_bounds.clone() else { + return Err(FunctionCallError::RespondToModel( + "window capture requested but no matching window bounds were available".to_string(), + )); + }; + (CaptureMode::Window, bounds) + } else { + (CaptureMode::Display, context.display.bounds.clone()) + }; + let width = rounded_dimension(bounds.width, "capture width")?; + let height = rounded_dimension(bounds.height, "capture height")?; + + Ok(CaptureTarget { + mode, + bounds, + width, + height, + host_self_exclude_adjusted, + window_title: if mode == CaptureMode::Window { + context.window_title.clone() + } else { + None + }, + window_count: if mode == CaptureMode::Window { + context.window_count + } else { + None + }, + window_capture_strategy: if mode == CaptureMode::Window { + context.window_capture_strategy.clone() + } else { + None + }, + }) +} + +async fn capture_context( + app: Option<&str>, + activate_app: bool, + window_selection: Option<&WindowSelector>, +) -> Result { + let app = app.map(String::from); + let window_selection = window_selection.cloned(); + tokio::task::spawn_blocking(move || { + default_gui_platform().capture_context( + app.as_deref(), + activate_app, + window_selection.as_ref(), + ) + }) + .await + .map_err(|e| FunctionCallError::RespondToModel(format!("gui platform task panicked: {e}")))? +} + +async fn run_gui_event( + event_mode: &str, + app: Option<&str>, + float_env: &[(&str, f64)], + string_env: &[(&str, String)], +) -> Result<(), FunctionCallError> { + let event_mode = event_mode.to_string(); + let app = app.map(String::from); + let float_env: Vec<(String, f64)> = + float_env.iter().map(|(k, v)| (k.to_string(), *v)).collect(); + let string_env: Vec<(String, String)> = string_env + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(); + tokio::task::spawn_blocking(move || { + let float_refs: Vec<(&str, f64)> = + float_env.iter().map(|(k, v)| (k.as_str(), *v)).collect(); + let string_refs: Vec<(&str, String)> = string_env + .iter() + .map(|(k, v)| (k.as_str(), v.clone())) + .collect(); + default_gui_platform().run_event(&event_mode, app.as_deref(), &float_refs, &string_refs) + }) + .await + .map_err(|e| FunctionCallError::RespondToModel(format!("gui platform task panicked: {e}")))? +} + +async fn prepare_targeted_gui_action( + app: Option<&str>, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, +) -> Result<(), FunctionCallError> { + if app.is_none() && capture_mode.is_none() && window_selection.is_none() { + return Ok(()); + } + + let context = capture_context(app, true, window_selection).await?; + if capture_mode.is_some() || window_selection.is_some() { + let _ = resolve_capture_target( + &context, + capture_mode, + window_selection.is_some(), + app.is_some(), + )?; + } + Ok(()) +} + +fn describe_capture_subject(state: &ObserveState) -> String { + if state.capture.capture_mode == CaptureMode::Window { + state + .capture + .window_title + .clone() + .unwrap_or_else(|| "current window".to_string()) + } else { + format!("display {}", state.capture.display_index) + } +} + +fn describe_click_action(button: &str, clicks: i64, hold: bool) -> String { + match (button, clicks, hold) { + ("none", _, _) => "Hovered pointer".to_string(), + ("left", 1, true) => "Click-and-held".to_string(), + ("left", 2, _) => "Double-clicked".to_string(), + ("right", 1, _) => "Right-clicked".to_string(), + ("left", _, _) => "Clicked".to_string(), + (other, _, _) => format!("Interacted with button `{other}`"), + } +} + +async fn capture_region( + bounds: &HelperRect, + target_width: u32, + target_height: u32, +) -> Result, FunctionCallError> { + let bounds = bounds.clone(); + tokio::task::spawn_blocking(move || { + default_gui_platform().capture_region(&bounds, target_width, target_height) + }) + .await + .map_err(|e| FunctionCallError::RespondToModel(format!("gui platform task panicked: {e}")))? +} + +async fn observe_platform( + app: Option<&str>, + activate_app: bool, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, + prefer_window_when_available: bool, +) -> Result { + let app = app.map(String::from); + let capture_mode = capture_mode.map(String::from); + let window_selection = window_selection.cloned(); + tokio::task::spawn_blocking(move || { + default_gui_platform().observe( + app.as_deref(), + activate_app, + capture_mode.as_deref(), + window_selection.as_ref(), + prefer_window_when_available, + ) + }) + .await + .map_err(|e| FunctionCallError::RespondToModel(format!("gui platform task panicked: {e}")))? +} + +pub(super) fn data_url(bytes: &[u8], mime_type: &str) -> String { + format!("data:{mime_type};base64,{}", BASE64_STANDARD.encode(bytes)) +} + +/// Convert raw PNG screenshot bytes to a JPEG data URL for the model. +/// JPEG at quality 75 is typically 3-5x smaller than PNG, significantly +/// reducing payload size and token consumption. Falls back to PNG if +/// JPEG encoding fails (e.g. corrupt image data). +fn screenshot_data_url(png_bytes: &[u8]) -> String { + if let Ok(img) = image::load_from_memory(png_bytes) { + let mut buf = std::io::Cursor::new(Vec::new()); + let mut encoder = + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, SCREENSHOT_JPEG_QUALITY); + if encoder.encode_image(&img).is_ok() { + return data_url(buf.get_ref(), "image/jpeg"); + } + } + // Fallback: serve the original PNG. + data_url(png_bytes, "image/png") +} + +fn resolve_type_value(args: &TypeArgs) -> Result { + let literal_text = args.value.clone(); + let secret_env_var = normalize_optional_string(args.secret_env_var.as_deref()); + let secret_command_env_var = normalize_optional_string(args.secret_command_env_var.as_deref()); + let configured_source_count = [ + literal_text.is_some(), + secret_env_var.is_some(), + secret_command_env_var.is_some(), + ] + .into_iter() + .filter(|configured| *configured) + .count(); + if configured_source_count == 0 { + return Err(FunctionCallError::RespondToModel( + "gui_type requires a text source: provide exactly one of `value`, `secret_env_var`, or `secret_command_env_var`" + .to_string(), + )); + } + if configured_source_count > 1 { + return Err(FunctionCallError::RespondToModel( + "gui_type accepts only one text source: provide exactly one of `value`, `secret_env_var`, or `secret_command_env_var`" + .to_string(), + )); + } + if let Some(text) = literal_text { + return Ok(text); + } + if let Some(secret_env_var) = secret_env_var { + return std::env::var(&secret_env_var).map_err(|_| { + FunctionCallError::RespondToModel(format!( + "gui_type secret env var `{secret_env_var}` is missing or empty" + )) + }); + } + let Some(secret_command_env_var) = secret_command_env_var else { + return Err(FunctionCallError::RespondToModel( + "gui_type input source could not be resolved".to_string(), + )); + }; + let command = std::env::var(&secret_command_env_var).map_err(|_| { + FunctionCallError::RespondToModel(format!( + "gui_type secret command env var `{secret_command_env_var}` is missing or empty" + )) + })?; + let output = Command::new("/bin/sh") + .args(["-c", &command]) + .output() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to resolve gui_type secret command `{secret_command_env_var}`: {error}" + )) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "gui_type secret command `{secret_command_env_var}` failed: {}", + stderr.trim() + ))); + } + let text = String::from_utf8_lossy(&output.stdout) + .trim_end_matches(&['\r', '\n'][..]) + .to_string(); + if text.is_empty() { + return Err(FunctionCallError::RespondToModel(format!( + "gui_type secret command `{secret_command_env_var}` produced empty output" + ))); + } + Ok(text) +} + +async fn run_system_events_type( + app: Option<&str>, + window_selection: Option<&WindowSelector>, + text: &str, + replace: bool, + submit: bool, + strategy: &str, +) -> Result<(), FunctionCallError> { + let app = app.map(String::from); + let window_selection = window_selection.cloned(); + let text = text.to_string(); + let strategy = strategy.to_string(); + tokio::task::spawn_blocking(move || { + default_gui_platform().run_system_events_type( + app.as_deref(), + window_selection.as_ref(), + &text, + replace, + submit, + &strategy, + ) + }) + .await + .map_err(|e| FunctionCallError::RespondToModel(format!("gui platform task panicked: {e}")))? +} + +#[cfg(test)] +fn resolve_helper_binary() -> Result { + default_gui_platform().resolve_helper_binary() +} + +fn resolve_key_code(key: &str, modifiers: &mut Vec) -> Result { + let trimmed = key.trim(); + if trimmed.is_empty() { + return Err(FunctionCallError::RespondToModel( + "gui_key.key must not be empty".to_string(), + )); + } + + let normalized = trimmed.to_lowercase(); + let named = match normalized.as_str() { + "enter" | "return" => Some(36), + "tab" => Some(48), + "escape" | "esc" => Some(53), + "delete" | "backspace" => Some(51), + "home" => Some(115), + "pageup" => Some(116), + "pagedown" => Some(121), + "end" => Some(119), + "up" | "arrowup" => Some(126), + "down" | "arrowdown" => Some(125), + "left" | "arrowleft" => Some(123), + "right" | "arrowright" => Some(124), + "space" | "spacebar" => Some(49), + _ => None, + }; + if let Some(code) = named { + return Ok(code); + } + + let mut chars = trimmed.chars(); + let Some(first) = chars.next() else { + return Err(FunctionCallError::RespondToModel( + "gui_key.key must not be empty".to_string(), + )); + }; + if chars.next().is_some() { + return Err(FunctionCallError::RespondToModel(format!( + "unsupported gui_key.key `{trimmed}`; use a named key like `Enter` or a single printable character" + ))); + } + + let (code, needs_shift) = match first { + 'a' | 'A' => (0, first.is_uppercase()), + 's' | 'S' => (1, first.is_uppercase()), + 'd' | 'D' => (2, first.is_uppercase()), + 'f' | 'F' => (3, first.is_uppercase()), + 'h' | 'H' => (4, first.is_uppercase()), + 'g' | 'G' => (5, first.is_uppercase()), + 'z' | 'Z' => (6, first.is_uppercase()), + 'x' | 'X' => (7, first.is_uppercase()), + 'c' | 'C' => (8, first.is_uppercase()), + 'v' | 'V' => (9, first.is_uppercase()), + 'b' | 'B' => (11, first.is_uppercase()), + 'q' | 'Q' => (12, first.is_uppercase()), + 'w' | 'W' => (13, first.is_uppercase()), + 'e' | 'E' => (14, first.is_uppercase()), + 'r' | 'R' => (15, first.is_uppercase()), + 'y' | 'Y' => (16, first.is_uppercase()), + 't' | 'T' => (17, first.is_uppercase()), + '1' => (18, false), + '2' => (19, false), + '3' => (20, false), + '4' => (21, false), + '6' => (22, false), + '5' => (23, false), + '=' => (24, false), + '9' => (25, false), + '7' => (26, false), + '-' => (27, false), + '8' => (28, false), + '0' => (29, false), + ']' => (30, false), + 'o' | 'O' => (31, first.is_uppercase()), + 'u' | 'U' => (32, first.is_uppercase()), + '[' => (33, false), + 'i' | 'I' => (34, first.is_uppercase()), + 'p' | 'P' => (35, first.is_uppercase()), + 'l' | 'L' => (37, first.is_uppercase()), + 'j' | 'J' => (38, first.is_uppercase()), + '\'' => (39, false), + 'k' | 'K' => (40, first.is_uppercase()), + ';' => (41, false), + '\\' => (42, false), + ',' => (43, false), + '/' => (44, false), + 'n' | 'N' => (45, first.is_uppercase()), + 'm' | 'M' => (46, first.is_uppercase()), + '.' => (47, false), + ' ' => (49, false), + '!' => (18, true), + '@' => (19, true), + '#' => (20, true), + '$' => (21, true), + '^' => (22, true), + '%' => (23, true), + '+' => (24, true), + '(' => (25, true), + '&' => (26, true), + '_' => (27, true), + '*' => (28, true), + ')' => (29, true), + '}' => (30, true), + '{' => (33, true), + '"' => (39, true), + ':' => (41, true), + '|' => (42, true), + '<' => (43, true), + '?' => (44, true), + '>' => (47, true), + _ => { + return Err(FunctionCallError::RespondToModel(format!( + "unsupported gui_key.key `{trimmed}`" + ))); + } + }; + + if needs_shift + && !modifiers + .iter() + .any(|modifier| modifier.eq_ignore_ascii_case("shift")) + { + modifiers.push("shift".to_string()); + } + + Ok(code) +} diff --git a/codex-rs/core/src/tools/handlers/gui/benchmark_renderer.swift b/codex-rs/core/src/tools/handlers/gui/benchmark_renderer.swift new file mode 100644 index 000000000000..e518fa2c437c --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/benchmark_renderer.swift @@ -0,0 +1,206 @@ +import AppKit +import Foundation +import WebKit + +enum BenchmarkRendererError: Error, CustomStringConvertible { + case usage + case readFailed(String) + case javascriptFailed(String) + case snapshotFailed(String) + case writeFailed(String) + + var description: String { + switch self { + case .usage: + return "usage: benchmark_renderer " + case .readFailed(let message): + return "read failed: \(message)" + case .javascriptFailed(let message): + return "javascript failed: \(message)" + case .snapshotFailed(let message): + return "snapshot failed: \(message)" + case .writeFailed(let message): + return "write failed: \(message)" + } + } +} + +private final class BenchmarkRenderer: NSObject, WKNavigationDelegate { + private let html: String + private let casesJSON: String + private let screenshotPath: String + private let truthsPath: String + private let webView: WKWebView + private var snapshotWidth: CGFloat = 1280 + private var snapshotHeight: CGFloat = 920 + + init(html: String, casesJSON: String, screenshotPath: String, truthsPath: String) { + self.html = html + self.casesJSON = casesJSON + self.screenshotPath = screenshotPath + self.truthsPath = truthsPath + self.webView = WKWebView(frame: NSRect(x: 0, y: 0, width: 1280, height: 920)) + super.init() + self.webView.navigationDelegate = self + } + + func run() { + webView.loadHTMLString(html, baseURL: nil) + RunLoop.main.run() + } + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.10) { + self.evaluateTruths() + } + } + + private func evaluateTruths() { + let escapedCases = casesJSON + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "'", with: "\\'") + .replacingOccurrences(of: "\n", with: "\\n") + let script = """ + (() => { + const cases = JSON.parse('\(escapedCases)'); + const doc = document.documentElement; + const body = document.body; + const pageWidth = Math.max( + doc?.scrollWidth ?? 0, + doc?.offsetWidth ?? 0, + body?.scrollWidth ?? 0, + body?.offsetWidth ?? 0, + window.innerWidth ?? 0, + ); + const pageHeight = Math.max( + doc?.scrollHeight ?? 0, + doc?.offsetHeight ?? 0, + body?.scrollHeight ?? 0, + body?.offsetHeight ?? 0, + window.innerHeight ?? 0, + ); + return JSON.stringify({ + page: { + width: Math.round(pageWidth), + height: Math.round(pageHeight), + }, + truths: cases.map((testCase) => { + const benchmarkId = testCase.elementId ?? testCase.id; + const node = document.querySelector(`[data-benchmark-id="${benchmarkId}"]`); + if (!node) { + throw new Error(`Missing benchmark node for ${benchmarkId}`); + } + const rect = node.getBoundingClientRect(); + return { + ...testCase, + box: { + x: Math.round(rect.left), + y: Math.round(rect.top), + width: Math.round(rect.width), + height: Math.round(rect.height), + }, + point: { + x: Math.round(rect.left + (rect.width / 2)), + y: Math.round(rect.top + (rect.height / 2)), + }, + }; + }), + }); + })(); + """ + + webView.evaluateJavaScript(script) { result, error in + if let error { + self.fail(.javascriptFailed(error.localizedDescription)) + return + } + guard let payloadJSON = result as? String else { + self.fail(.javascriptFailed("renderer returned non-string truth payload")) + return + } + do { + let payloadData = Data(payloadJSON.utf8) + let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any] + let truths = payload?["truths"] ?? [] + if let page = payload?["page"] as? [String: Any] { + if let width = page["width"] as? NSNumber { + self.snapshotWidth = max(1, CGFloat(truncating: width)) + } + if let height = page["height"] as? NSNumber { + self.snapshotHeight = max(1, CGFloat(truncating: height)) + } + } + let truthsOnlyData = try JSONSerialization.data( + withJSONObject: ["truths": truths], + options: [.prettyPrinted] + ) + try truthsOnlyData.write( + to: URL(fileURLWithPath: self.truthsPath), + options: .atomic + ) + } catch { + self.fail(.writeFailed(error.localizedDescription)) + return + } + self.webView.setFrameSize( + NSSize(width: self.snapshotWidth, height: self.snapshotHeight) + ) + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + self.captureSnapshot() + } + } + } + + private func captureSnapshot() { + let config = WKSnapshotConfiguration() + config.rect = NSRect(x: 0, y: 0, width: snapshotWidth, height: snapshotHeight) + webView.takeSnapshot(with: config) { image, error in + if let error { + self.fail(.snapshotFailed(error.localizedDescription)) + return + } + guard + let image, + let tiff = image.tiffRepresentation, + let rep = NSBitmapImageRep(data: tiff), + let png = rep.representation(using: .png, properties: [:]) + else { + self.fail(.snapshotFailed("unable to encode PNG")) + return + } + do { + try png.write(to: URL(fileURLWithPath: self.screenshotPath)) + exit(EXIT_SUCCESS) + } catch { + self.fail(.writeFailed(error.localizedDescription)) + } + } + } + + private func fail(_ error: BenchmarkRendererError) { + fputs("\(error)\n", stderr) + exit(EXIT_FAILURE) + } +} + +let args = CommandLine.arguments +guard args.count == 5 else { + fputs("\(BenchmarkRendererError.usage)\n", stderr) + exit(EXIT_FAILURE) +} + +do { + let html = try String(contentsOfFile: args[1], encoding: .utf8) + let casesJSON = try String(contentsOfFile: args[2], encoding: .utf8) + _ = NSApplication.shared + NSApp.setActivationPolicy(.prohibited) + BenchmarkRenderer( + html: html, + casesJSON: casesJSON, + screenshotPath: args[3], + truthsPath: args[4] + ).run() +} catch { + fputs("\(BenchmarkRendererError.readFailed(error.localizedDescription))\n", stderr) + exit(EXIT_FAILURE) +} diff --git a/codex-rs/core/src/tools/handlers/gui/grounding.rs b/codex-rs/core/src/tools/handlers/gui/grounding.rs new file mode 100644 index 000000000000..25b814b083c1 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/grounding.rs @@ -0,0 +1,2810 @@ +use std::io::Cursor; +use std::time::Duration; + +use crate::Prompt; +use crate::client_common::ResponseEvent; +use crate::function_tool::FunctionCallError; +use crate::tools::context::ToolInvocation; +use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use futures::StreamExt; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageFormat; +use image::Rgba; +use serde::Deserialize; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::Value as JsonValue; + +use super::CaptureArtifact; +use super::GroundingBoundingBox; +use super::GroundingModelResponse; +use super::GuiTargetRequest; +use super::HelperPoint; +use super::HelperRect; +use super::ObserveState; +use super::ResolvedTarget; +use super::data_url; +use super::image_point_within_capture; +use super::local_rect_within_state; +use super::normalize_grounding_mode; + +const GUI_GROUNDING_SYSTEM_PROMPT: &str = concat!( + "You are grounding a GUI target inside a screenshot. ", + "Return JSON only, following the provided schema exactly. ", + "Resolve the requested target only when it is clearly visible in the screenshot. ", + "If the target is not confidently visible, return `status` = `not_found`, `found` = false, ", + "and null coordinates." +); + +const GUI_GROUNDING_VALIDATION_SYSTEM_PROMPT: &str = concat!( + "You are validating a GUI grounding prediction inside a screenshot. ", + "A highlighted marker indicates the proposed click point and target region. ", + "Return JSON only, following the provided schema exactly. ", + "Approve the prediction only when the highlighted point is a good interaction point for the requested target." +); + +const GUI_GROUNDING_REFINEMENT_SYSTEM_PROMPT: &str = concat!( + "You are refining a GUI grounding candidate inside a zoomed crop from the original screenshot. ", + "Return JSON only, following the provided schema exactly. ", + "Refine the point and box to the exact actionable or editable surface inside this crop. ", + "If the crop does not actually contain the requested target, return `status` = `not_found`, `found` = false, and null coordinates." +); + +const REFINEMENT_TINY_TARGET_MAX_DIMENSION: f64 = 160.0; +const REFINEMENT_TINY_TARGET_MAX_AREA_FRACTION: f64 = 0.02; +const REFINEMENT_MIN_CROP_WIDTH: u32 = 360; +const REFINEMENT_MIN_CROP_HEIGHT: u32 = 320; +const REFINEMENT_DEFAULT_CANDIDATE_BOX: f64 = 24.0; +const REFINEMENT_MIN_LONGEST_EDGE: f64 = 1200.0; +const REFINEMENT_MAX_SCALE_FACTOR: f64 = 4.0; +const REFINEMENT_MAX_IMAGE_DIMENSION: f64 = 2000.0; +const MODEL_IMAGE_MAX_BYTES: usize = 4_718_592; +const MODEL_IMAGE_MAX_WIDTH: u32 = 2000; +const MODEL_IMAGE_MAX_HEIGHT: u32 = 2000; +const MODEL_IMAGE_DEFAULT_JPEG_QUALITY: u8 = 80; +const MODEL_IMAGE_JPEG_QUALITY_STEPS: [u8; 4] = [70, 55, 40, 25]; +const MODEL_IMAGE_SCALE_STEPS: [f64; 5] = [1.0, 0.75, 0.5, 0.35, 0.25]; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct GroundingValidationResponse { + status: String, + approved: bool, + confidence: Option, + reason: Option, + failure_kind: Option, + retry_hint: Option, +} + +#[derive(Clone, Debug)] +pub(super) struct RefinementCrop { + pub(super) image_bytes: Vec, + pub(super) offset_x: f64, + pub(super) offset_y: f64, + pub(super) crop_width: u32, + pub(super) crop_height: u32, + pub(super) model_width: u32, + pub(super) model_height: u32, + pub(super) model_scale_x: f64, + pub(super) model_scale_y: f64, +} + +#[derive(Clone, Debug)] +pub(super) struct PreparedGroundingImage { + pub(super) bytes: Vec, + pub(super) mime_type: &'static str, + pub(super) original_width: u32, + pub(super) original_height: u32, + pub(super) working_width: u32, + pub(super) working_height: u32, + pub(super) model_width: u32, + pub(super) model_height: u32, + pub(super) was_resized: bool, + pub(super) logical_normalization_applied: bool, + pub(super) working_to_original_scale_x: f64, + pub(super) working_to_original_scale_y: f64, + pub(super) model_to_original_scale_x: f64, + pub(super) model_to_original_scale_y: f64, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct GroundingModelImageConfig { + pub(super) logical_width: Option, + pub(super) logical_height: Option, + pub(super) scale_x: Option, + pub(super) scale_y: Option, + pub(super) allow_logical_normalization: bool, +} + +#[derive(Clone, Copy)] +struct ModelInputImage<'a> { + bytes: &'a [u8], + mime_type: &'a str, +} + +pub(super) fn gui_grounding_provider_name(invocation: &ToolInvocation) -> String { + format!( + "{}:{}", + invocation.turn.config.model_provider_id, invocation.turn.model_info.slug + ) +} + +pub(super) fn gui_grounding_output_schema() -> JsonValue { + serde_json::json!({ + "type": "object", + "properties": { + "status": { "type": "string" }, + "found": { "type": "boolean" }, + "confidence": { "type": ["number", "null"] }, + "reason": { "type": ["string", "null"] }, + "coordinate_space": { "type": ["string", "null"] }, + "click_point": { + "type": ["object", "null"], + "properties": { + "x": { "type": "number" }, + "y": { "type": "number" } + }, + "required": ["x", "y"], + "additionalProperties": false + }, + "bbox": { + "type": ["object", "null"], + "properties": { + "x1": { "type": "number" }, + "y1": { "type": "number" }, + "x2": { "type": "number" }, + "y2": { "type": "number" } + }, + "required": ["x1", "y1", "x2", "y2"], + "additionalProperties": false + } + }, + "required": [ + "status", + "found", + "confidence", + "reason", + "coordinate_space", + "click_point", + "bbox" + ], + "additionalProperties": false + }) +} + +fn gui_grounding_validation_output_schema() -> JsonValue { + serde_json::json!({ + "type": "object", + "properties": { + "status": { "type": "string" }, + "approved": { "type": "boolean" }, + "confidence": { "type": ["number", "null"] }, + "reason": { "type": ["string", "null"] }, + "failure_kind": { "type": ["string", "null"] }, + "retry_hint": { "type": ["string", "null"] } + }, + "required": [ + "status", + "approved", + "confidence", + "reason", + "failure_kind", + "retry_hint" + ], + "additionalProperties": false + }) +} + +fn format_grounding_action_intent(action: &str) -> &'static str { + match action { + "observe" => "identify the requested visible UI target for inspection", + "wait" => "identify whether the requested visible UI target is present", + "click" => "identify the best clickable hit target", + "type" => "identify the editable surface that should receive text", + "scroll" => "identify the target control or scrollable region", + "drag_source" => "identify the visible drag source surface", + "drag_destination" => "identify the visible drop destination surface", + _ => "identify the requested visible UI target", + } +} + +fn action_specific_grounding_instructions(action: &str) -> &'static [&'static str] { + match action { + "click" => &[ + "Resolve the actionable surface that visibly supports the requested click, not a broad region, container, badge strip, or generic panel background.", + "If you can identify only a broad region or container but not the actionable control itself, return `status` = `not_found` instead of guessing a background point.", + "For clicks, target the clickable surface near the control center unless the visible affordance suggests a safer interaction point.", + "For clicking, confirmation and primary-action buttons inside dialogs, sheets, popovers, drawers, side panels, and footers are valid targets even when the surrounding surface also contains labels or fields.", + "When the scope or location hint mentions a dialog, modal, sheet, drawer, panel, footer, or bottom-right region, prefer the visible actionable control in that region instead of nearby headings, copy, or input fields.", + "When the target names a visible button label inside a dialog, sheet, popover, drawer, panel, or footer, prefer the labeled button itself over nearby headings, copy, badges, or input fields.", + "For icon-only controls, match the visible symbol or glyph shape itself and return the individual icon-bearing control rather than the whole toolbar, icon row, or surrounding panel.", + "For dense toolbar or icon-row controls, use the visible symbol shape, local grouping, and neighboring control order together to distinguish adjacent buttons.", + ], + "type" => &[ + "For typing, prefer the editable surface itself instead of its surrounding label or container.", + "If the UI shows a collapsed control, search affordance, or icon-only affordance that would first reveal or focus the editable field, that visible control is also a valid target when it is the only actionable way to reach the field.", + ], + "scroll" => &[ + "For scrolling, target the visible scrollable region or the control that clearly owns scrolling, not a nearby heading or label.", + ], + "drag_source" | "drag_destination" => &[ + "For drag actions, visible rows, cards, list items, tree items, and labeled surfaces are valid targets even when there is no dedicated drag handle.", + "Prefer the actual draggable or droppable surface instead of a broad background region.", + ], + _ => &[], + } +} + +fn failure_kind_retry_guidance(failure_kind: &str) -> Option<&'static str> { + match failure_kind { + "wrong_region" | "scope_mismatch" => Some( + "Search a different visible area or panel instead of staying near the previous candidate.", + ), + "wrong_control" => Some( + "Keep the same semantic goal, but choose a different visible control that serves it.", + ), + "wrong_point" => Some( + "Stay on the matched control, but move the click point onto the inner actionable or editable surface.", + ), + "state_mismatch" => Some( + "Re-check the visible control state and pick the candidate whose current state best matches the request.", + ), + "partial_visibility" => Some( + "Only resolve a partially visible target when a safe interaction point is clearly visible inside the visible portion.", + ), + _ => None, + } +} + +pub(super) fn build_not_found_retry_notes( + request: GuiTargetRequest<'_>, + round: usize, +) -> Vec { + let mut notes = vec![format!( + "Round {round} returned not_found. Broaden the search while keeping the same semantic goal and visible scope." + )]; + notes.push( + "Match the target by visible meaning, label, icon, state, and nearby context together rather than requiring the exact control type named in the request." + .to_string(), + ); + notes.push( + "Equivalent visible controls may include buttons, icon buttons, toolbar items, tabs, rows, links, toggles, menu items, fields, search boxes, combo boxes, text areas, or editors when they clearly serve the same purpose." + .to_string(), + ); + notes.extend( + action_specific_grounding_instructions(request.action) + .iter() + .map(|instruction| instruction.to_string()), + ); + notes +} + +fn append_unique_retry_note(retry_notes: &mut Vec, note: String) { + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } +} + +fn append_unique_retry_notes(retry_notes: &mut Vec, notes: I) +where + I: IntoIterator, +{ + for note in notes { + append_unique_retry_note(retry_notes, note); + } +} + +pub(super) fn should_generate_retry_guide(failure_kind: Option<&str>) -> bool { + !matches!(failure_kind, Some("wrong_region" | "scope_mismatch")) +} + +fn gui_grounding_debug_enabled() -> bool { + std::env::var("CODEX_GUI_GROUNDING_DEBUG") + .map(|value| { + let normalized = value.trim().to_ascii_lowercase(); + normalized == "1" || normalized == "true" || normalized == "yes" || normalized == "on" + }) + .unwrap_or(false) +} + +fn emit_gui_grounding_debug( + request: GuiTargetRequest<'_>, + grounding_mode: &str, + round_artifacts: &[JsonValue], +) { + if !gui_grounding_debug_enabled() { + return; + } + let payload = serde_json::json!({ + "target": request.target, + "scope": request.scope, + "location_hint": request.location_hint, + "action": request.action, + "grounding_mode": grounding_mode, + "round_artifacts": round_artifacts, + }); + eprintln!( + "[codex-gui-grounding-debug] {}", + serde_json::to_string_pretty(&payload).unwrap_or_else(|_| { + "{\"error\":\"failed to serialize grounding debug payload\"}".to_string() + }) + ); +} + +pub(super) fn build_gui_grounding_prompt( + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + grounding_mode: &str, + retry_notes: &[String], + has_guide_image: bool, +) -> String { + let mut lines = vec![ + format!("action: {}", request.action), + format!( + "action_intent: {}", + format_grounding_action_intent(request.action) + ), + format!("grounding_mode: {grounding_mode}"), + format!("target: {}", request.target), + format!("capture_mode: {}", capture_state.capture.capture_mode), + format!( + "capture_size: {}x{}", + capture_state.capture.image_width, capture_state.capture.image_height + ), + ]; + if let Some(app) = request.app { + lines.push(format!("app: {app}")); + } + if let Some(window_title) = capture_state.capture.window_title.as_deref() { + lines.push(format!("window_title: {window_title}")); + } + if let Some(scope) = request.scope { + lines.push(format!("scope: {scope}")); + } + if let Some(location_hint) = request.location_hint { + lines.push(format!("location_hint: {location_hint}")); + } + if let Some(related_target) = request.related_target { + lines.push(format!("related_target: {related_target}")); + } + if let Some(related_scope) = request.related_scope { + lines.push(format!("related_scope: {related_scope}")); + } + if let Some(related_location_hint) = request.related_location_hint { + lines.push(format!("related_location_hint: {related_location_hint}")); + } + if let Some(related_point) = request.related_point { + lines.push(format!( + "related_point_display_pixels: ({}, {})", + related_point.x.round(), + related_point.y.round() + )); + } + lines.push("Ground the single best visible UI target in this screenshot.".to_string()); + lines.push( + "Use only visible screenshot evidence. Do not rely on hidden accessibility labels, DOM ids, or implementation names." + .to_string(), + ); + lines.push( + "Disambiguate similar controls using scope, coarse location, nearby visible text, local grouping, visible state, and relative order." + .to_string(), + ); + lines.push( + "Match subtle or weakly labeled controls by the visible label, symbol, indicator, shape, and surrounding context together." + .to_string(), + ); + lines.push( + "Match the target by visible meaning, label, icon or symbol, state, and surrounding context together; do not require the exact control type named in the request." + .to_string(), + ); + lines.push( + "The requested target may appear as a button, icon button, toolbar item, tab, row, link, toggle, menu item, field, search box, combo box, text area, or editor when those clearly fulfill the same visible intent." + .to_string(), + ); + lines.push( + "Choose the smallest obvious actionable or editable surface, and keep the click point on the visible hit target instead of whitespace, padding, decoration, or generic container background." + .to_string(), + ); + lines.push( + "The bbox must tightly cover the actionable or editable surface itself, not a larger container." + .to_string(), + ); + lines.push( + "When the request refers to nearby text, target the actual control or editable surface rather than the descriptive text alone." + .to_string(), + ); + lines.push( + "If the target has a visible state qualifier such as selected, checked, active, highlighted, or disabled, use that state to disambiguate among similar controls." + .to_string(), + ); + lines.push( + "If a matching control appears disabled or greyed-out and the request does not explicitly ask for a disabled control, prefer an enabled matching control if one exists." + .to_string(), + ); + lines.push( + "If the requested target is only partially visible at the edge of the screenshot, resolve it only when a safe interaction point is clearly visible inside the visible portion; otherwise return `status` = `not_found`." + .to_string(), + ); + lines.extend( + action_specific_grounding_instructions(request.action) + .iter() + .map(|instruction| instruction.to_string()), + ); + if !retry_notes.is_empty() { + lines.push("Retry context:".to_string()); + lines.extend(retry_notes.iter().map(|line| format!("- {line}"))); + } + if has_guide_image { + lines.push( + "An additional guide image is attached with a red overlay showing the previously rejected candidate." + .to_string(), + ); + lines.push( + "Do not repeat the red marked candidate unless the screenshot clearly contradicts the rejection." + .to_string(), + ); + } + lines.push( + "Return `status: \"resolved\"` only if the screenshot clearly shows the requested target." + .to_string(), + ); + lines.push( + "Return `status: \"not_found\"`, `found: false`, and null coordinates when the target is not confidently visible." + .to_string(), + ); + lines.push( + "Use `coordinate_space: \"image_pixels\"` and make `click_point` the best interaction point inside the visible target." + .to_string(), + ); + lines.push( + "Make `bbox` a tight visible box around the matched target in screenshot pixels." + .to_string(), + ); + lines.push("Respond with JSON only.".to_string()); + lines.join("\n") +} + +pub(super) fn build_gui_grounding_validation_prompt( + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + predicted_point: &HelperPoint, + predicted_bbox: Option<&GroundingBoundingBox>, + zoomed_crop_context: bool, +) -> String { + let mut lines = vec![ + format!("action: {}", request.action), + "grounding_mode: complex".to_string(), + format!("target: {}", request.target), + format!("capture_mode: {}", capture_state.capture.capture_mode), + format!( + "capture_size: {}x{}", + capture_state.capture.image_width, capture_state.capture.image_height + ), + format!( + "predicted_click_point_image_pixels: ({}, {})", + predicted_point.x.round(), + predicted_point.y.round() + ), + ]; + if let Some(bbox) = predicted_bbox { + lines.push(format!( + "predicted_bbox_image_pixels: ({}, {}, {}, {})", + bbox.x1.round(), + bbox.y1.round(), + bbox.x2.round(), + bbox.y2.round() + )); + } + if let Some(app) = request.app { + lines.push(format!("app: {app}")); + } + if let Some(window_title) = capture_state.capture.window_title.as_deref() { + lines.push(format!("window_title: {window_title}")); + } + if let Some(scope) = request.scope { + lines.push(format!("scope: {scope}")); + } + if let Some(location_hint) = request.location_hint { + lines.push(format!("location_hint: {location_hint}")); + } + if let Some(related_target) = request.related_target { + lines.push(format!("related_target: {related_target}")); + } + if let Some(related_scope) = request.related_scope { + lines.push(format!("related_scope: {related_scope}")); + } + if let Some(related_location_hint) = request.related_location_hint { + lines.push(format!("related_location_hint: {related_location_hint}")); + } + if zoomed_crop_context { + lines.push( + "The attached screenshot is a zoomed crop around the candidate from the original screenshot." + .to_string(), + ); + lines.push( + "Judge whether the marked control inside this crop is the correct target from the original request." + .to_string(), + ); + } + lines.push( + "The attached screenshot includes a red crosshair at the predicted click point and a red box around the predicted target." + .to_string(), + ); + lines.push( + "Return `status: \"approved\"` and `approved: true` only if that highlighted point is a correct interaction point for the requested target." + .to_string(), + ); + lines.push( + "Return `status: \"rejected\"` and `approved: false` if the highlight is incorrect, too ambiguous, or points to the wrong element." + .to_string(), + ); + lines.push( + "Reject if the highlighted action lands on whitespace, padding, decoration, generic container background, or on a neighboring control whose visible evidence does not match the request." + .to_string(), + ); + lines.push( + "If a scope or location hint was provided, the candidate must be inside that scope or region; distinguish matching controls in different panels, rows, toolbars, dialogs, or footers by scope." + .to_string(), + ); + lines.push( + "For subtle, tightly packed, or low-contrast controls, approve only when the marked point sits on the visible hit target itself. Minor offset inside the visible hit area is acceptable only when the control is still clearly the intended one." + .to_string(), + ); + lines.push( + "For weakly worded, paraphrased, or ambiguous requests, approve the marked control when it is the strongest visible semantic match among nearby candidates, even if the iconography is stylized rather than literally labeled." + .to_string(), + ); + lines.push( + "When rejecting, use `failure_kind` to classify the issue as `wrong_region`, `scope_mismatch`, `wrong_control`, `wrong_point`, `state_mismatch`, `partial_visibility`, or `other`." + .to_string(), + ); + lines.push( + "When rejecting, provide a short `retry_hint` describing how the next grounding round should search differently." + .to_string(), + ); + lines.push( + "Keep `reason` terse, at most 10 words. Keep `retry_hint` terse, at most 18 words." + .to_string(), + ); + lines.push("Respond with JSON only.".to_string()); + lines.join("\n") +} + +pub(super) fn build_gui_grounding_refinement_prompt( + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + crop: &RefinementCrop, + prior_point: Option<&HelperPoint>, + prior_bbox: Option<&GroundingBoundingBox>, +) -> String { + let mut retry_notes = vec![ + "The provided screenshot is a zoomed crop around a previous candidate from the original image.".to_string(), + ]; + if let Some(prior_point) = prior_point { + retry_notes.push(format!( + "Previous crop-relative point: ({}, {}).", + prior_point.x.round(), + prior_point.y.round() + )); + } + if let Some(prior_bbox) = prior_bbox { + retry_notes.push(format!( + "Previous crop-relative box: ({}, {}, {}, {}).", + prior_bbox.x1.round(), + prior_bbox.y1.round(), + prior_bbox.x2.round(), + prior_bbox.y2.round() + )); + } + retry_notes.push( + "Refine the target inside this crop; if the crop does not actually contain the target, return not_found." + .to_string(), + ); + build_gui_grounding_prompt( + GuiTargetRequest { + target: request.target, + scope: request.scope, + app: request.app, + location_hint: request.location_hint, + window_selection: request.window_selection, + grounding_mode: Some("single"), + action: request.action, + capture_mode: request.capture_mode, + related_target: request.related_target, + related_scope: request.related_scope, + related_location_hint: request.related_location_hint, + related_point: None, + }, + &ObserveState { + capture: CaptureArtifact { + origin_x: crop.offset_x, + origin_y: crop.offset_y, + width: crop.crop_width, + height: crop.crop_height, + image_width: crop.model_width, + image_height: crop.model_height, + display_index: capture_state.capture.display_index, + capture_mode: capture_state.capture.capture_mode, + window_title: capture_state.capture.window_title.clone(), + window_count: capture_state.capture.window_count, + window_capture_strategy: capture_state.capture.window_capture_strategy.clone(), + host_exclusion: capture_state.capture.host_exclusion.clone(), + }, + app_name: capture_state.app_name.clone(), + }, + "single", + &retry_notes, + /*has_guide_image*/ false, + ) +} + +pub(super) fn extract_grounding_json(text: &str) -> Option<&str> { + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + if trimmed.starts_with('{') && trimmed.ends_with('}') { + return Some(trimmed); + } + let start = trimmed.find('{')?; + let end = trimmed.rfind('}')?; + (end > start).then_some(&trimmed[start..=end]) +} + +pub(super) fn summarize_grounding_response(text: &str) -> String { + let trimmed = text.trim(); + if trimmed.chars().count() <= 240 { + trimmed.to_string() + } else { + format!("{}...", trimmed.chars().take(240).collect::()) + } +} + +pub(super) async fn collect_model_output_text( + mut stream: crate::ResponseStream, +) -> Result { + let mut result = String::new(); + while let Some(event) = stream.next().await.transpose().map_err(|error| { + FunctionCallError::RespondToModel(format!("GUI grounding stream failed: {error}")) + })? { + match event { + ResponseEvent::OutputTextDelta(delta) => result.push_str(&delta), + ResponseEvent::OutputItemDone(item) => { + if result.is_empty() + && let ResponseItem::Message { content, .. } = item + && let Some(text) = crate::compact::content_items_to_text(&content) + { + result.push_str(&text); + } + } + ResponseEvent::Completed { .. } => break, + _ => {} + } + } + Ok(result) +} + +async fn request_model_json( + invocation: &ToolInvocation, + prompt_text: String, + image: ModelInputImage<'_>, + system_prompt: &str, + output_schema: JsonValue, + request_label: &str, +) -> Result<(T, JsonValue, String), FunctionCallError> +where + T: DeserializeOwned, +{ + request_model_json_with_images( + invocation, + prompt_text, + &[image], + system_prompt, + output_schema, + request_label, + ) + .await +} + +#[cfg(test)] +pub(super) async fn request_model_json_from_image( + invocation: &ToolInvocation, + prompt_text: String, + image_bytes: &[u8], + mime_type: &str, + system_prompt: &str, + output_schema: JsonValue, + request_label: &str, +) -> Result<(T, JsonValue, String), FunctionCallError> +where + T: DeserializeOwned, +{ + request_model_json( + invocation, + prompt_text, + ModelInputImage { + bytes: image_bytes, + mime_type, + }, + system_prompt, + output_schema, + request_label, + ) + .await +} + +async fn request_model_json_with_images( + invocation: &ToolInvocation, + prompt_text: String, + images: &[ModelInputImage<'_>], + system_prompt: &str, + output_schema: JsonValue, + request_label: &str, +) -> Result<(T, JsonValue, String), FunctionCallError> +where + T: DeserializeOwned, +{ + let mut content = vec![ContentItem::InputText { text: prompt_text }]; + content.extend(images.iter().map(|image| ContentItem::InputImage { + image_url: data_url(image.bytes, image.mime_type), + })); + let prompt = Prompt { + input: vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content, + end_turn: None, + phase: None, + }], + tools: Vec::new(), + parallel_tool_calls: false, + base_instructions: BaseInstructions { + text: system_prompt.to_string(), + }, + personality: None, + output_schema: Some(output_schema), + }; + let turn_metadata_header = invocation.turn.turn_metadata_state.current_header_value(); + let response_text = { + let mut last_error: Option = None; + let mut text: Option = None; + for attempt in 1..=3 { + let mut client_session = invocation.session.services.model_client.new_session(); + let stream = client_session + .stream( + &prompt, + &invocation.turn.model_info, + &invocation.turn.session_telemetry, + invocation.turn.reasoning_effort, + invocation.turn.reasoning_summary, + invocation.turn.config.service_tier, + turn_metadata_header.as_deref(), + ) + .await; + match stream { + Ok(stream) => match collect_model_output_text(stream).await { + Ok(response_text) => { + text = Some(response_text); + break; + } + Err(error) => { + last_error = Some(error.to_string()); + } + }, + Err(error) => { + last_error = Some(format!("{request_label} request failed: {error}")); + } + } + if attempt < 3 { + tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await; + } + } + text.ok_or_else(|| { + FunctionCallError::RespondToModel(last_error.unwrap_or_else(|| { + format!("{request_label} request failed without an error message") + })) + })? + }; + let json_payload = extract_grounding_json(&response_text).ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "{request_label} response was empty or not JSON: {}", + summarize_grounding_response(&response_text) + )) + })?; + let raw: JsonValue = serde_json::from_str(json_payload).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "{request_label} response was invalid JSON: {error}. Raw: {}", + summarize_grounding_response(&response_text) + )) + })?; + let parsed: T = serde_json::from_value(raw.clone()).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "{request_label} response did not match the expected schema: {error}" + )) + })?; + Ok((parsed, raw, response_text)) +} + +fn draw_marker_pixel(image: &mut image::RgbaImage, x: i32, y: i32, color: Rgba) { + if x >= 0 && y >= 0 && x < image.width() as i32 && y < image.height() as i32 { + image.put_pixel(x as u32, y as u32, color); + } +} + +fn draw_rect_outline(image: &mut image::RgbaImage, rect: &HelperRect, color: Rgba) { + let left = rect.x.floor() as i32; + let top = rect.y.floor() as i32; + let right = (rect.x + rect.width - 1.0).ceil() as i32; + let bottom = (rect.y + rect.height - 1.0).ceil() as i32; + for x in left..=right { + draw_marker_pixel(image, x, top, color); + draw_marker_pixel(image, x, bottom, color); + } + for y in top..=bottom { + draw_marker_pixel(image, left, y, color); + draw_marker_pixel(image, right, y, color); + } +} + +fn draw_crosshair(image: &mut image::RgbaImage, point: &HelperPoint, color: Rgba) { + let x = point.x.round() as i32; + let y = point.y.round() as i32; + for offset in -12..=12 { + draw_marker_pixel(image, x + offset, y, color); + draw_marker_pixel(image, x, y + offset, color); + } + for offset in -3..=3 { + draw_marker_pixel(image, x + offset, y + offset, Rgba([255, 255, 255, 255])); + draw_marker_pixel(image, x + offset, y - offset, Rgba([255, 255, 255, 255])); + } +} + +pub(super) fn render_validation_overlay( + image_bytes: &[u8], + predicted_point: &HelperPoint, + predicted_bbox: Option<&GroundingBoundingBox>, +) -> Result, FunctionCallError> { + let mut image = image::load_from_memory(image_bytes) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode GUI grounding image for validation: {error}" + )) + })? + .into_rgba8(); + let accent = Rgba([255, 48, 48, 255]); + if let Some(bbox) = predicted_bbox.and_then(grounding_bbox_to_rect) { + draw_rect_outline(&mut image, &bbox, accent); + } + draw_crosshair(&mut image, predicted_point, accent); + let mut encoded = Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut encoded, ImageFormat::Png) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode GUI grounding validation image: {error}" + )) + })?; + Ok(encoded.into_inner()) +} + +pub(super) fn render_guide_overlay( + image_bytes: &[u8], + rejected_point: &HelperPoint, + rejected_bbox: Option<&GroundingBoundingBox>, +) -> Result, FunctionCallError> { + render_validation_overlay(image_bytes, rejected_point, rejected_bbox) +} + +pub(super) fn annotate_grounding_raw( + raw: &mut JsonValue, + grounding_mode: &str, + selected_attempt: &str, + validation_triggered: bool, + validation_status: &str, + validation_reason: Option<&str>, + validation_confidence: Option, + rounds_attempted: i64, +) { + if let JsonValue::Object(fields) = raw { + fields.insert( + "grounding_mode_requested".to_string(), + JsonValue::String(grounding_mode.to_string()), + ); + fields.insert( + "grounding_mode_effective".to_string(), + JsonValue::String(grounding_mode.to_string()), + ); + fields.insert( + "selected_attempt".to_string(), + JsonValue::String(selected_attempt.to_string()), + ); + fields.insert( + "grounding_validation_triggered".to_string(), + JsonValue::Bool(validation_triggered), + ); + fields.insert( + "grounding_rounds_attempted".to_string(), + JsonValue::Number(rounds_attempted.into()), + ); + let mut validation = serde_json::Map::new(); + validation.insert( + "status".to_string(), + JsonValue::String(validation_status.to_string()), + ); + validation.insert( + "reason".to_string(), + validation_reason + .map(|reason| JsonValue::String(reason.to_string())) + .unwrap_or(JsonValue::Null), + ); + validation.insert( + "confidence".to_string(), + validation_confidence + .and_then(serde_json::Number::from_f64) + .map(JsonValue::Number) + .unwrap_or(JsonValue::Null), + ); + fields.insert("validation".to_string(), JsonValue::Object(validation)); + } +} + +pub(super) fn annotate_grounding_round_artifacts( + raw: &mut JsonValue, + round_artifacts: &[JsonValue], +) { + if let JsonValue::Object(fields) = raw { + fields.insert( + "grounding_round_artifacts".to_string(), + JsonValue::Array(round_artifacts.to_vec()), + ); + } +} + +fn annotate_prepared_grounding_image_raw(raw: &mut JsonValue, prepared: &PreparedGroundingImage) { + if let JsonValue::Object(fields) = raw { + fields.insert( + "grounding_model_image".to_string(), + prepared_grounding_image_metadata(prepared), + ); + } +} + +fn prepared_grounding_image_metadata(prepared: &PreparedGroundingImage) -> JsonValue { + serde_json::json!({ + "mime_type": prepared.mime_type, + "original_width": prepared.original_width, + "original_height": prepared.original_height, + "working_width": prepared.working_width, + "working_height": prepared.working_height, + "model_width": prepared.model_width, + "model_height": prepared.model_height, + "was_resized": prepared.was_resized, + "logical_normalization_applied": prepared.logical_normalization_applied, + "working_to_original_scale_x": prepared.working_to_original_scale_x, + "working_to_original_scale_y": prepared.working_to_original_scale_y, + "model_to_original_scale_x": prepared.model_to_original_scale_x, + "model_to_original_scale_y": prepared.model_to_original_scale_y, + "byte_length": prepared.bytes.len(), + }) +} + +fn refinement_crop_metadata(crop: &RefinementCrop) -> JsonValue { + serde_json::json!({ + "crop": { + "x": crop.offset_x, + "y": crop.offset_y, + "width": crop.crop_width, + "height": crop.crop_height, + }, + "model_image": { + "width": crop.model_width, + "height": crop.model_height, + "scale_x": crop.model_scale_x, + "scale_y": crop.model_scale_y, + "byte_length": crop.image_bytes.len(), + "mime_type": "image/png", + } + }) +} + +fn predictor_outcome_metadata( + decision: &GroundingModelResponse, + point: Option<&HelperPoint>, + bbox: Option<&GroundingBoundingBox>, +) -> JsonValue { + serde_json::json!({ + "status": decision.status, + "found": decision.found, + "confidence": decision.confidence, + "reason": decision.reason, + "coordinate_space": decision.coordinate_space, + "point": point.map(|point| serde_json::json!({ "x": point.x, "y": point.y })), + "bbox": bbox.map(|bbox| serde_json::json!({ + "x1": bbox.x1, + "y1": bbox.y1, + "x2": bbox.x2, + "y2": bbox.y2, + })), + }) +} + +fn normalize_preferred_dimension( + preferred_dimension: Option, + original_dimension: u32, + scale_hint: Option, +) -> Option { + if let Some(preferred_dimension) = preferred_dimension.filter(|value| *value > 0) { + return Some(preferred_dimension.min(original_dimension).max(1)); + } + if let Some(scale_hint) = scale_hint.filter(|value| value.is_finite() && *value > 1.0) { + return Some( + ((original_dimension as f64) / scale_hint) + .round() + .clamp(1.0, original_dimension as f64) as u32, + ); + } + None +} + +fn resolve_working_dimensions( + original_width: u32, + original_height: u32, + config: GroundingModelImageConfig, +) -> (u32, u32, bool) { + if !config.allow_logical_normalization || original_width == 0 || original_height == 0 { + return (original_width, original_height, false); + } + let original_aspect = original_width as f64 / original_height as f64; + let mut preferred_width = + normalize_preferred_dimension(config.logical_width, original_width, config.scale_x); + let mut preferred_height = + normalize_preferred_dimension(config.logical_height, original_height, config.scale_y); + + if preferred_width.is_some() && preferred_height.is_none() { + preferred_height = Some( + ((preferred_width.unwrap() as f64) / original_aspect) + .round() + .clamp(1.0, original_height as f64) as u32, + ); + } + if preferred_height.is_some() && preferred_width.is_none() { + preferred_width = Some( + ((preferred_height.unwrap() as f64) * original_aspect) + .round() + .clamp(1.0, original_width as f64) as u32, + ); + } + if let (Some(width), Some(height)) = (preferred_width, preferred_height) { + let preferred_aspect = width as f64 / height as f64; + if (preferred_aspect - original_aspect).abs() > 0.02 { + let height_from_width = ((width as f64) / original_aspect) + .round() + .clamp(1.0, original_height as f64) as u32; + let width_from_height = ((height as f64) * original_aspect) + .round() + .clamp(1.0, original_width as f64) as u32; + if (height_from_width as i64 - height as i64).abs() + <= (width_from_height as i64 - width as i64).abs() + { + preferred_height = Some(height_from_width); + } else { + preferred_width = Some(width_from_height); + } + } + } + + let working_width = preferred_width.unwrap_or(original_width); + let working_height = preferred_height.unwrap_or(original_height); + ( + working_width, + working_height, + working_width != original_width || working_height != original_height, + ) +} + +fn constrain_model_dimensions(width: u32, height: u32) -> (u32, u32) { + if width == 0 || height == 0 { + return (width, height); + } + let scale = (MODEL_IMAGE_MAX_WIDTH as f64 / width as f64) + .min(MODEL_IMAGE_MAX_HEIGHT as f64 / height as f64) + .min(1.0); + let w = ((width as f64 * scale).round() as u32).max(1); + let h = ((height as f64 * scale).round() as u32).max(1); + (w, h) +} + +fn encode_image_to_jpeg( + image: &DynamicImage, + quality: u8, + label: &str, +) -> Result, FunctionCallError> { + let mut encoded = Cursor::new(Vec::new()); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, quality); + encoder.encode_image(image).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode GUI grounding {label} as jpeg: {error}" + )) + })?; + Ok(encoded.into_inner()) +} + +fn resize_exact_if_needed(image: &DynamicImage, width: u32, height: u32) -> DynamicImage { + if image.width() == width && image.height() == height { + image.clone() + } else { + image.resize_exact(width, height, image::imageops::FilterType::Lanczos3) + } +} + +pub(super) fn prepare_grounding_model_image( + image_bytes: &[u8], + config: GroundingModelImageConfig, + label: &str, +) -> Result { + let original = image::load_from_memory(image_bytes).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode GUI grounding {label}: {error}" + )) + })?; + let original_width = original.width(); + let original_height = original.height(); + if original_width == 0 || original_height == 0 { + return Err(FunctionCallError::RespondToModel(format!( + "GUI grounding {label} had empty dimensions" + ))); + } + + let (working_width, working_height, logical_normalization_applied) = + resolve_working_dimensions(original_width, original_height, config); + let working = resize_exact_if_needed(&original, working_width, working_height); + let (initial_model_width, initial_model_height) = + constrain_model_dimensions(working_width, working_height); + + let evaluate_candidate = |width: u32, + height: u32, + jpeg_quality: u8| + -> Result, &'static str)>, FunctionCallError> { + if width == 0 || height == 0 { + return Ok(None); + } + let candidate = resize_exact_if_needed(&working, width, height); + let png = encode_image_to_png(&candidate, label)?; + let jpeg = encode_image_to_jpeg(&candidate, jpeg_quality, label)?; + let preferred = if png.len() <= MODEL_IMAGE_MAX_BYTES { + (png, "image/png") + } else if jpeg.len() <= MODEL_IMAGE_MAX_BYTES { + (jpeg, "image/jpeg") + } else if png.len() <= jpeg.len() { + (png, "image/png") + } else { + (jpeg, "image/jpeg") + }; + Ok(Some(preferred)) + }; + + let (mut best_bytes, mut best_mime_type) = evaluate_candidate( + initial_model_width, + initial_model_height, + MODEL_IMAGE_DEFAULT_JPEG_QUALITY, + )? + .ok_or_else(|| { + FunctionCallError::RespondToModel( + "GUI grounding image preparation produced an empty candidate".to_string(), + ) + })?; + let mut final_width = initial_model_width; + let mut final_height = initial_model_height; + let mut found_within_limit = best_bytes.len() <= MODEL_IMAGE_MAX_BYTES; + + for scale in MODEL_IMAGE_SCALE_STEPS { + let candidate_width = ((initial_model_width as f64) * scale) + .round() + .clamp(1.0, initial_model_width as f64) as u32; + let candidate_height = ((initial_model_height as f64) * scale) + .round() + .clamp(1.0, initial_model_height as f64) as u32; + if candidate_width < 100 || candidate_height < 100 { + break; + } + for quality in MODEL_IMAGE_JPEG_QUALITY_STEPS { + if let Some((candidate_bytes, candidate_mime_type)) = + evaluate_candidate(candidate_width, candidate_height, quality)? + { + let should_replace = if found_within_limit { + candidate_bytes.len() <= MODEL_IMAGE_MAX_BYTES + && candidate_bytes.len() < best_bytes.len() + } else if candidate_bytes.len() <= MODEL_IMAGE_MAX_BYTES { + found_within_limit = true; + true + } else { + candidate_bytes.len() < best_bytes.len() + }; + if should_replace { + best_bytes = candidate_bytes; + best_mime_type = candidate_mime_type; + final_width = candidate_width; + final_height = candidate_height; + } + } + if found_within_limit && best_bytes.len() <= MODEL_IMAGE_MAX_BYTES { + break; + } + } + if found_within_limit && best_bytes.len() <= MODEL_IMAGE_MAX_BYTES { + break; + } + } + + let working_to_original_scale_x = if working_width > 0 { + original_width as f64 / working_width as f64 + } else { + 1.0 + }; + let working_to_original_scale_y = if working_height > 0 { + original_height as f64 / working_height as f64 + } else { + 1.0 + }; + let model_to_working_scale_x = if final_width > 0 { + working_width as f64 / final_width as f64 + } else { + 1.0 + }; + let model_to_working_scale_y = if final_height > 0 { + working_height as f64 / final_height as f64 + } else { + 1.0 + }; + + Ok(PreparedGroundingImage { + bytes: best_bytes, + mime_type: best_mime_type, + original_width, + original_height, + working_width, + working_height, + model_width: final_width, + model_height: final_height, + was_resized: logical_normalization_applied + || final_width != original_width + || final_height != original_height, + logical_normalization_applied, + working_to_original_scale_x, + working_to_original_scale_y, + model_to_original_scale_x: model_to_working_scale_x * working_to_original_scale_x, + model_to_original_scale_y: model_to_working_scale_y * working_to_original_scale_y, + }) +} + +pub(super) fn grounding_bbox_to_rect(bbox: &GroundingBoundingBox) -> Option { + let width = bbox.x2 - bbox.x1; + let height = bbox.y2 - bbox.y1; + if !bbox.x1.is_finite() + || !bbox.y1.is_finite() + || !bbox.x2.is_finite() + || !bbox.y2.is_finite() + || width <= 0.0 + || height <= 0.0 + { + return None; + } + Some(HelperRect { + x: bbox.x1, + y: bbox.y1, + width, + height, + }) +} + +fn fallback_candidate_rect(point: &HelperPoint) -> HelperRect { + HelperRect { + x: (point.x - (REFINEMENT_DEFAULT_CANDIDATE_BOX / 2.0)).max(0.0), + y: (point.y - (REFINEMENT_DEFAULT_CANDIDATE_BOX / 2.0)).max(0.0), + width: REFINEMENT_DEFAULT_CANDIDATE_BOX, + height: REFINEMENT_DEFAULT_CANDIDATE_BOX, + } +} + +pub(super) fn should_use_high_resolution_refinement( + capture_state: &ObserveState, + point: &HelperPoint, + bbox: Option<&GroundingBoundingBox>, +) -> bool { + let candidate_box = bbox + .and_then(grounding_bbox_to_rect) + .unwrap_or_else(|| fallback_candidate_rect(point)); + let max_dimension = candidate_box.width.max(candidate_box.height); + let image_area = + (capture_state.capture.image_width as f64) * (capture_state.capture.image_height as f64); + let box_area = candidate_box.width * candidate_box.height; + max_dimension <= REFINEMENT_TINY_TARGET_MAX_DIMENSION + || (image_area > 0.0 && (box_area / image_area) <= REFINEMENT_TINY_TARGET_MAX_AREA_FRACTION) +} + +fn encode_image_to_png( + image: &image::DynamicImage, + label: &str, +) -> Result, FunctionCallError> { + let mut encoded = Cursor::new(Vec::new()); + image + .write_to(&mut encoded, ImageFormat::Png) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode GUI grounding {label}: {error}" + )) + })?; + Ok(encoded.into_inner()) +} + +pub(super) fn create_refinement_crop( + image_bytes: &[u8], + point: &HelperPoint, + bbox: Option<&GroundingBoundingBox>, +) -> Result, FunctionCallError> { + let image = image::load_from_memory(image_bytes).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode GUI grounding image for refinement: {error}" + )) + })?; + let (image_width, image_height) = image.dimensions(); + if image_width == 0 || image_height == 0 { + return Ok(None); + } + + let candidate_box = bbox + .and_then(grounding_bbox_to_rect) + .unwrap_or_else(|| fallback_candidate_rect(point)); + let crop_width = ((candidate_box.width * 5.0) + .round() + .max((candidate_box.height * 6.0).round()) + .max(REFINEMENT_MIN_CROP_WIDTH as f64)) + .min(image_width as f64) as u32; + let crop_height = ((candidate_box.height * 5.0) + .round() + .max((candidate_box.width * 4.0).round()) + .max(REFINEMENT_MIN_CROP_HEIGHT as f64)) + .min(image_height as f64) as u32; + if crop_width == 0 || crop_height == 0 { + return Ok(None); + } + + let center_x = point.x.round(); + let center_y = point.y.round(); + let left = (center_x - (crop_width as f64 / 2.0)) + .clamp(0.0, (image_width.saturating_sub(crop_width)) as f64) + .round() as u32; + let top = (center_y - (crop_height as f64 / 2.0)) + .clamp(0.0, (image_height.saturating_sub(crop_height)) as f64) + .round() as u32; + + let cropped = image.crop_imm(left, top, crop_width, crop_height); + let longest_edge = crop_width.max(crop_height) as f64; + let desired_scale = if longest_edge > 0.0 && longest_edge < REFINEMENT_MIN_LONGEST_EDGE { + (REFINEMENT_MIN_LONGEST_EDGE / longest_edge) + .min(REFINEMENT_MAX_SCALE_FACTOR) + .min(REFINEMENT_MAX_IMAGE_DIMENSION / longest_edge) + } else { + 1.0 + }; + let model_width = ((crop_width as f64) * desired_scale) + .round() + .clamp(1.0, REFINEMENT_MAX_IMAGE_DIMENSION) as u32; + let model_height = ((crop_height as f64) * desired_scale) + .round() + .clamp(1.0, REFINEMENT_MAX_IMAGE_DIMENSION) as u32; + let prepared = if model_width != crop_width || model_height != crop_height { + cropped.resize_exact( + model_width, + model_height, + image::imageops::FilterType::Lanczos3, + ) + } else { + cropped + }; + let image_bytes = encode_image_to_png(&prepared, "refinement image")?; + + Ok(Some(RefinementCrop { + image_bytes, + offset_x: left as f64, + offset_y: top as f64, + crop_width, + crop_height, + model_width, + model_height, + model_scale_x: model_width as f64 / crop_width as f64, + model_scale_y: model_height as f64 / crop_height as f64, + })) +} + +pub(super) fn translate_refinement_point_to_original( + crop: &RefinementCrop, + point: &HelperPoint, +) -> HelperPoint { + HelperPoint { + x: crop.offset_x + (point.x / crop.model_scale_x), + y: crop.offset_y + (point.y / crop.model_scale_y), + } +} + +fn translate_refinement_bbox_to_original( + crop: &RefinementCrop, + bbox: &GroundingBoundingBox, +) -> GroundingBoundingBox { + GroundingBoundingBox { + x1: crop.offset_x + (bbox.x1 / crop.model_scale_x), + y1: crop.offset_y + (bbox.y1 / crop.model_scale_y), + x2: crop.offset_x + (bbox.x2 / crop.model_scale_x), + y2: crop.offset_y + (bbox.y2 / crop.model_scale_y), + } +} + +pub(super) fn translate_model_point_to_original( + prepared: &PreparedGroundingImage, + point: &HelperPoint, +) -> HelperPoint { + HelperPoint { + x: point.x * prepared.model_to_original_scale_x, + y: point.y * prepared.model_to_original_scale_y, + } +} + +fn translate_model_bbox_to_original( + prepared: &PreparedGroundingImage, + bbox: &GroundingBoundingBox, +) -> GroundingBoundingBox { + GroundingBoundingBox { + x1: bbox.x1 * prepared.model_to_original_scale_x, + y1: bbox.y1 * prepared.model_to_original_scale_y, + x2: bbox.x2 * prepared.model_to_original_scale_x, + y2: bbox.y2 * prepared.model_to_original_scale_y, + } +} + +pub(super) fn translate_original_point_to_refinement( + crop: &RefinementCrop, + point: &HelperPoint, +) -> HelperPoint { + HelperPoint { + x: (point.x - crop.offset_x) * crop.model_scale_x, + y: (point.y - crop.offset_y) * crop.model_scale_y, + } +} + +fn translate_original_bbox_to_refinement( + crop: &RefinementCrop, + bbox: &GroundingBoundingBox, +) -> GroundingBoundingBox { + GroundingBoundingBox { + x1: (bbox.x1 - crop.offset_x) * crop.model_scale_x, + y1: (bbox.y1 - crop.offset_y) * crop.model_scale_y, + x2: (bbox.x2 - crop.offset_x) * crop.model_scale_x, + y2: (bbox.y2 - crop.offset_y) * crop.model_scale_y, + } +} + +pub(super) fn translate_original_point_to_model( + prepared: &PreparedGroundingImage, + point: &HelperPoint, +) -> HelperPoint { + HelperPoint { + x: point.x / prepared.model_to_original_scale_x, + y: point.y / prepared.model_to_original_scale_y, + } +} + +fn translate_original_bbox_to_model( + prepared: &PreparedGroundingImage, + bbox: &GroundingBoundingBox, +) -> GroundingBoundingBox { + GroundingBoundingBox { + x1: bbox.x1 / prepared.model_to_original_scale_x, + y1: bbox.y1 / prepared.model_to_original_scale_y, + x2: bbox.x2 / prepared.model_to_original_scale_x, + y2: bbox.y2 / prepared.model_to_original_scale_y, + } +} + +pub(super) fn image_point_to_display(state: &ObserveState, point: &HelperPoint) -> HelperPoint { + let scale_x = state.capture.scale_x(); + let scale_y = state.capture.scale_y(); + HelperPoint { + x: state.capture.origin_x + (point.x / scale_x), + y: state.capture.origin_y + (point.y / scale_y), + } +} + +pub(super) fn image_rect_to_display(state: &ObserveState, rect: &HelperRect) -> HelperRect { + let scale_x = state.capture.scale_x(); + let scale_y = state.capture.scale_y(); + HelperRect { + x: state.capture.origin_x + (rect.x / scale_x), + y: state.capture.origin_y + (rect.y / scale_y), + width: rect.width / scale_x, + height: rect.height / scale_y, + } +} + +pub(super) async fn resolve_grounded_target( + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + image_bytes: &[u8], +) -> Result, FunctionCallError> { + let grounding_mode = normalize_grounding_mode(request.grounding_mode, request.action)?; + let max_rounds = if grounding_mode == "complex" { 3 } else { 2 }; + let prepared_grounding_image = prepare_grounding_model_image( + image_bytes, + GroundingModelImageConfig { + logical_width: Some(capture_state.capture.width), + logical_height: Some(capture_state.capture.height), + scale_x: Some(capture_state.capture.scale_x()), + scale_y: Some(capture_state.capture.scale_y()), + allow_logical_normalization: true, + }, + "grounding image", + )?; + let mut model_capture_state = capture_state.clone(); + model_capture_state.capture.image_width = prepared_grounding_image.model_width; + model_capture_state.capture.image_height = prepared_grounding_image.model_height; + let mut retry_notes: Vec = Vec::new(); + let mut guide_image: Option> = None; + let mut round_artifacts: Vec = Vec::new(); + let mut selected_round = 1_i64; + let mut final_decision: Option = None; + let mut final_raw: Option = None; + let mut final_validation: Option<(GroundingValidationResponse, JsonValue, String, usize)> = + None; + + for round in 1..=max_rounds { + let mut validation_zoom_context = false; + let mut validation_image_bytes_override: Option> = None; + let mut validation_state_override: Option = None; + let mut validation_point_override: Option = None; + let mut validation_bbox_override: Option = None; + let mut grounding_images = vec![ModelInputImage { + bytes: &prepared_grounding_image.bytes, + mime_type: prepared_grounding_image.mime_type, + }]; + if let Some(guide_image_bytes) = guide_image.as_deref() { + grounding_images.push(ModelInputImage { + bytes: guide_image_bytes, + mime_type: "image/png", + }); + } + let (decision, raw, _) = request_model_json_with_images::( + invocation, + build_gui_grounding_prompt( + request, + &model_capture_state, + grounding_mode, + &retry_notes, + guide_image.is_some(), + ), + &grounding_images, + GUI_GROUNDING_SYSTEM_PROMPT, + gui_grounding_output_schema(), + "GUI grounding", + ) + .await?; + let mut round_artifact = serde_json::json!({ + "round": round, + "guide_image_attached": guide_image.is_some(), + "model_image": prepared_grounding_image_metadata(&prepared_grounding_image), + "retry_notes_before_round": retry_notes.clone(), + "predictor": { + "raw": raw.clone(), + "outcome": predictor_outcome_metadata(&decision, decision.click_point.as_ref(), decision.bbox.as_ref()), + }, + }); + if decision.status == "not_found" || !decision.found { + if let Some(reason) = decision.reason.as_deref() { + append_unique_retry_note( + &mut retry_notes, + format!("Round {round} predictor rationale: {reason}"), + ); + } + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "terminal_state".to_string(), + JsonValue::String(if round < max_rounds { + "predictor_not_found_retry".to_string() + } else { + "predictor_not_found_final".to_string() + }), + ); + fields.insert( + "retry_notes_after_round".to_string(), + JsonValue::Array(retry_notes.iter().cloned().map(JsonValue::String).collect()), + ); + } + round_artifacts.push(round_artifact); + if round < max_rounds { + append_unique_retry_notes( + &mut retry_notes, + build_not_found_retry_notes(request, round), + ); + guide_image = None; + continue; + } + emit_gui_grounding_debug(request, grounding_mode, &round_artifacts); + return Ok(None); + } + if decision.status != "resolved" { + return Err(FunctionCallError::RespondToModel(format!( + "GUI grounding returned unsupported status `{}`", + decision.status + ))); + } + let coordinate_space = decision + .coordinate_space + .as_deref() + .unwrap_or("image_pixels"); + if coordinate_space != "image_pixels" { + return Err(FunctionCallError::RespondToModel(format!( + "GUI grounding returned unsupported coordinate space `{coordinate_space}`" + ))); + } + let model_image_point = decision.click_point.clone().ok_or_else(|| { + FunctionCallError::RespondToModel( + "GUI grounding resolved a target without a click_point".to_string(), + ) + })?; + let mut candidate_decision = decision.clone(); + let mut candidate_raw = raw.clone(); + let image_point = + translate_model_point_to_original(&prepared_grounding_image, &model_image_point); + candidate_decision.click_point = Some(image_point.clone()); + candidate_decision.bbox = decision + .bbox + .as_ref() + .map(|bbox| translate_model_bbox_to_original(&prepared_grounding_image, bbox)); + annotate_prepared_grounding_image_raw(&mut candidate_raw, &prepared_grounding_image); + if grounding_mode == "complex" + && should_use_high_resolution_refinement( + capture_state, + &image_point, + candidate_decision.bbox.as_ref(), + ) + && let Some(refinement_crop) = + create_refinement_crop(image_bytes, &image_point, candidate_decision.bbox.as_ref())? + { + let prior_point = + translate_original_point_to_refinement(&refinement_crop, &image_point); + let prior_bbox = candidate_decision + .bbox + .as_ref() + .map(|bbox| translate_original_bbox_to_refinement(&refinement_crop, bbox)); + let (refinement_decision, refinement_raw, _) = + request_model_json::( + invocation, + build_gui_grounding_refinement_prompt( + request, + capture_state, + &refinement_crop, + Some(&prior_point), + prior_bbox.as_ref(), + ), + ModelInputImage { + bytes: &refinement_crop.image_bytes, + mime_type: "image/png", + }, + GUI_GROUNDING_REFINEMENT_SYSTEM_PROMPT, + gui_grounding_output_schema(), + "GUI grounding refinement", + ) + .await?; + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "refinement".to_string(), + serde_json::json!({ + "attempted": true, + "raw": refinement_raw.clone(), + "crop": refinement_crop_metadata(&refinement_crop), + "outcome": predictor_outcome_metadata( + &refinement_decision, + refinement_decision.click_point.as_ref(), + refinement_decision.bbox.as_ref(), + ), + }), + ); + } + if refinement_decision.status == "resolved" + && refinement_decision.found + && refinement_decision + .coordinate_space + .as_deref() + .unwrap_or("image_pixels") + == "image_pixels" + && let Some(refined_point) = refinement_decision.click_point.as_ref() + { + let refined_point = + translate_refinement_point_to_original(&refinement_crop, refined_point); + candidate_decision.click_point = Some(refined_point); + candidate_decision.bbox = refinement_decision + .bbox + .as_ref() + .map(|bbox| translate_refinement_bbox_to_original(&refinement_crop, bbox)); + if let JsonValue::Object(fields) = &mut candidate_raw { + fields.insert("initial_prediction_raw".to_string(), raw.clone()); + fields.insert("refinement_raw".to_string(), refinement_raw); + fields.insert( + "grounding_refinement_applied".to_string(), + JsonValue::Bool(true), + ); + fields.insert( + "grounding_refinement_crop".to_string(), + serde_json::json!({ + "x": refinement_crop.offset_x, + "y": refinement_crop.offset_y, + "width": refinement_crop.crop_width, + "height": refinement_crop.crop_height, + }), + ); + fields.insert( + "grounding_refinement_model_image".to_string(), + serde_json::json!({ + "width": refinement_crop.model_width, + "height": refinement_crop.model_height, + "scale_x": refinement_crop.model_scale_x, + "scale_y": refinement_crop.model_scale_y, + }), + ); + } + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert("refinement_applied".to_string(), JsonValue::Bool(true)); + } + validation_zoom_context = true; + validation_image_bytes_override = Some(refinement_crop.image_bytes.clone()); + validation_state_override = Some(ObserveState { + capture: CaptureArtifact { + origin_x: refinement_crop.offset_x, + origin_y: refinement_crop.offset_y, + width: refinement_crop.crop_width, + height: refinement_crop.crop_height, + image_width: refinement_crop.model_width, + image_height: refinement_crop.model_height, + display_index: capture_state.capture.display_index, + capture_mode: capture_state.capture.capture_mode, + window_title: capture_state.capture.window_title.clone(), + window_count: capture_state.capture.window_count, + window_capture_strategy: capture_state + .capture + .window_capture_strategy + .clone(), + host_exclusion: capture_state.capture.host_exclusion.clone(), + }, + app_name: capture_state.app_name.clone(), + }); + validation_point_override = candidate_decision + .click_point + .as_ref() + .map(|point| translate_original_point_to_refinement(&refinement_crop, point)); + validation_bbox_override = candidate_decision + .bbox + .as_ref() + .map(|bbox| translate_original_bbox_to_refinement(&refinement_crop, bbox)); + } + } else if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "refinement".to_string(), + serde_json::json!({ + "attempted": false, + }), + ); + } + let decision = candidate_decision; + let raw = candidate_raw; + let image_point = decision.click_point.clone().ok_or_else(|| { + FunctionCallError::RespondToModel( + "GUI grounding refinement resolved a target without a click_point".to_string(), + ) + })?; + let validation = if grounding_mode == "complex" { + let (validation_point, validation_bbox, validation_capture_state, validation_image) = + if let (Some(point), Some(state), Some(image_bytes)) = ( + validation_point_override.clone(), + validation_state_override.clone(), + validation_image_bytes_override.as_ref(), + ) { + ( + point, + validation_bbox_override.clone(), + state, + image_bytes.as_slice(), + ) + } else { + ( + translate_original_point_to_model(&prepared_grounding_image, &image_point), + decision.bbox.as_ref().map(|bbox| { + translate_original_bbox_to_model(&prepared_grounding_image, bbox) + }), + model_capture_state.clone(), + prepared_grounding_image.bytes.as_slice(), + ) + }; + let validation_image = render_validation_overlay( + validation_image, + &validation_point, + validation_bbox.as_ref(), + )?; + Some({ + let (validation_result, validation_raw, validation_text) = + request_model_json::( + invocation, + build_gui_grounding_validation_prompt( + request, + &validation_capture_state, + &validation_point, + validation_bbox.as_ref(), + validation_zoom_context, + ), + ModelInputImage { + bytes: &validation_image, + mime_type: "image/png", + }, + GUI_GROUNDING_VALIDATION_SYSTEM_PROMPT, + gui_grounding_validation_output_schema(), + "GUI grounding validation", + ) + .await?; + ( + validation_result, + validation_raw, + validation_text, + validation_image.len(), + ) + }) + } else { + None + }; + let rejected = validation + .as_ref() + .is_some_and(|(validation_result, _, _, _)| { + validation_result.status != "approved" || !validation_result.approved + }); + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "selected_candidate".to_string(), + predictor_outcome_metadata( + &decision, + decision.click_point.as_ref(), + decision.bbox.as_ref(), + ), + ); + } + if let Some((validation_result, validation_raw, _, validation_image_len)) = &validation + && let JsonValue::Object(fields) = &mut round_artifact + { + fields.insert( + "validation".to_string(), + serde_json::json!({ + "triggered": true, + "raw": validation_raw, + "outcome": { + "status": validation_result.status, + "approved": validation_result.approved, + "confidence": validation_result.confidence, + "reason": validation_result.reason, + "failure_kind": validation_result.failure_kind, + "retry_hint": validation_result.retry_hint, + }, + "overlay_image": { + "mime_type": "image/png", + "byte_length": validation_image_len, + }, + }), + ); + } else if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "validation".to_string(), + serde_json::json!({ + "triggered": false, + }), + ); + } + if rejected && round < max_rounds { + if let Some(reason) = decision.reason.as_deref() { + append_unique_retry_note( + &mut retry_notes, + format!("Round {round} predictor rationale: {reason}"), + ); + } + if let Some((validation_result, _, _, _)) = &validation { + let validation_reason = validation_result + .reason + .as_deref() + .unwrap_or("validator rejected the simulated action"); + append_unique_retry_note( + &mut retry_notes, + format!("Round {round} validator rejected the candidate: {validation_reason}"), + ); + if let Some(failure_kind) = validation_result.failure_kind.as_deref() { + append_unique_retry_note( + &mut retry_notes, + format!("Round {round} validator failure kind: {failure_kind}"), + ); + if let Some(guidance) = failure_kind_retry_guidance(failure_kind) { + append_unique_retry_note(&mut retry_notes, guidance.to_string()); + } + } + if let Some(retry_hint) = validation_result.retry_hint.as_deref() { + append_unique_retry_note( + &mut retry_notes, + format!("Round {round} retry hint: {retry_hint}"), + ); + } + } + append_unique_retry_note( + &mut retry_notes, + format!( + "Round {round} rejected_point_image_pixels: ({}, {})", + image_point.x.round(), + image_point.y.round() + ), + ); + if let Some(bbox) = decision.bbox.as_ref() { + append_unique_retry_note( + &mut retry_notes, + format!( + "Round {round} rejected_bbox_image_pixels: ({}, {}, {}, {})", + bbox.x1.round(), + bbox.y1.round(), + bbox.x2.round(), + bbox.y2.round() + ), + ); + } + let should_generate_guide = should_generate_retry_guide(validation.as_ref().and_then( + |(validation_result, _, _, _)| validation_result.failure_kind.as_deref(), + )); + guide_image = if should_generate_guide { + let guide_point = + translate_original_point_to_model(&prepared_grounding_image, &image_point); + let guide_bbox = decision + .bbox + .as_ref() + .map(|bbox| translate_original_bbox_to_model(&prepared_grounding_image, bbox)); + Some(render_guide_overlay( + &prepared_grounding_image.bytes, + &guide_point, + guide_bbox.as_ref(), + )?) + } else { + None + }; + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "terminal_state".to_string(), + JsonValue::String("validator_rejected_retry".to_string()), + ); + fields.insert( + "retry_notes_after_round".to_string(), + JsonValue::Array(retry_notes.iter().cloned().map(JsonValue::String).collect()), + ); + fields.insert( + "guide_image_generated".to_string(), + JsonValue::Bool(should_generate_guide), + ); + } + round_artifacts.push(round_artifact); + continue; + } + + if let JsonValue::Object(fields) = &mut round_artifact { + fields.insert( + "terminal_state".to_string(), + JsonValue::String(if rejected { + "validator_rejected_final".to_string() + } else { + "accepted".to_string() + }), + ); + fields.insert("selected".to_string(), JsonValue::Bool(true)); + fields.insert( + "retry_notes_after_round".to_string(), + JsonValue::Array(retry_notes.iter().cloned().map(JsonValue::String).collect()), + ); + } + round_artifacts.push(round_artifact); + + selected_round = round as i64; + final_decision = Some(decision); + final_raw = Some(raw); + final_validation = validation; + break; + } + + let decision = final_decision.ok_or_else(|| { + FunctionCallError::RespondToModel("GUI grounding exhausted retry rounds".to_string()) + })?; + let mut raw = final_raw.ok_or_else(|| { + FunctionCallError::RespondToModel( + "GUI grounding exhausted retry rounds without raw output".to_string(), + ) + })?; + if let Some((validation_result, _, _, _)) = &final_validation + && (validation_result.status != "approved" || !validation_result.approved) + { + emit_gui_grounding_debug(request, grounding_mode, &round_artifacts); + return Ok(None); + } + let image_point = decision.click_point.clone().ok_or_else(|| { + FunctionCallError::RespondToModel( + "GUI grounding resolved a target without a click_point".to_string(), + ) + })?; + let local_point = image_point_within_capture(capture_state, &image_point).ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "GUI grounding resolved `{}` to image point ({}, {}), which falls outside the screenshot.", + request.target, + image_point.x.round(), + image_point.y.round() + )) + })?; + let local_bounds = decision + .bbox + .as_ref() + .and_then(grounding_bbox_to_rect) + .and_then(|rect| local_rect_within_state(capture_state, &rect)) + .unwrap_or_else(|| HelperRect { + x: local_point.x.max(0.0), + y: local_point.y.max(0.0), + width: 1.0, + height: 1.0, + }); + let display_point = image_point_to_display(capture_state, &local_point); + let display_bounds = image_rect_to_display(capture_state, &local_bounds); + let ( + selected_attempt, + validation_triggered, + validation_status, + validation_reason, + validation_confidence, + rounds_attempted, + ) = if let Some((validation_result, validation_raw, _, _)) = final_validation { + if let JsonValue::Object(fields) = &mut raw { + fields.insert("validation_raw".to_string(), validation_raw); + } + ( + if selected_round > 1 { + "validated_retry" + } else { + "validated" + }, + true, + validation_result.status, + validation_result.reason, + validation_result.confidence, + selected_round, + ) + } else { + ( + if selected_round > 1 { + "retry" + } else { + "initial" + }, + false, + "skipped".to_string(), + None, + None, + selected_round, + ) + }; + annotate_grounding_round_artifacts(&mut raw, &round_artifacts); + annotate_grounding_raw( + &mut raw, + grounding_mode, + selected_attempt, + validation_triggered, + &validation_status, + validation_reason.as_deref(), + validation_confidence, + rounds_attempted, + ); + Ok(Some(ResolvedTarget { + window_title: capture_state.capture.window_title.clone(), + provider: gui_grounding_provider_name(invocation), + confidence: decision.confidence.unwrap_or(0.0).clamp(0.0, 1.0), + reason: decision.reason, + grounding_mode_requested: grounding_mode.to_string(), + grounding_mode_effective: grounding_mode.to_string(), + scope: request.scope.map(ToOwned::to_owned), + point: display_point, + bounds: display_bounds, + local_point: Some(local_point), + local_bounds: Some(local_bounds), + raw: Some(raw), + capture_state: capture_state.clone(), + })) +} + +// --------------------------------------------------------------------------- +// Batch grounding: resolve multiple targets in parallel individual calls +// --------------------------------------------------------------------------- + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum BatchGroundingRole { + /// Primary target (click, type, scroll, or drag source). + Primary, + /// Secondary target for drag destination. + DragDestination, +} + +#[derive(Clone, Debug)] +pub(super) struct BatchGroundingTarget { + pub(super) step_index: usize, + pub(super) role: BatchGroundingRole, + pub(super) target: String, + pub(super) action: String, + pub(super) location_hint: Option, + pub(super) scope: Option, +} + +/// Resolve multiple semantic targets using parallel individual grounding calls. +/// +/// Each target is grounded independently using the proven single-target +/// `resolve_grounded_target()` function, run concurrently via `join_all`. +/// All calls share the same screenshot, so the wall-clock time is roughly +/// `max(individual call durations)` rather than their sum. +/// +/// Returns a `Vec` indexed by position in `targets`. Each entry is `Some` +/// when the target was resolved, or `None` when it was not found. +pub(super) async fn resolve_batch_grounded_targets( + invocation: &ToolInvocation, + targets: &[BatchGroundingTarget], + capture_state: &ObserveState, + image_bytes: &[u8], +) -> Result>, FunctionCallError> { + if targets.is_empty() { + return Ok(Vec::new()); + } + + // Launch one grounding call per target, all in parallel. + // GuiTargetRequest derives Copy, so it can be shared across futures. + let futures: Vec<_> = targets + .iter() + .map(|target| { + let action: &'static str = match target.action.as_str() { + "click" => "click", + "type" => "type", + "scroll" => "scroll", + _ => "click", + }; + let request = super::GuiTargetRequest { + app: capture_state.app_name.as_deref(), + capture_mode: None, + window_selection: None, + target: &target.target, + location_hint: target.location_hint.as_deref(), + scope: target.scope.as_deref(), + grounding_mode: None, + action, + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }; + resolve_grounded_target(invocation, request, capture_state, image_bytes) + }) + .collect(); + + let results = futures::future::join_all(futures).await; + results + .into_iter() + .collect::, _>>() +} + +// --------------------------------------------------------------------------- +// Unified batch grounding: multi-target predictor + validator rounds +// --------------------------------------------------------------------------- + +const GUI_BATCH_GROUNDING_SYSTEM_PROMPT: &str = concat!( + "You are grounding multiple GUI targets inside a single screenshot. ", + "Return JSON only, following the provided schema exactly. ", + "Resolve each requested target independently. ", + "If any target is not confidently visible, return `status` = `not_found`, `found` = false, ", + "and null coordinates for that specific target only." +); + +const GUI_BATCH_VALIDATION_SYSTEM_PROMPT: &str = concat!( + "You are validating multiple GUI grounding predictions inside a screenshot. ", + "Each prediction is highlighted with a colored crosshair marker. ", + "Return JSON only, following the provided schema exactly. ", + "Approve each prediction only when the highlighted point is a correct interaction point for the corresponding target." +); + +const MARKER_COLOR_NAMES: [&str; 10] = [ + "RED", "BLUE", "GREEN", "ORANGE", "PURPLE", + "CYAN", "YELLOW", "PINK", "BROWN", "GRAY", +]; + +const MARKER_COLORS: [Rgba; 10] = [ + Rgba([255, 48, 48, 255]), + Rgba([48, 128, 255, 255]), + Rgba([48, 200, 48, 255]), + Rgba([255, 165, 0, 255]), + Rgba([200, 48, 200, 255]), + Rgba([0, 200, 200, 255]), + Rgba([255, 220, 0, 255]), + Rgba([255, 105, 180, 255]), + Rgba([139, 90, 43, 255]), + Rgba([128, 128, 128, 255]), +]; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct BatchGroundingModelResponse { + targets: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct BatchGroundingTargetResponse { + index: usize, + status: String, + found: bool, + confidence: Option, + reason: Option, + coordinate_space: Option, + click_point: Option, + bbox: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct BatchValidationResponse { + targets: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +struct BatchValidationTargetResponse { + index: usize, + status: String, + approved: bool, + confidence: Option, + reason: Option, + failure_kind: Option, + retry_hint: Option, +} + +fn gui_batch_grounding_output_schema() -> JsonValue { + let target_schema = serde_json::json!({ + "type": "object", + "properties": { + "index": { "type": "number" }, + "status": { "type": "string" }, + "found": { "type": "boolean" }, + "confidence": { "type": ["number", "null"] }, + "reason": { "type": ["string", "null"] }, + "coordinate_space": { "type": ["string", "null"] }, + "click_point": { + "type": ["object", "null"], + "properties": { "x": { "type": "number" }, "y": { "type": "number" } }, + "required": ["x", "y"], + "additionalProperties": false + }, + "bbox": { + "type": ["object", "null"], + "properties": { + "x1": { "type": "number" }, "y1": { "type": "number" }, + "x2": { "type": "number" }, "y2": { "type": "number" } + }, + "required": ["x1", "y1", "x2", "y2"], + "additionalProperties": false + } + }, + "required": ["index", "status", "found", "confidence", "reason", "coordinate_space", "click_point", "bbox"], + "additionalProperties": false + }); + serde_json::json!({ + "type": "object", + "properties": { "targets": { "type": "array", "items": target_schema } }, + "required": ["targets"], + "additionalProperties": false + }) +} + +fn gui_batch_validation_output_schema() -> JsonValue { + let target_schema = serde_json::json!({ + "type": "object", + "properties": { + "index": { "type": "number" }, + "status": { "type": "string" }, + "approved": { "type": "boolean" }, + "confidence": { "type": ["number", "null"] }, + "reason": { "type": ["string", "null"] }, + "failure_kind": { "type": ["string", "null"] }, + "retry_hint": { "type": ["string", "null"] } + }, + "required": ["index", "status", "approved", "confidence", "reason", "failure_kind", "retry_hint"], + "additionalProperties": false + }); + serde_json::json!({ + "type": "object", + "properties": { "targets": { "type": "array", "items": target_schema } }, + "required": ["targets"], + "additionalProperties": false + }) +} + +fn build_batch_gui_grounding_prompt( + targets: &[(usize, &BatchGroundingTarget)], + capture_state: &ObserveState, + retry_notes_per_target: &std::collections::HashMap>, + has_guide_image: bool, +) -> String { + let mut lines = vec![ + format!("capture_mode: {}", capture_state.capture.capture_mode), + format!( + "capture_size: {}x{}", + capture_state.capture.image_width, capture_state.capture.image_height + ), + ]; + if let Some(app) = capture_state.app_name.as_deref() { + lines.push(format!("app: {app}")); + } + if let Some(window_title) = capture_state.capture.window_title.as_deref() { + lines.push(format!("window_title: {window_title}")); + } + lines.push(String::new()); + lines.push(format!( + "Resolve the following {} GUI targets in this screenshot.", + targets.len() + )); + lines.push(String::new()); + for &(idx, target) in targets { + lines.push(format!("[Target {}]", idx)); + lines.push(format!("action: {}", target.action)); + lines.push(format!( + "action_intent: {}", + format_grounding_action_intent(&target.action) + )); + lines.push(format!("target: {}", target.target)); + if let Some(scope) = &target.scope { + lines.push(format!("scope: {scope}")); + } + if let Some(location_hint) = &target.location_hint { + lines.push(format!("location_hint: {location_hint}")); + } + if let Some(notes) = retry_notes_per_target.get(&idx) { + if !notes.is_empty() { + lines.push("Retry context:".to_string()); + lines.extend(notes.iter().map(|n| format!("- {n}"))); + } + } + lines.push(String::new()); + } + if has_guide_image { + lines.push( + "An additional guide image is attached with RED overlays marking previously rejected candidates. Do not repeat those marked positions." + .to_string(), + ); + } + lines.push( + "Ground each target independently. Use only visible screenshot evidence." + .to_string(), + ); + lines.push( + "Choose the smallest obvious actionable or editable surface, and keep the click point on the visible hit target." + .to_string(), + ); + lines.push( + "Use `coordinate_space: \"image_pixels\"`. Return one entry per target in the `targets` array." + .to_string(), + ); + lines.push("Respond with JSON only.".to_string()); + lines.join("\n") +} + +fn build_batch_gui_grounding_validation_prompt( + candidates: &[(usize, &BatchGroundingTarget, &HelperPoint, Option<&GroundingBoundingBox>)], + capture_state: &ObserveState, +) -> String { + let mut lines = vec![ + format!("capture_mode: {}", capture_state.capture.capture_mode), + format!( + "capture_size: {}x{}", + capture_state.capture.image_width, capture_state.capture.image_height + ), + ]; + if let Some(app) = capture_state.app_name.as_deref() { + lines.push(format!("app: {app}")); + } + if let Some(window_title) = capture_state.capture.window_title.as_deref() { + lines.push(format!("window_title: {window_title}")); + } + lines.push(String::new()); + lines.push(format!( + "Validate the following {} grounding predictions. Each is highlighted with a colored crosshair.", + candidates.len() + )); + lines.push(String::new()); + for (i, &(idx, target, point, bbox)) in candidates.iter().enumerate() { + let color_name = MARKER_COLOR_NAMES[i % MARKER_COLOR_NAMES.len()]; + lines.push(format!( + "[Target {} — {color_name} marker]", + idx + )); + lines.push(format!("action: {}", target.action)); + lines.push(format!("target: {}", target.target)); + if let Some(scope) = &target.scope { + lines.push(format!("scope: {scope}")); + } + lines.push(format!( + "predicted_click_point_image_pixels: ({}, {})", + point.x.round(), + point.y.round() + )); + if let Some(bbox) = bbox { + lines.push(format!( + "predicted_bbox_image_pixels: ({}, {}, {}, {})", + bbox.x1.round(), bbox.y1.round(), bbox.x2.round(), bbox.y2.round() + )); + } + lines.push(String::new()); + } + lines.push( + "Approve each prediction only when the highlighted point is a correct interaction point for the corresponding target." + .to_string(), + ); + lines.push( + "Reject if the highlight lands on whitespace, padding, decoration, or the wrong control." + .to_string(), + ); + lines.push( + "Use `failure_kind`: `wrong_region`, `scope_mismatch`, `wrong_control`, `wrong_point`, `state_mismatch`, `partial_visibility`, or `other`." + .to_string(), + ); + lines.push( + "Return one entry per target in the `targets` array." + .to_string(), + ); + lines.push("Respond with JSON only.".to_string()); + lines.join("\n") +} + +fn render_batch_validation_overlay( + image_bytes: &[u8], + markers: &[(usize, &HelperPoint, Option<&GroundingBoundingBox>)], +) -> Result, FunctionCallError> { + let mut image = image::load_from_memory(image_bytes) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode image for batch validation overlay: {error}" + )) + })? + .into_rgba8(); + for (i, &(_, point, bbox)) in markers.iter().enumerate() { + let color = MARKER_COLORS[i % MARKER_COLORS.len()]; + if let Some(rect) = bbox.and_then(grounding_bbox_to_rect) { + draw_rect_outline(&mut image, &rect, color); + } + draw_crosshair(&mut image, point, color); + } + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut encoded, ImageFormat::Png) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode batch validation overlay: {error}" + )) + })?; + Ok(encoded.into_inner()) +} + +fn render_batch_guide_overlay( + image_bytes: &[u8], + rejected_points: &[(&HelperPoint, Option<&GroundingBoundingBox>)], +) -> Result, FunctionCallError> { + let mut image = image::load_from_memory(image_bytes) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode image for batch guide overlay: {error}" + )) + })? + .into_rgba8(); + let accent = Rgba([255, 48, 48, 255]); + for &(point, bbox) in rejected_points { + if let Some(rect) = bbox.and_then(grounding_bbox_to_rect) { + draw_rect_outline(&mut image, &rect, accent); + } + draw_crosshair(&mut image, point, accent); + } + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut encoded, ImageFormat::Png) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode batch guide overlay: {error}" + )) + })?; + Ok(encoded.into_inner()) +} + +/// Resolve multiple semantic targets using unified multi-target grounding +/// with multi-round predictor + validator flow (complex mode). +/// +/// Each round makes 2 model calls (predictor + validator) for ALL pending +/// targets. Approved targets are finalized; rejected targets enter the next +/// round with retry hints and guide overlays. +/// +/// Total model calls: 2–6 (2 per round × max 3 rounds), regardless of N. +pub(super) async fn resolve_batch_grounded_targets_unified( + invocation: &ToolInvocation, + targets: &[BatchGroundingTarget], + capture_state: &ObserveState, + image_bytes: &[u8], +) -> Result>, FunctionCallError> { + if targets.is_empty() { + return Ok(Vec::new()); + } + + let max_rounds = 3; + let prepared = prepare_grounding_model_image( + image_bytes, + GroundingModelImageConfig { + logical_width: Some(capture_state.capture.width), + logical_height: Some(capture_state.capture.height), + scale_x: Some(capture_state.capture.scale_x()), + scale_y: Some(capture_state.capture.scale_y()), + allow_logical_normalization: true, + }, + "unified batch grounding image", + )?; + let mut model_capture_state = capture_state.clone(); + model_capture_state.capture.image_width = prepared.model_width; + model_capture_state.capture.image_height = prepared.model_height; + + let provider = gui_grounding_provider_name(invocation); + + // Per-target state. + let mut results: Vec> = vec![None; targets.len()]; + let mut pending: Vec = (0..targets.len()).collect(); // indices into `targets` + let mut retry_notes: std::collections::HashMap> = + std::collections::HashMap::new(); + let mut guide_image: Option> = None; + + for round in 1..=max_rounds { + if pending.is_empty() { + break; + } + + // ── PREDICTOR: one call for all pending targets ──────────────── + let pending_targets: Vec<(usize, &BatchGroundingTarget)> = + pending.iter().map(|&i| (i, &targets[i])).collect(); + + let mut grounding_images = vec![ModelInputImage { + bytes: &prepared.bytes, + mime_type: prepared.mime_type, + }]; + if let Some(guide_bytes) = guide_image.as_deref() { + grounding_images.push(ModelInputImage { + bytes: guide_bytes, + mime_type: "image/png", + }); + } + + let prompt = build_batch_gui_grounding_prompt( + &pending_targets, + &model_capture_state, + &retry_notes, + guide_image.is_some(), + ); + let (pred_response, _, _) = + request_model_json_with_images::( + invocation, + prompt, + &grounding_images, + GUI_BATCH_GROUNDING_SYSTEM_PROMPT, + gui_batch_grounding_output_schema(), + &format!("unified batch grounding round {round}"), + ) + .await?; + + // Parse predictor results into a map. + let mut pred_map: std::collections::HashMap = + std::collections::HashMap::new(); + for resp in pred_response.targets { + pred_map.insert(resp.index, resp); + } + + // Collect resolved candidates for validation. + struct Candidate { + target_idx: usize, + image_point: HelperPoint, + bbox: Option, + confidence: f64, + reason: Option, + } + let mut candidates: Vec = Vec::new(); + let mut not_found_this_round: Vec = Vec::new(); + + for &i in &pending { + match pred_map.remove(&i) { + Some(resp) + if resp.status == "resolved" + && resp.found + && resp.click_point.is_some() => + { + let model_point = resp.click_point.unwrap(); + let image_point = + translate_model_point_to_original(&prepared, &model_point); + let bbox = resp + .bbox + .as_ref() + .map(|b| translate_model_bbox_to_original(&prepared, b)); + candidates.push(Candidate { + target_idx: i, + image_point, + bbox, + confidence: resp.confidence.unwrap_or(0.0), + reason: resp.reason, + }); + } + Some(resp) => { + // Not found — add retry notes. + if let Some(reason) = resp.reason.as_deref() { + retry_notes + .entry(i) + .or_default() + .push(format!("Round {round} predictor: {reason}")); + } + not_found_this_round.push(i); + } + None => { + not_found_this_round.push(i); + } + } + } + + if candidates.is_empty() { + // All pending targets not found this round. + if round == max_rounds { + break; // remaining pending → None + } + guide_image = None; + // Update pending to exclude those we won't retry. + pending.retain(|i| not_found_this_round.contains(i)); + continue; + } + + // ── VALIDATOR: one call to approve/reject all candidates ─────── + let validation_markers: Vec<(usize, &HelperPoint, Option<&GroundingBoundingBox>)> = + candidates + .iter() + .enumerate() + .map(|(visual_idx, c)| { + (visual_idx, &c.image_point, c.bbox.as_ref()) + }) + .collect(); + + let validation_image = + render_batch_validation_overlay(&prepared.bytes, &validation_markers)?; + + let val_candidates: Vec<(usize, &BatchGroundingTarget, &HelperPoint, Option<&GroundingBoundingBox>)> = + candidates + .iter() + .map(|c| { + (c.target_idx, &targets[c.target_idx], &c.image_point, c.bbox.as_ref()) + }) + .collect(); + + let val_prompt = build_batch_gui_grounding_validation_prompt( + &val_candidates, + &model_capture_state, + ); + let (val_response, _, _) = + request_model_json_with_images::( + invocation, + val_prompt, + &[ModelInputImage { + bytes: &validation_image, + mime_type: "image/png", + }], + GUI_BATCH_VALIDATION_SYSTEM_PROMPT, + gui_batch_validation_output_schema(), + &format!("unified batch validation round {round}"), + ) + .await?; + + // Parse validation results. + let mut val_map: std::collections::HashMap = + std::collections::HashMap::new(); + for vr in val_response.targets { + val_map.insert(vr.index, vr); + } + + // Process each candidate. + let mut rejected_points: Vec<(&HelperPoint, Option<&GroundingBoundingBox>)> = Vec::new(); + let mut newly_resolved: Vec = Vec::new(); + + for candidate in &candidates { + let i = candidate.target_idx; + let approved = val_map + .get(&i) + .map(|v| v.status == "approved" && v.approved) + .unwrap_or(false); + + if approved { + // Finalize this target. + let local_point = + image_point_within_capture(capture_state, &candidate.image_point); + if let Some(local_point) = local_point { + let local_bounds = candidate + .bbox + .as_ref() + .and_then(grounding_bbox_to_rect) + .and_then(|rect| local_rect_within_state(capture_state, &rect)) + .unwrap_or_else(|| HelperRect { + x: local_point.x.max(0.0), + y: local_point.y.max(0.0), + width: 1.0, + height: 1.0, + }); + let display_point = image_point_to_display(capture_state, &local_point); + let display_bounds = image_rect_to_display(capture_state, &local_bounds); + results[i] = Some(ResolvedTarget { + window_title: capture_state.capture.window_title.clone(), + provider: provider.clone(), + confidence: candidate.confidence.clamp(0.0, 1.0), + reason: candidate.reason.clone(), + grounding_mode_requested: "unified".to_string(), + grounding_mode_effective: "unified".to_string(), + scope: targets[i].scope.clone(), + point: display_point, + bounds: display_bounds, + local_point: Some(local_point), + local_bounds: Some(local_bounds), + raw: None, + capture_state: capture_state.clone(), + }); + } + newly_resolved.push(i); + } else { + // Rejected — collect retry notes. + if let Some(vr) = val_map.get(&i) { + let notes = retry_notes.entry(i).or_default(); + if let Some(reason) = vr.reason.as_deref() { + notes.push(format!("Round {round} validator rejected: {reason}")); + } + if let Some(fk) = vr.failure_kind.as_deref() { + notes.push(format!("Round {round} failure kind: {fk}")); + if let Some(guidance) = failure_kind_retry_guidance(fk) { + notes.push(guidance.to_string()); + } + } + if let Some(hint) = vr.retry_hint.as_deref() { + notes.push(format!("Round {round} retry hint: {hint}")); + } + notes.push(format!( + "Round {round} rejected_point: ({}, {})", + candidate.image_point.x.round(), + candidate.image_point.y.round() + )); + } + rejected_points.push((&candidate.image_point, candidate.bbox.as_ref())); + } + } + + // Update pending: remove resolved, keep rejected + not_found. + pending.retain(|i| !newly_resolved.contains(i)); + + // Generate guide overlay for next round if there are rejected points. + guide_image = if !rejected_points.is_empty() && round < max_rounds { + Some(render_batch_guide_overlay( + &prepared.bytes, + &rejected_points, + )?) + } else { + None + }; + } + + Ok(results) +} diff --git a/codex-rs/core/src/tools/handlers/gui/native_helper.swift b/codex-rs/core/src/tools/handlers/gui/native_helper.swift new file mode 100644 index 000000000000..30225a9b1d8a --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/native_helper.swift @@ -0,0 +1,1406 @@ +import Foundation +import AppKit +import ApplicationServices +import CoreGraphics +import Carbon.HIToolbox + +enum HelperError: Error, CustomStringConvertible { + case invalidCommand(String) + case invalidEnv(String) + case missingEnv(String) + case applicationNotFound(String) + case activationFailed(String) + case eventCreationFailed(String) + case screenshotFailed(String) + case fileReadFailed(String) + + var description: String { + switch self { + case .invalidCommand(let value): + return "invalidCommand(\(value))" + case .invalidEnv(let key): + return "invalidEnv(\(key))" + case .missingEnv(let key): + return "missingEnv(\(key))" + case .applicationNotFound(let name): + return "applicationNotFound(\(name))" + case .activationFailed(let name): + return "activationFailed(\(name))" + case .eventCreationFailed(let detail): + return "eventCreationFailed(\(detail))" + case .screenshotFailed(let detail): + return "screenshotFailed(\(detail))" + case .fileReadFailed(let detail): + return "fileReadFailed(\(detail))" + } + } +} + +struct Rect: Codable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct Point: Codable { + let x: Double + let y: Double +} + +struct DisplayDescriptor: Codable { + let index: Int + let bounds: Rect +} + +struct CaptureContext: Codable { + let appName: String? + let display: DisplayDescriptor + let cursor: Point + let windowId: Int? + let windowTitle: String? + let windowBounds: Rect? + let windowCount: Int? + let windowCaptureStrategy: String? + let hostSelfExcludeApplied: Bool? + let hostFrontmostExcluded: Bool? + let hostFrontmostAppName: String? + let hostFrontmostBundleId: String? +} + +struct ObservePayload: Codable { + let appName: String? + let display: DisplayDescriptor + let cursor: Point + let windowId: Int? + let windowTitle: String? + let windowBounds: Rect? + let windowCount: Int? + let windowCaptureStrategy: String? + let hostSelfExcludeApplied: Bool? + let hostFrontmostExcluded: Bool? + let hostFrontmostAppName: String? + let hostFrontmostBundleId: String? + let captureMode: String + let captureBounds: Rect + let captureWidth: Int + let captureHeight: Int + let imagePath: String +} + +struct WindowMatch { + let id: Int + let title: String? + let bounds: CGRect + let layer: Int +} + +struct WindowSelection { + let primary: WindowMatch + let captureBounds: CGRect + let windowCount: Int + let captureStrategy: String +} + +struct RedactionResult: Codable { + let redactionCount: Int +} + +struct WindowExclusions { + let ownerNames: Set + let bundleIds: Set + + var isEmpty: Bool { + ownerNames.isEmpty && bundleIds.isEmpty + } +} + +/// Per-request environment override. In `serve` mode, each JSON-line +/// request supplies its own `env` dictionary. The `env()` helper checks +/// this thread-local override first, falling back to the real process +/// environment. For one-shot commands this stays `nil` so the original +/// behaviour is preserved. +var requestEnvOverride: [String: String]? = nil + +func env(_ key: String) -> String { + if let override_ = requestEnvOverride, let value = override_[key] { + return value + } + return ProcessInfo.processInfo.environment[key] ?? "" +} + +func trimmedEnv(_ key: String) -> String? { + let value = env(key).trimmingCharacters(in: .whitespacesAndNewlines) + return value.isEmpty ? nil : value +} + +func normalizedText(_ value: String?) -> String? { + guard let value else { return nil } + let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return normalized.isEmpty ? nil : normalized +} + +func parseDelimitedEnv(_ key: String) -> [String] { + env(key) + .split(whereSeparator: { $0 == "," || $0.isNewline }) + .map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } +} + +func mergeNormalizedValues(_ values: [String]) -> Set { + Set(values.compactMap { normalizedText($0) }) +} + +func loadWindowExclusions() -> WindowExclusions { + let ownerNames = mergeNormalizedValues( + parseDelimitedEnv("CODEX_GUI_EXCLUDED_OWNER_NAMES") + + parseDelimitedEnv("CODEX_GUI_AUTO_EXCLUDED_OWNER_NAMES") + ) + let bundleIds = mergeNormalizedValues( + parseDelimitedEnv("CODEX_GUI_EXCLUDED_BUNDLE_IDS") + + parseDelimitedEnv("CODEX_GUI_AUTO_EXCLUDED_BUNDLE_IDS") + ) + return WindowExclusions(ownerNames: ownerNames, bundleIds: bundleIds) +} + +func matchesExcludedApplication(_ app: NSRunningApplication?, exclusions: WindowExclusions) -> Bool { + guard let app else { + return false + } + if let localizedName = normalizedText(app.localizedName), exclusions.ownerNames.contains(localizedName) { + return true + } + if let bundleId = normalizedText(app.bundleIdentifier), exclusions.bundleIds.contains(bundleId) { + return true + } + return false +} + +func matchesExcludedWindow(_ info: [String: Any], exclusions: WindowExclusions) -> Bool { + if exclusions.isEmpty { + return false + } + if let ownerName = normalizedText(info[kCGWindowOwnerName as String] as? String), exclusions.ownerNames.contains(ownerName) { + return true + } + guard + let ownerPidValue = info[kCGWindowOwnerPID as String] as? NSNumber, + !exclusions.bundleIds.isEmpty + else { + return false + } + let ownerPid = pid_t(ownerPidValue.int32Value) + guard let app = NSRunningApplication(processIdentifier: ownerPid) else { + return false + } + return exclusions.bundleIds.contains(app.bundleIdentifier?.lowercased() ?? "") +} + +func requiredDouble(_ key: String) throws -> Double { + let raw = env(key) + guard let value = Double(raw) else { throw HelperError.invalidEnv(key) } + return value +} + +func requiredInt(_ key: String) throws -> Int { + let raw = env(key) + guard let value = Int(raw) else { throw HelperError.invalidEnv(key) } + return value +} + +func requiredInt32(_ key: String) throws -> Int32 { + let raw = env(key) + guard let value = Int32(raw) else { throw HelperError.invalidEnv(key) } + return value +} + +func optionalInt(_ key: String) -> Int? { + let raw = env(key) + guard !raw.isEmpty else { return nil } + return Int(raw) +} + +func optionalDouble(_ key: String) -> Double? { + let raw = env(key) + guard !raw.isEmpty else { return nil } + return Double(raw) +} + +func requiredPath(_ key: String) throws -> String { + guard let value = trimmedEnv(key) else { + throw HelperError.missingEnv(key) + } + return value +} + +func shouldActivateApp() -> Bool { + env("CODEX_GUI_ACTIVATE_APP") != "0" +} + +func preferWindowWhenAvailable() -> Bool { + env("CODEX_GUI_PREFER_WINDOW") == "1" +} + +func requestedCaptureMode() throws -> String? { + guard let value = trimmedEnv("CODEX_GUI_CAPTURE_MODE") else { + return nil + } + switch value { + case "display", "window": + return value + default: + throw HelperError.invalidEnv("CODEX_GUI_CAPTURE_MODE") + } +} + +func matchesAppName(_ app: NSRunningApplication, requestedName: String) -> Bool { + let normalized = requestedName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if normalized.isEmpty { return false } + if app.localizedName?.lowercased() == normalized { return true } + if app.bundleIdentifier?.lowercased() == normalized { return true } + if app.bundleURL?.deletingPathExtension().lastPathComponent.lowercased() == normalized { return true } + return false +} + +func findRunningApplication(named name: String) -> NSRunningApplication? { + let candidates = NSWorkspace.shared.runningApplications.filter { app in + matchesAppName(app, requestedName: name) && !app.isTerminated + } + if let active = candidates.first(where: { $0.isActive }) { + return active + } + return candidates.first +} + +func resolveRequestedApplication(named name: String?) -> NSRunningApplication? { + guard let name else { + return NSWorkspace.shared.frontmostApplication + } + return findRunningApplication(named: name) ?? NSWorkspace.shared.frontmostApplication +} + +func activateApplication(named name: String?) throws { + guard let name else { return } + if let frontmost = NSWorkspace.shared.frontmostApplication, matchesAppName(frontmost, requestedName: name) { + return + } + guard let app = findRunningApplication(named: name) else { + throw HelperError.applicationNotFound(name) + } + if #available(macOS 14.0, *) { + app.activate() + } else { + if !app.activate(options: [.activateIgnoringOtherApps]) { + throw HelperError.activationFailed(name) + } + } + usleep(100_000) +} + +func rect(_ value: CGRect) -> Rect { + Rect( + x: value.origin.x.rounded(), + y: value.origin.y.rounded(), + width: value.size.width.rounded(), + height: value.size.height.rounded() + ) +} + +func point(_ value: CGPoint) -> Point { + Point(x: value.x.rounded(), y: value.y.rounded()) +} + +func activeDisplays() -> [(index: Int, bounds: CGRect)] { + var count: UInt32 = 0 + CGGetActiveDisplayList(0, nil, &count) + var displayIDs = Array(repeating: CGDirectDisplayID(0), count: Int(count)) + CGGetActiveDisplayList(count, &displayIDs, &count) + return Array(displayIDs.prefix(Int(count))).enumerated().map { item in + (index: item.offset + 1, bounds: CGDisplayBounds(item.element)) + } +} + +func displayForPoint(_ point: CGPoint, displays: [(index: Int, bounds: CGRect)]) -> (index: Int, bounds: CGRect) { + for display in displays where display.bounds.contains(point) { + return display + } + return displays.first ?? (index: 1, bounds: CGDisplayBounds(CGMainDisplayID())) +} + +func firstVisibleOwnerName(excluding exclusions: WindowExclusions) -> String? { + guard let windowInfo = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + return nil + } + for info in windowInfo { + let alpha = info[kCGWindowAlpha as String] as? Double ?? 1 + if alpha <= 0.01 { + continue + } + if matchesExcludedWindow(info, exclusions: exclusions) { + continue + } + guard let rawBounds = info[kCGWindowBounds as String] else { + continue + } + let boundsDict = rawBounds as! CFDictionary + guard let bounds = CGRect(dictionaryRepresentation: boundsDict), + bounds.width >= 80, + bounds.height >= 80 + else { + continue + } + if let owner = info[kCGWindowOwnerName as String] as? String, !owner.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return owner + } + } + return nil +} + +func matchingWindows( + ownerName: String?, + exactTitle: String?, + titleContains: String?, + exclusions: WindowExclusions +) -> [WindowMatch] { + guard let windowInfo = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + return [] + } + let normalizedExactTitle = normalizedText(exactTitle) + let normalizedContainsTitle = normalizedText(titleContains) + var matches: [WindowMatch] = [] + for info in windowInfo { + let alpha = info[kCGWindowAlpha as String] as? Double ?? 1 + let layer = info[kCGWindowLayer as String] as? Int ?? 0 + let owner = info[kCGWindowOwnerName as String] as? String ?? "" + if alpha <= 0.01 { + continue + } + if matchesExcludedWindow(info, exclusions: exclusions) { + continue + } + if let ownerName, owner != ownerName { + continue + } + guard let rawBounds = info[kCGWindowBounds as String] else { + continue + } + let boundsDict = rawBounds as! CFDictionary + guard + let bounds = CGRect(dictionaryRepresentation: boundsDict), + bounds.width >= 80, + bounds.height >= 80 + else { + continue + } + let windowId = (info[kCGWindowNumber as String] as? NSNumber)?.intValue + let title = info[kCGWindowName as String] as? String + let normalizedTitle = normalizedText(title) ?? "" + if let normalizedExactTitle, normalizedTitle != normalizedExactTitle { + continue + } + if let normalizedContainsTitle, !normalizedTitle.contains(normalizedContainsTitle) { + continue + } + matches.append(WindowMatch( + id: windowId ?? 0, + title: title, + bounds: bounds.integral, + layer: layer + )) + } + return matches +} + +func rankWindows(_ matches: [WindowMatch]) -> [WindowMatch] { + return matches.sorted { lhs, rhs in + let lhsPrimaryLayer = lhs.layer == 0 ? 0 : 1 + let rhsPrimaryLayer = rhs.layer == 0 ? 0 : 1 + if lhsPrimaryLayer != rhsPrimaryLayer { + return lhsPrimaryLayer < rhsPrimaryLayer + } + if lhs.layer != rhs.layer { + return lhs.layer < rhs.layer + } + let lhsArea = lhs.bounds.width * lhs.bounds.height + let rhsArea = rhs.bounds.width * rhs.bounds.height + if lhsArea != rhsArea { + return lhsArea > rhsArea + } + let lhsHasTitle = normalizedText(lhs.title) != nil + let rhsHasTitle = normalizedText(rhs.title) != nil + if lhsHasTitle != rhsHasTitle { + return lhsHasTitle && !rhsHasTitle + } + return lhs.id < rhs.id + } +} + +func unionBounds(for matches: [WindowMatch]) -> CGRect? { + guard let first = matches.first else { + return nil + } + return matches.dropFirst().reduce(first.bounds) { partial, match in + partial.union(match.bounds) + }.integral +} + +func selectedWindow( + ownerName: String?, + exactTitle: String?, + titleContains: String?, + index: Int?, + exclusions: WindowExclusions +) -> WindowSelection? { + let matches = matchingWindows( + ownerName: ownerName, + exactTitle: exactTitle, + titleContains: titleContains, + exclusions: exclusions + ) + guard !matches.isEmpty else { + return nil + } + let hasExplicitSelection = normalizedText(exactTitle) != nil || normalizedText(titleContains) != nil || index != nil + let ranked = rankWindows(matches) + if let index, index > 0, index <= ranked.count { + let window = ranked[index - 1] + return WindowSelection( + primary: window, + captureBounds: window.bounds.integral, + windowCount: 1, + captureStrategy: "selected_window" + ) + } + guard let primary = ranked.first else { + return nil + } + if hasExplicitSelection { + return WindowSelection( + primary: primary, + captureBounds: primary.bounds.integral, + windowCount: 1, + captureStrategy: "selected_window" + ) + } + if matches.count == 1 { + return WindowSelection( + primary: primary, + captureBounds: primary.bounds.integral, + windowCount: 1, + captureStrategy: "main_window" + ) + } + guard let combinedBounds = unionBounds(for: matches) else { + return nil + } + return WindowSelection( + primary: primary, + captureBounds: combinedBounds, + windowCount: matches.count, + captureStrategy: "app_union" + ) +} + +func handleCaptureContext() throws { + let requestedApp = trimmedEnv("CODEX_GUI_APP") + let requestedWindowTitle = trimmedEnv("CODEX_GUI_WINDOW_TITLE") + let requestedWindowTitleContains = trimmedEnv("CODEX_GUI_WINDOW_TITLE_CONTAINS") + let requestedWindowIndex = optionalInt("CODEX_GUI_WINDOW_INDEX") + let exclusions = loadWindowExclusions() + let frontmostApp = NSWorkspace.shared.frontmostApplication + let hostFrontmostExcluded = matchesExcludedApplication(frontmostApp, exclusions: exclusions) + if shouldActivateApp() { + try activateApplication(named: requestedApp) + } + let resolvedApp = resolveRequestedApplication(named: requestedApp) + let shouldPreferVisibleNonExcludedWindow = + requestedApp == nil + && ( + (resolvedApp?.localizedName).flatMap(normalizedText).map { exclusions.ownerNames.contains($0) } ?? false + || (resolvedApp?.bundleIdentifier).flatMap(normalizedText).map { exclusions.bundleIds.contains($0) } ?? false + ) + let targetApp = shouldPreferVisibleNonExcludedWindow + ? firstVisibleOwnerName(excluding: exclusions) + : (resolvedApp?.localizedName ?? requestedApp ?? NSWorkspace.shared.frontmostApplication?.localizedName) + let cursorLocation = CGEvent(source: nil)?.location ?? .zero + let displays = activeDisplays() + let window = selectedWindow( + ownerName: targetApp, + exactTitle: requestedWindowTitle, + titleContains: requestedWindowTitleContains, + index: requestedWindowIndex, + exclusions: exclusions + ) + let anchorPoint = window.map { CGPoint(x: $0.primary.bounds.midX, y: $0.primary.bounds.midY) } ?? cursorLocation + let display = displayForPoint(anchorPoint, displays: displays) + let payload = CaptureContext( + appName: targetApp, + display: DisplayDescriptor(index: display.index, bounds: rect(display.bounds)), + cursor: point(cursorLocation), + windowId: window?.primary.id, + windowTitle: window?.primary.title, + windowBounds: window.map { rect($0.captureBounds) }, + windowCount: window?.windowCount, + windowCaptureStrategy: window?.captureStrategy, + hostSelfExcludeApplied: !exclusions.isEmpty, + hostFrontmostExcluded: hostFrontmostExcluded, + hostFrontmostAppName: frontmostApp?.localizedName, + hostFrontmostBundleId: frontmostApp?.bundleIdentifier + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(payload) + FileHandle.standardOutput.write(data) +} + +func captureScreenshot(bounds: CGRect) throws -> String { + let imagePath = (NSTemporaryDirectory() as NSString).appendingPathComponent( + "codex-gui-observe-\(UUID().uuidString).png" + ) + let region = [ + String(Int(bounds.origin.x.rounded())), + String(Int(bounds.origin.y.rounded())), + String(Int(bounds.width.rounded())), + String(Int(bounds.height.rounded())) + ].joined(separator: ",") + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture") + process.arguments = ["-x", "-C", "-R", region, imagePath] + let stderrPipe = Pipe() + process.standardError = stderrPipe + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { + let stderrData = stderrPipe.fileHandleForReading.readDataToEndOfFile() + let stderr = String(data: stderrData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + throw HelperError.screenshotFailed(stderr.isEmpty ? "screencapture_exit_\(process.terminationStatus)" : stderr) + } + guard FileManager.default.fileExists(atPath: imagePath) else { + throw HelperError.fileReadFailed("missing_capture_\(imagePath)") + } + return imagePath +} + +func excludedWindowRedactionRects( + captureBounds: CGRect, + imagePixelWidth: Int, + imagePixelHeight: Int, + exclusions: WindowExclusions +) -> [CGRect] { + guard + !exclusions.isEmpty, + imagePixelWidth > 0, + imagePixelHeight > 0, + captureBounds.width > 0, + captureBounds.height > 0, + let windowInfo = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] + else { + return [] + } + let scaleX = CGFloat(imagePixelWidth) / captureBounds.width + let scaleY = CGFloat(imagePixelHeight) / captureBounds.height + let imageBounds = CGRect(x: 0, y: 0, width: imagePixelWidth, height: imagePixelHeight) + var rects: [CGRect] = [] + for info in windowInfo { + let alpha = info[kCGWindowAlpha as String] as? Double ?? 1 + if alpha <= 0.01 { + continue + } + if !matchesExcludedWindow(info, exclusions: exclusions) { + continue + } + guard let rawBounds = info[kCGWindowBounds as String] else { + continue + } + let boundsDict = rawBounds as! CFDictionary + guard let bounds = CGRect(dictionaryRepresentation: boundsDict) else { + continue + } + let intersection = bounds.intersection(captureBounds) + if intersection.isNull || intersection.width <= 0 || intersection.height <= 0 { + continue + } + let rect = CGRect( + x: (intersection.origin.x - captureBounds.origin.x) * scaleX, + y: (intersection.origin.y - captureBounds.origin.y) * scaleY, + width: intersection.width * scaleX, + height: intersection.height * scaleY + ).integral.intersection(imageBounds) + if rect.isNull || rect.width <= 0 || rect.height <= 0 { + continue + } + rects.append(rect) + } + return rects +} + +func redactImageAtPath(_ imagePath: String, redactions: [CGRect]) throws { + guard !redactions.isEmpty else { + return + } + let fileURL = URL(fileURLWithPath: imagePath) + let imageData = try Data(contentsOf: fileURL) + guard let dataProvider = CGDataProvider(data: imageData as CFData), + let cgImage = CGImage( + pngDataProviderSource: dataProvider, + decode: nil, + shouldInterpolate: true, + intent: .defaultIntent + ) + else { + throw HelperError.eventCreationFailed("redaction_image_load") + } + let width = cgImage.width + let height = cgImage.height + let colorSpace = cgImage.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)! + let hasAlpha = cgImage.alphaInfo != .none && cgImage.alphaInfo != .noneSkipLast && cgImage.alphaInfo != .noneSkipFirst + let alphaInfo: CGImageAlphaInfo = hasAlpha ? .premultipliedLast : .noneSkipLast + guard let ctx = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: colorSpace, + bitmapInfo: alphaInfo.rawValue + ) else { + throw HelperError.eventCreationFailed("redaction_context_create") + } + ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height)) + ctx.setFillColor(CGColor(gray: 0.97, alpha: 1.0)) + for rect in redactions { + ctx.fill(rect) + } + guard let outputImage = ctx.makeImage(), + let destination = CGImageDestinationCreateWithURL( + fileURL as CFURL, "public.png" as CFString, 1, nil + ) + else { + throw HelperError.eventCreationFailed("redaction_image_render") + } + CGImageDestinationAddImage(destination, outputImage, nil) + guard CGImageDestinationFinalize(destination) else { + throw HelperError.eventCreationFailed("redaction_image_encode") + } +} + +func handleRedactHostWindows() throws { + let imagePath = try requiredPath("CODEX_GUI_IMAGE_PATH") + let captureBounds = CGRect( + x: try requiredDouble("CODEX_GUI_CAPTURE_X"), + y: try requiredDouble("CODEX_GUI_CAPTURE_Y"), + width: try requiredDouble("CODEX_GUI_CAPTURE_WIDTH"), + height: try requiredDouble("CODEX_GUI_CAPTURE_HEIGHT") + ) + let exclusions = loadWindowExclusions() + guard !exclusions.isEmpty else { + let data = try JSONEncoder().encode(RedactionResult(redactionCount: 0)) + FileHandle.standardOutput.write(data) + return + } + let fileURL = URL(fileURLWithPath: imagePath) + let imageData = try Data(contentsOf: fileURL) + guard let dataProvider = CGDataProvider(data: imageData as CFData), + let cgImage = CGImage( + pngDataProviderSource: dataProvider, + decode: nil, + shouldInterpolate: true, + intent: .defaultIntent + ) + else { + throw HelperError.eventCreationFailed("redaction_bitmap_load") + } + let redactions = excludedWindowRedactionRects( + captureBounds: captureBounds, + imagePixelWidth: cgImage.width, + imagePixelHeight: cgImage.height, + exclusions: exclusions + ) + try redactImageAtPath(imagePath, redactions: redactions) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(RedactionResult(redactionCount: redactions.count)) + FileHandle.standardOutput.write(data) +} + +func handleObserve() throws { + let requestedApp = trimmedEnv("CODEX_GUI_APP") + let requestedWindowTitle = trimmedEnv("CODEX_GUI_WINDOW_TITLE") + let requestedWindowTitleContains = trimmedEnv("CODEX_GUI_WINDOW_TITLE_CONTAINS") + let requestedWindowIndex = optionalInt("CODEX_GUI_WINDOW_INDEX") + let requestedMode = try requestedCaptureMode() + let preferWindow = preferWindowWhenAvailable() + let exclusions = loadWindowExclusions() + let frontmostApp = NSWorkspace.shared.frontmostApplication + let hostFrontmostExcluded = matchesExcludedApplication(frontmostApp, exclusions: exclusions) + let windowSelectionRequested = + normalizedText(requestedWindowTitle) != nil || + normalizedText(requestedWindowTitleContains) != nil || + requestedWindowIndex != nil + if shouldActivateApp() { + try activateApplication(named: requestedApp) + } + + let resolvedApp = resolveRequestedApplication(named: requestedApp) + let shouldPreferVisibleNonExcludedWindow = + requestedApp == nil + && ( + (resolvedApp?.localizedName).flatMap(normalizedText).map { exclusions.ownerNames.contains($0) } ?? false + || (resolvedApp?.bundleIdentifier).flatMap(normalizedText).map { exclusions.bundleIds.contains($0) } ?? false + ) + let targetApp = shouldPreferVisibleNonExcludedWindow + ? firstVisibleOwnerName(excluding: exclusions) + : (resolvedApp?.localizedName ?? requestedApp ?? NSWorkspace.shared.frontmostApplication?.localizedName) + let cursorLocation = CGEvent(source: nil)?.location ?? .zero + let displays = activeDisplays() + let window = selectedWindow( + ownerName: targetApp, + exactTitle: requestedWindowTitle, + titleContains: requestedWindowTitleContains, + index: requestedWindowIndex, + exclusions: exclusions + ) + if windowSelectionRequested && window == nil { + throw HelperError.applicationNotFound("requested_window") + } + + let anchorPoint = window.map { CGPoint(x: $0.primary.bounds.midX, y: $0.primary.bounds.midY) } ?? cursorLocation + let display = displayForPoint(anchorPoint, displays: displays) + let useWindow: Bool + switch requestedMode { + case "window": + useWindow = window != nil + case "display": + useWindow = false + default: + useWindow = + windowSelectionRequested + || ( + !preferWindow + && !exclusions.isEmpty + && hostFrontmostExcluded + && window != nil + ) + || (preferWindow && window != nil) + } + guard requestedMode != "window" || window != nil else { + throw HelperError.invalidEnv("CODEX_GUI_CAPTURE_MODE") + } + + let captureMode = useWindow ? "window" : "display" + let captureBounds = useWindow ? window!.captureBounds.integral : display.bounds.integral + let captureWidth = Int(captureBounds.width.rounded()) + let captureHeight = Int(captureBounds.height.rounded()) + let imagePath = try captureScreenshot(bounds: captureBounds) + let payload = ObservePayload( + appName: targetApp, + display: DisplayDescriptor(index: display.index, bounds: rect(display.bounds)), + cursor: point(cursorLocation), + windowId: useWindow ? window?.primary.id : nil, + windowTitle: useWindow ? window?.primary.title : nil, + windowBounds: useWindow ? window.map { rect($0.captureBounds) } : nil, + windowCount: useWindow ? window?.windowCount : nil, + windowCaptureStrategy: useWindow ? window?.captureStrategy : nil, + hostSelfExcludeApplied: !exclusions.isEmpty, + hostFrontmostExcluded: hostFrontmostExcluded, + hostFrontmostAppName: frontmostApp?.localizedName, + hostFrontmostBundleId: frontmostApp?.bundleIdentifier, + captureMode: captureMode, + captureBounds: rect(captureBounds), + captureWidth: captureWidth, + captureHeight: captureHeight, + imagePath: imagePath + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(payload) + FileHandle.standardOutput.write(data) +} + +func makeMouseEvent(_ type: CGEventType, point: CGPoint, button: CGMouseButton = .left) throws -> CGEvent { + guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: button) else { + throw HelperError.eventCreationFailed("mouse_\(type.rawValue)") + } + return event +} + +func post(_ event: CGEvent) { + event.post(tap: .cghidEventTap) +} + +func moveCursor(to point: CGPoint) throws { + post(try makeMouseEvent(.mouseMoved, point: point)) + usleep(30_000) +} + +func leftDown(at point: CGPoint, clickState: Int64 = 1) throws { + let event = try makeMouseEvent(.leftMouseDown, point: point) + event.setIntegerValueField(.mouseEventClickState, value: clickState) + post(event) +} + +func leftUp(at point: CGPoint, clickState: Int64 = 1) throws { + let event = try makeMouseEvent(.leftMouseUp, point: point) + event.setIntegerValueField(.mouseEventClickState, value: clickState) + post(event) +} + +func rightDown(at point: CGPoint, clickState: Int64 = 1) throws { + let event = try makeMouseEvent(.rightMouseDown, point: point, button: .right) + event.setIntegerValueField(.mouseEventClickState, value: clickState) + post(event) +} + +func rightUp(at point: CGPoint, clickState: Int64 = 1) throws { + let event = try makeMouseEvent(.rightMouseUp, point: point, button: .right) + event.setIntegerValueField(.mouseEventClickState, value: clickState) + post(event) +} + +func drag(from start: CGPoint, to end: CGPoint, steps: Int, durationMs: Int) throws { + let stepCount = max(1, steps) + let sleepMicros = useconds_t(max(10_000, (durationMs * 1_000) / stepCount)) + try moveCursor(to: start) + try leftDown(at: start) + usleep(50_000) + for index in 1...stepCount { + let progress = Double(index) / Double(stepCount) + let point = CGPoint( + x: start.x + ((end.x - start.x) * progress), + y: start.y + ((end.y - start.y) * progress) + ) + post(try makeMouseEvent(.leftMouseDragged, point: point)) + usleep(sleepMicros) + } + try leftUp(at: end) +} + +func postKeyboardEvent(keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags = []) throws { + guard let event = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: keyDown) else { + throw HelperError.eventCreationFailed("keyboard_\(keyCode)_\(keyDown ? "down" : "up")") + } + event.flags = flags + post(event) +} + +let shiftKeyCode: CGKeyCode = 56 +let controlKeyCode: CGKeyCode = 59 +let optionKeyCode: CGKeyCode = 58 +let commandKeyCode: CGKeyCode = 55 + +func modifierKeySequence(for flags: CGEventFlags) -> [(keyCode: CGKeyCode, mask: CGEventFlags)] { + var sequence: [(keyCode: CGKeyCode, mask: CGEventFlags)] = [] + if flags.contains(.maskControl) { sequence.append((controlKeyCode, .maskControl)) } + if flags.contains(.maskAlternate) { sequence.append((optionKeyCode, .maskAlternate)) } + if flags.contains(.maskCommand) { sequence.append((commandKeyCode, .maskCommand)) } + if flags.contains(.maskShift) { sequence.append((shiftKeyCode, .maskShift)) } + return sequence +} + +func pressKeyCode(_ keyCode: CGKeyCode, flags: CGEventFlags = []) throws { + let modifierSequence = modifierKeySequence(for: flags) + var activeFlags: CGEventFlags = [] + for modifier in modifierSequence { + activeFlags.formUnion(modifier.mask) + try postKeyboardEvent(keyCode: modifier.keyCode, keyDown: true, flags: activeFlags) + usleep(20_000) + } + try postKeyboardEvent(keyCode: keyCode, keyDown: true, flags: flags) + usleep(20_000) + try postKeyboardEvent(keyCode: keyCode, keyDown: false, flags: flags) + for modifier in modifierSequence.reversed() { + activeFlags.subtract(modifier.mask) + try postKeyboardEvent(keyCode: modifier.keyCode, keyDown: false, flags: activeFlags.union(modifier.mask)) + usleep(20_000) + } + usleep(20_000) +} + +func pressKeyCodeRepeated(_ keyCode: CGKeyCode, count: Int, flags: CGEventFlags = []) throws { + let repeatCount = max(0, count) + for _ in 0.. TISInputSource? { + let properties = [kTISPropertyInputSourceID as String: id] as CFDictionary + guard let listRef = TISCreateInputSourceList(properties, false)?.takeRetainedValue() else { + return nil + } + let sources = listRef as NSArray + return sources.firstObject as! TISInputSource? +} + +func selectPhysicalTypingInputSource() -> TISInputSource? { + let previous = TISCopyCurrentKeyboardInputSource()?.takeRetainedValue() + for sourceID in preferredPhysicalTypingInputSourceIDs { + guard let source = findInputSource(by: sourceID) else { continue } + if TISSelectInputSource(source) == noErr { + usleep(250_000) + break + } + } + return previous +} + +func restoreInputSource(_ source: TISInputSource?) { + guard let source else { return } + _ = TISSelectInputSource(source) + usleep(250_000) +} + +let baseKeyCodes: [Character: CGKeyCode] = [ + "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, + "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, + "3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28, + "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "l": 37, "j": 38, + "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44, "n": 45, "m": 46, ".": 47, + " ": 49 +] + +let shiftedKeyCodes: [Character: CGKeyCode] = [ + "A": 0, "S": 1, "D": 2, "F": 3, "H": 4, "G": 5, "Z": 6, "X": 7, "C": 8, "V": 9, + "B": 11, "Q": 12, "W": 13, "E": 14, "R": 15, "Y": 16, "T": 17, "!": 18, "@": 19, + "#": 20, "$": 21, "^": 22, "%": 23, "+": 24, "(": 25, "&": 26, "_": 27, "*": 28, + ")": 29, "}": 30, "O": 31, "U": 32, "{": 33, "I": 34, "P": 35, "L": 37, "J": 38, + "\"": 39, "K": 40, ":": 41, "|": 42, "<": 43, "?": 44, "N": 45, "M": 46, ">": 47 +] + +func keyPressForCharacter(_ character: Character) throws -> (keyCode: CGKeyCode, flags: CGEventFlags) { + if let keyCode = baseKeyCodes[character] { + return (keyCode, []) + } + if let keyCode = shiftedKeyCodes[character] { + return (keyCode, .maskShift) + } + throw HelperError.eventCreationFailed("unsupported_physical_key_\(character)") +} + +func typeUnicodeText(_ text: String) throws { + let utf16 = Array(text.utf16) + guard !utf16.isEmpty else { return } + guard + let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) + else { + throw HelperError.eventCreationFailed("unicode_text") + } + keyDown.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) + keyUp.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) + post(keyDown) + usleep(30_000) + post(keyUp) + usleep(30_000) +} + +func typePhysicalKeyText(_ text: String) throws { + let previousInputSource = selectPhysicalTypingInputSource() + defer { restoreInputSource(previousInputSource) } + for character in text { + let keyPress = try keyPressForCharacter(character) + try pressKeyCode(keyPress.keyCode, flags: keyPress.flags) + usleep(90_000) + } +} + +/// Pastes `text` via the system clipboard and Cmd+V. +/// +/// **Limitations:** +/// - Only preserves `.string` clipboard content; images, files, and rich +/// text on the clipboard before this call will be lost. +/// - The 150ms delay after Cmd+V is a best-effort wait; slow applications +/// may not process the paste in time, causing the restored clipboard to +/// overwrite the paste. +func pasteText(_ text: String) throws { + let pasteboard = NSPasteboard.general + let previousString = pasteboard.string(forType: .string) + pasteboard.clearContents() + guard pasteboard.setString(text, forType: .string) else { + throw HelperError.eventCreationFailed("pasteboard_set") + } + usleep(100_000) + try pressKeyCode(9, flags: .maskCommand) + usleep(150_000) + pasteboard.clearContents() + if let previousString { + _ = pasteboard.setString(previousString, forType: .string) + } +} + +func releaseAllModifiers() throws { + for keyCode in [shiftKeyCode, controlKeyCode, optionKeyCode, commandKeyCode] { + try postKeyboardEvent(keyCode: keyCode, keyDown: false, flags: []) + usleep(10_000) + } +} + +func releaseMouseButtons() { + let point = CGEvent(source: nil)?.location ?? .zero + if CGEventSource.buttonState(.combinedSessionState, button: .left) { + try? leftUp(at: point) + } + if CGEventSource.buttonState(.combinedSessionState, button: .right) { + try? rightUp(at: point) + } +} + +func parseModifierFlags(_ raw: String?) -> CGEventFlags { + guard let raw else { return [] } + var flags: CGEventFlags = [] + for modifier in raw.split(separator: ",").map({ $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }) { + switch modifier { + case "command": + flags.insert(.maskCommand) + case "shift": + flags.insert(.maskShift) + case "option", "alt": + flags.insert(.maskAlternate) + case "control", "ctrl": + flags.insert(.maskControl) + default: + continue + } + } + return flags +} + +func handleEvent() throws { + let requestedApp = trimmedEnv("CODEX_GUI_APP") + if shouldActivateApp() { + try activateApplication(named: requestedApp) + } + + switch env("CODEX_GUI_EVENT_MODE") { + case "move_cursor": + let point = CGPoint(x: try requiredDouble("CODEX_GUI_X"), y: try requiredDouble("CODEX_GUI_Y")) + let settleMs = max(1, optionalInt("CODEX_GUI_SETTLE_MS") ?? 200) + try moveCursor(to: point) + usleep(useconds_t(settleMs * 1_000)) + print("cg_move_cursor") + case "click": + let point = CGPoint(x: try requiredDouble("CODEX_GUI_X"), y: try requiredDouble("CODEX_GUI_Y")) + try moveCursor(to: point) + try leftDown(at: point) + usleep(30_000) + try leftUp(at: point) + print("cg_click") + case "click_and_hold": + let point = CGPoint(x: try requiredDouble("CODEX_GUI_X"), y: try requiredDouble("CODEX_GUI_Y")) + let holdMs = max(1, optionalInt("CODEX_GUI_HOLD_MS") ?? 650) + try moveCursor(to: point) + try leftDown(at: point) + usleep(useconds_t(holdMs * 1_000)) + try leftUp(at: point) + print("cg_click_and_hold") + case "right_click": + let point = CGPoint(x: try requiredDouble("CODEX_GUI_X"), y: try requiredDouble("CODEX_GUI_Y")) + try moveCursor(to: point) + try rightDown(at: point) + usleep(30_000) + try rightUp(at: point) + print("cg_right_click") + case "double_click": + let point = CGPoint(x: try requiredDouble("CODEX_GUI_X"), y: try requiredDouble("CODEX_GUI_Y")) + try moveCursor(to: point) + for state in [Int64(1), Int64(2)] { + try leftDown(at: point, clickState: state) + usleep(30_000) + try leftUp(at: point, clickState: state) + usleep(80_000) + } + print("cg_double_click") + case "drag": + let start = CGPoint(x: try requiredDouble("CODEX_GUI_FROM_X"), y: try requiredDouble("CODEX_GUI_FROM_Y")) + let end = CGPoint(x: try requiredDouble("CODEX_GUI_TO_X"), y: try requiredDouble("CODEX_GUI_TO_Y")) + let durationMs = try requiredInt("CODEX_GUI_DURATION_MS") + let steps = try requiredInt("CODEX_GUI_STEPS") + try drag(from: start, to: end, steps: steps, durationMs: durationMs) + print("cg_drag") + case "scroll": + if let x = optionalDouble("CODEX_GUI_X"), let y = optionalDouble("CODEX_GUI_Y") { + try moveCursor(to: CGPoint(x: x, y: y)) + } + let vertical = try requiredInt32("CODEX_GUI_SCROLL_Y") + let horizontal = try requiredInt32("CODEX_GUI_SCROLL_X") + let scrollUnit = trimmedEnv("CODEX_GUI_SCROLL_UNIT") + let units: CGScrollEventUnit = scrollUnit == "pixel" ? .pixel : .line + guard let event = CGEvent( + scrollWheelEvent2Source: nil, + units: units, + wheelCount: 2, + wheel1: vertical, + wheel2: horizontal, + wheel3: 0 + ) else { + throw HelperError.eventCreationFailed("scroll") + } + post(event) + print("cg_scroll") + case "type_text": + let rawText = env("CODEX_GUI_TEXT") + let shouldReplace = env("CODEX_GUI_REPLACE") != "0" + let shouldSubmit = env("CODEX_GUI_SUBMIT") == "1" + let typeStrategy = trimmedEnv("CODEX_GUI_TYPE_STRATEGY") ?? "unicode" + let clearRepeat = max(0, optionalInt("CODEX_GUI_CLEAR_REPEAT") ?? (typeStrategy == "clipboard_paste" ? 48 : 0)) + if shouldReplace { + if typeStrategy == "clipboard_paste" || typeStrategy == "physical_keys" { + try pressKeyCodeRepeated(51, count: clearRepeat) + } else { + try pressKeyCode(0, flags: .maskCommand) + } + } + if typeStrategy == "clipboard_paste" { + try pasteText(rawText) + } else if typeStrategy == "physical_keys" { + try typePhysicalKeyText(rawText) + } else { + try typeUnicodeText(rawText) + } + if shouldSubmit { + try pressKeyCode(36) + } + print("cg_type_text") + case "key_press": + let keyCode = CGKeyCode(try requiredInt("CODEX_GUI_KEY_CODE")) + let repeatCount = max(1, optionalInt("CODEX_GUI_REPEAT") ?? 1) + let flags = parseModifierFlags(trimmedEnv("CODEX_GUI_MODIFIERS")) + for _ in 0..= 0 { + fflush(stdout) + stdoutPipe?.fileHandleForWriting.closeFile() + dup2(originalStdout, STDOUT_FILENO) + close(originalStdout) + originalStdout = -1 + stdoutPipe = nil + } + response = ServeResponse(status: "error", output: nil, error: "\(error)") + } + + // Write response as a single JSON line to stdout. + if let responseData = try? encoder.encode(response), + let responseString = String(data: responseData, encoding: .utf8) { + fputs(responseString + "\n", stdout) + fflush(stdout) + } else { + fputs("{\"status\":\"error\",\"error\":\"failed to encode response\"}\n", stdout) + fflush(stdout) + } + } +} + +/// Dispatch a single command (shared between one-shot and serve modes). +func dispatchCommand(_ command: String) throws { + switch command { + case "capture-context": + try handleCaptureContext() + case "observe": + try handleObserve() + case "event": + try handleEvent() + case "cleanup": + try handleCleanup() + case "redact-host-windows": + try handleRedactHostWindows() + case "hide-other-apps": + try handleHideOtherApps() + case "unhide-apps": + try handleUnhideApps() + default: + throw HelperError.invalidCommand(command) + } +} + +// MARK: - Main + +do { + let command = CommandLine.arguments.dropFirst().first ?? "" + if command == "serve" { + serve() + } else if command == "monitor-escape" { + // monitor-escape is inherently long-lived and must NOT go through + // serve mode — it uses its own stdout protocol. + try monitorEscape() + } else { + try dispatchCommand(command) + } +} catch { + fputs("Codex native GUI helper failed: \(error)\n", stderr) + exit(1) +} diff --git a/codex-rs/core/src/tools/handlers/gui/platform.rs b/codex-rs/core/src/tools/handlers/gui/platform.rs new file mode 100644 index 000000000000..d25032148547 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/platform.rs @@ -0,0 +1,323 @@ +use std::collections::HashMap; +use std::io::BufRead; +use std::io::BufReader; +use std::path::PathBuf; +use std::process::Child; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread::JoinHandle; + +use crate::function_tool::FunctionCallError; + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +use super::GUI_UNSUPPORTED_MESSAGE; +use super::HelperCaptureContext; +use super::HelperRect; +use super::ObserveState; +use super::WindowSelector; +use super::readiness::GUI_TOOL_NAMES; +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +use super::readiness::GuiEnvironmentReadinessCheck; +use super::readiness::GuiEnvironmentReadinessSnapshot; +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +use super::readiness::GuiReadinessStatus; +use super::readiness::GuiToolCapability; + +#[cfg(target_os = "macos")] +pub(super) mod platform_macos; +#[cfg(target_os = "windows")] +pub(super) mod platform_windows; + +pub(super) trait GuiPlatform: Send + Sync { + fn readiness_snapshot(&self) -> GuiEnvironmentReadinessSnapshot; + + fn tool_capabilities(&self) -> HashMap<&'static str, GuiToolCapability> { + GUI_TOOL_NAMES + .into_iter() + .map(|tool_name| { + ( + tool_name, + GuiToolCapability { + enabled: true, + reason: None, + targetless_only: false, + }, + ) + }) + .collect() + } + + fn resolve_helper_binary(&self) -> Result; + + fn cleanup_input_state(&self) -> Result<(), FunctionCallError> { + Ok(()) + } + + /// Hide all regular applications except the target app and the host + /// terminal. Returns the PIDs that were hidden so they can be restored + /// later via [`unhide_apps`]. + fn hide_other_apps(&self, _app: Option<&str>) -> Result, FunctionCallError> { + Ok(vec![]) + } + + /// Restore applications that were previously hidden by [`hide_other_apps`]. + fn unhide_apps(&self, _pids: &[i32]) -> Result<(), FunctionCallError> { + Ok(()) + } + + fn start_emergency_stop_monitor( + &self, + ) -> Result, FunctionCallError> { + Ok(None) + } + + fn capture_context( + &self, + app: Option<&str>, + activate_app: bool, + window_selection: Option<&WindowSelector>, + ) -> Result; + + fn capture_region( + &self, + bounds: &HelperRect, + target_width: u32, + target_height: u32, + ) -> Result, FunctionCallError>; + + fn observe( + &self, + app: Option<&str>, + activate_app: bool, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, + prefer_window_when_available: bool, + ) -> Result; + + fn run_event( + &self, + event_mode: &str, + app: Option<&str>, + float_env: &[(&str, f64)], + string_env: &[(&str, String)], + ) -> Result<(), FunctionCallError>; + + fn run_system_events_type( + &self, + app: Option<&str>, + window_selection: Option<&WindowSelector>, + text: &str, + replace: bool, + submit: bool, + strategy: &str, + ) -> Result<(), FunctionCallError>; +} + +pub(super) struct PlatformObservation { + pub(super) state: ObserveState, + pub(super) image_bytes: Vec, +} + +pub(super) struct GuiEmergencyStopMonitor { + child: Arc>, + triggered: Arc, + /// When set, the next Escape detection is treated as an expected + /// programmatic keypress (e.g. `gui_key("Escape")`) rather than a + /// user-initiated emergency stop. The monitor thread clears this + /// flag instead of setting `triggered`. + suppress_next: Arc, + output_thread: Option>, +} + +impl GuiEmergencyStopMonitor { + pub(super) fn from_child(mut child: Child) -> Result { + let stdout = child.stdout.take().ok_or_else(|| { + FunctionCallError::RespondToModel( + "native GUI emergency stop monitor did not expose stdout".to_string(), + ) + })?; + let child = Arc::new(Mutex::new(child)); + let triggered = Arc::new(AtomicBool::new(false)); + let suppress_next = Arc::new(AtomicBool::new(false)); + let thread_triggered = Arc::clone(&triggered); + let thread_suppress = Arc::clone(&suppress_next); + let output_thread = std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines().map_while(Result::ok) { + if line.trim().eq_ignore_ascii_case("escape") { + // If the caller told us to expect this Escape, consume + // the suppression flag instead of triggering a stop. + if thread_suppress + .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + continue; + } + thread_triggered.store(true, Ordering::SeqCst); + break; + } + } + }); + Ok(Self { + child, + triggered, + suppress_next, + output_thread: Some(output_thread), + }) + } + + pub(super) fn triggered(&self) -> bool { + self.triggered.load(Ordering::SeqCst) + } + + /// Tell the monitor to treat the next detected Escape keypress as an + /// expected programmatic event rather than a user abort signal. Must + /// be called *before* the Escape key event is injected. + pub(super) fn expect_escape(&self) { + self.suppress_next.store(true, Ordering::SeqCst); + } + + pub(super) fn stop(&mut self) { + if let Ok(mut child) = self.child.lock() { + let _ = child.kill(); + let _ = child.wait(); + } + if let Some(output_thread) = self.output_thread.take() { + let _ = output_thread.join(); + } + } +} + +impl Drop for GuiEmergencyStopMonitor { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +struct UnsupportedGuiPlatform; + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +impl GuiPlatform for UnsupportedGuiPlatform { + fn readiness_snapshot(&self) -> GuiEnvironmentReadinessSnapshot { + GuiEnvironmentReadinessSnapshot { + status: "unsupported", + checks: vec![GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Unsupported, + summary: GUI_UNSUPPORTED_MESSAGE.to_string(), + detail: None, + }], + } + } + + fn resolve_helper_binary(&self) -> Result { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } + + fn tool_capabilities(&self) -> HashMap<&'static str, GuiToolCapability> { + GUI_TOOL_NAMES + .into_iter() + .map(|tool_name| { + ( + tool_name, + GuiToolCapability { + enabled: false, + reason: Some(GUI_UNSUPPORTED_MESSAGE.to_string()), + targetless_only: false, + }, + ) + }) + .collect() + } + + fn capture_context( + &self, + _app: Option<&str>, + _activate_app: bool, + _window_selection: Option<&WindowSelector>, + ) -> Result { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } + + fn capture_region( + &self, + _bounds: &HelperRect, + _target_width: u32, + _target_height: u32, + ) -> Result, FunctionCallError> { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } + + fn run_event( + &self, + _event_mode: &str, + _app: Option<&str>, + _float_env: &[(&str, f64)], + _string_env: &[(&str, String)], + ) -> Result<(), FunctionCallError> { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } + + fn observe( + &self, + _app: Option<&str>, + _activate_app: bool, + _capture_mode: Option<&str>, + _window_selection: Option<&WindowSelector>, + _prefer_window_when_available: bool, + ) -> Result { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } + + fn run_system_events_type( + &self, + _app: Option<&str>, + _window_selection: Option<&WindowSelector>, + _text: &str, + _replace: bool, + _submit: bool, + _strategy: &str, + ) -> Result<(), FunctionCallError> { + Err(FunctionCallError::RespondToModel( + GUI_UNSUPPORTED_MESSAGE.to_string(), + )) + } +} + +pub(super) fn default_gui_platform() -> &'static dyn GuiPlatform { + #[cfg(target_os = "macos")] + { + static PLATFORM: platform_macos::MacOSPlatform = platform_macos::MacOSPlatform; + return &PLATFORM; + } + + #[cfg(target_os = "windows")] + { + static PLATFORM: platform_windows::WindowsPlatform = platform_windows::WindowsPlatform; + return &PLATFORM; + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + static PLATFORM: UnsupportedGuiPlatform = UnsupportedGuiPlatform; + &PLATFORM + } +} + +pub(super) fn resolve_gui_platform_tool_capabilities() -> HashMap<&'static str, GuiToolCapability> { + default_gui_platform().tool_capabilities() +} diff --git a/codex-rs/core/src/tools/handlers/gui/platform_macos.rs b/codex-rs/core/src/tools/handlers/gui/platform_macos.rs new file mode 100644 index 000000000000..1c9fd8d7cf61 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/platform_macos.rs @@ -0,0 +1,1046 @@ +use serde::Deserialize; +use serde::Serialize; +use serde_json::from_str as parse_json; +use sha1::Digest; +use sha1::Sha1; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::io::BufRead; +use std::io::BufReader; +use std::io::Write; +use std::path::PathBuf; +use std::process::Child; +use std::process::Command; +use std::process::Stdio; +use std::sync::Mutex; +use std::sync::OnceLock; +use tempfile::tempdir; + +use crate::function_tool::FunctionCallError; + +use super::super::HelperCaptureContext; +use super::super::HelperRect; +use super::super::HostCaptureExclusionState; +use super::super::ObserveState; +use super::super::WindowSelector; +use super::super::readiness::GuiEnvironmentReadinessCheck; +use super::super::readiness::GuiEnvironmentReadinessSnapshot; +use super::super::readiness::GuiReadinessStatus; +use super::GuiEmergencyStopMonitor; +use super::GuiPlatform; +use super::PlatformObservation; + +const TYPE_SYSTEM_EVENTS_SCRIPT: &str = include_str!("type_system_events.applescript"); +const HELPER_SOURCE: &str = include_str!("native_helper.swift"); +const DEFAULT_NATIVE_TYPE_CLEAR_REPEAT: i64 = 48; +const DEFAULT_SYSTEM_EVENTS_PASTE_PRE_DELAY_MS: i64 = 220; +const DEFAULT_SYSTEM_EVENTS_PASTE_POST_DELAY_MS: i64 = 650; +const DEFAULT_SYSTEM_EVENTS_KEYSTROKE_CHAR_DELAY_MS: i64 = 55; + +pub(super) struct MacOSPlatform; + +// --------------------------------------------------------------------------- +// Persistent helper subprocess (JSON-line protocol over stdin/stdout) +// --------------------------------------------------------------------------- + +/// JSON-line request sent to the Swift helper in `serve` mode. +#[derive(Serialize)] +struct ServeRequest<'a> { + command: &'a str, + env: Option>, +} + +/// JSON-line response received from the Swift helper in `serve` mode. +#[derive(Deserialize)] +struct ServeResponse { + status: String, + output: Option, + error: Option, +} + +/// A persistent connection to the Swift helper subprocess running in +/// `serve` mode. Communication happens over newline-delimited JSON on +/// stdin/stdout. +struct HelperConnection { + child: Child, + reader: BufReader, +} + +impl HelperConnection { + /// Spawn a new helper in `serve` mode. + fn spawn(helper_path: &PathBuf) -> Result { + let mut child = Command::new(helper_path) + .arg("serve") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to start persistent GUI helper: {error}" + )) + })?; + let stdout = child.stdout.take().ok_or_else(|| { + FunctionCallError::RespondToModel( + "persistent GUI helper did not expose stdout".to_string(), + ) + })?; + Ok(Self { + child, + reader: BufReader::new(stdout), + }) + } + + /// Send a command with parameters and return the captured output. + fn send( + &mut self, + command: &str, + env: &[(&str, String)], + ) -> Result { + let env_map: HashMap<&str, &str> = + env.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let request = ServeRequest { + command, + env: if env_map.is_empty() { + None + } else { + Some(env_map) + }, + }; + let request_json = serde_json::to_string(&request).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode helper request: {error}" + )) + })?; + + let stdin = self.child.stdin.as_mut().ok_or_else(|| { + FunctionCallError::RespondToModel( + "persistent GUI helper stdin is closed".to_string(), + ) + })?; + writeln!(stdin, "{request_json}").map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to write to persistent GUI helper: {error}" + )) + })?; + stdin.flush().map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to flush persistent GUI helper stdin: {error}" + )) + })?; + + let mut line = String::new(); + self.reader.read_line(&mut line).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to read from persistent GUI helper: {error}" + )) + })?; + if line.is_empty() { + return Err(FunctionCallError::RespondToModel( + "persistent GUI helper closed stdout unexpectedly".to_string(), + )); + } + + let response: ServeResponse = parse_json(line.trim()).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode helper response: {error} (raw: {line})" + )) + })?; + match response.status.as_str() { + "ok" => Ok(response.output.unwrap_or_default()), + _ => Err(FunctionCallError::RespondToModel(format!( + "native GUI helper failed: {}", + response.error.unwrap_or_else(|| "unknown error".to_string()) + ))), + } + } + + /// Check whether the child process is still alive. + fn is_alive(&mut self) -> bool { + matches!(self.child.try_wait(), Ok(None)) + } +} + +impl Drop for HelperConnection { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Global persistent helper connection, lazily initialized on first use. +static HELPER_CONN: OnceLock>> = OnceLock::new(); + +/// Send a command to the persistent helper, spawning it if necessary. +/// Falls back to one-shot execution if the persistent connection fails. +fn run_helper_persistent( + platform: &MacOSPlatform, + command: &str, + env: &[(&str, String)], +) -> Result { + let helper_path = platform.resolve_helper_binary()?; + let mutex = HELPER_CONN.get_or_init(|| Mutex::new(None)); + let mut guard = mutex.lock().map_err(|_| { + FunctionCallError::RespondToModel( + "persistent GUI helper lock poisoned".to_string(), + ) + })?; + + // Spawn if needed or if previous connection died. + if guard.as_mut().map_or(true, |conn| !conn.is_alive()) { + *guard = Some(HelperConnection::spawn(&helper_path)?); + } + + let conn = guard.as_mut().unwrap(); + match conn.send(command, env) { + Ok(output) => Ok(output), + Err(e) => { + // Connection may have broken mid-request. Drop it so the + // next call respawns, and fall back to one-shot execution + // for this request. + *guard = None; + tracing::warn!("persistent GUI helper failed, falling back to one-shot: {e}"); + run_helper_oneshot(platform, command, env) + } + } +} + +/// Original one-shot helper execution (spawn → run → exit). +fn run_helper_oneshot( + platform: &MacOSPlatform, + command: &str, + env: &[(&str, String)], +) -> Result { + let helper_path = platform.resolve_helper_binary()?; + let mut cmd = Command::new(helper_path); + cmd.arg(command); + for (key, value) in env { + cmd.env(key, value); + } + let output = cmd.output().map_err(|error| { + FunctionCallError::RespondToModel(format!("failed to execute native GUI helper: {error}")) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "native GUI helper failed: {}", + stderr.trim() + ))); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +#[derive(Clone)] +struct ScriptWindowSelection { + title: Option, + title_contains: Option, + index: Option, + bounds: Option, +} + +pub(crate) fn normalize_capture_mode_env(value: &str) -> Option<&'static str> { + let trimmed = value + .trim() + .trim_matches(|char: char| !char.is_ascii_alphabetic()); + if trimmed.eq_ignore_ascii_case("display") { + Some("display") + } else if trimmed.eq_ignore_ascii_case("window") { + Some("window") + } else { + None + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ObserveHelperOutput { + app_name: Option, + display: super::super::HelperDisplayDescriptor, + window_title: Option, + window_count: Option, + window_capture_strategy: Option, + host_self_exclude_applied: Option, + host_frontmost_excluded: Option, + host_frontmost_app_name: Option, + host_frontmost_bundle_id: Option, + capture_mode: String, + capture_bounds: HelperRect, + capture_width: u32, + capture_height: u32, + image_path: String, +} + +const KNOWN_HOST_OWNER_NAME_HINTS: &[(&str, &[&str])] = &[ + ("com.apple.Terminal", &["Terminal"]), + ("com.googlecode.iterm2", &["iTerm2"]), + ("com.openai.codex", &["Codex", "Codex Desktop"]), + ("dev.warp.Warp-Stable", &["Warp"]), +]; + +fn parse_delimited_values(value: Option) -> Vec { + value + .unwrap_or_default() + .split(|char: char| char == ',' || char.is_ascii_whitespace()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn normalize_identity(value: &str) -> Option { + let trimmed = value.trim().to_lowercase(); + (!trimmed.is_empty()).then_some(trimmed) +} + +fn dedupe_case_insensitive(values: Vec) -> Vec { + let mut seen = BTreeSet::new(); + let mut deduped = Vec::new(); + for value in values { + let Some(identity) = normalize_identity(&value) else { + continue; + }; + if seen.insert(identity) { + deduped.push(value); + } + } + deduped +} + +fn title_case_token(value: &str) -> String { + let mut chars = value.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + format!("{}{}", first.to_ascii_uppercase(), chars.as_str()) +} + +fn derive_owner_name_hints_from_bundle_id(bundle_id: Option<&str>) -> Vec { + let Some(bundle_id) = bundle_id else { + return Vec::new(); + }; + let Some(tail) = bundle_id.split('.').next_back() else { + return Vec::new(); + }; + let normalized = tail.replace(".app", "").replace(['-', '_', '.'], " "); + let normalized = normalized.split_whitespace().collect::>().join(" "); + if normalized.is_empty() { + return Vec::new(); + } + dedupe_case_insensitive(vec![ + normalized.clone(), + normalized + .split(' ') + .map(title_case_token) + .collect::>() + .join(" "), + ]) +} + +fn owner_name_hints_for_bundle_id(bundle_id: Option<&str>) -> Vec { + let mut hints = Vec::new(); + if let Some(bundle_id) = bundle_id { + for (known_bundle_id, known_hints) in KNOWN_HOST_OWNER_NAME_HINTS { + if bundle_id.eq_ignore_ascii_case(known_bundle_id) { + hints.extend(known_hints.iter().map(|hint| (*hint).to_string())); + } + } + } + hints.extend(derive_owner_name_hints_from_bundle_id(bundle_id)); + hints +} + +fn requested_app_targets_host( + app: Option<&str>, + bundle_ids: &[String], + owner_names: &[String], +) -> bool { + let Some(app) = app.and_then(normalize_identity) else { + return false; + }; + bundle_ids + .iter() + .filter_map(|candidate| normalize_identity(candidate)) + .any(|candidate| candidate == app) + || owner_names + .iter() + .filter_map(|candidate| normalize_identity(candidate)) + .any(|candidate| candidate == app) +} + +fn build_host_window_exclusion_env(app: Option<&str>) -> Vec<(&'static str, String)> { + let host_bundle_id = std::env::var("__CFBundleIdentifier").ok(); + let configured_bundle_ids = + parse_delimited_values(std::env::var("CODEX_GUI_EXCLUDED_BUNDLE_IDS").ok()); + let configured_owner_names = + parse_delimited_values(std::env::var("CODEX_GUI_EXCLUDED_OWNER_NAMES").ok()); + let owner_names = dedupe_case_insensitive( + configured_owner_names + .into_iter() + .chain(owner_name_hints_for_bundle_id(host_bundle_id.as_deref())) + .chain(parse_delimited_values( + std::env::var("CODEX_INTERNAL_ORIGINATOR_OVERRIDE").ok(), + )) + .chain(parse_delimited_values(std::env::var("TERM_PROGRAM").ok())) + .collect(), + ); + let bundle_ids = dedupe_case_insensitive( + configured_bundle_ids + .into_iter() + .chain(host_bundle_id.clone()) + .collect(), + ); + if (bundle_ids.is_empty() && owner_names.is_empty()) + || requested_app_targets_host(app, &bundle_ids, &owner_names) + { + return Vec::new(); + } + + let mut env = Vec::new(); + if !bundle_ids.is_empty() { + env.push(("CODEX_GUI_AUTO_EXCLUDED_BUNDLE_IDS", bundle_ids.join(","))); + } + if !owner_names.is_empty() { + env.push(("CODEX_GUI_AUTO_EXCLUDED_OWNER_NAMES", owner_names.join(","))); + } + env +} + +impl GuiPlatform for MacOSPlatform { + fn readiness_snapshot(&self) -> GuiEnvironmentReadinessSnapshot { + let mut checks = vec![GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Ok, + summary: "macOS GUI runtime is available on this host.".to_string(), + detail: None, + }]; + + match run_swift_boolean_check( + r#"import ApplicationServices +print(AXIsProcessTrusted() ? "1" : "0")"#, + ) { + Ok(true) => checks.push(GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Ok, + summary: "Accessibility permission is granted for native GUI input.".to_string(), + detail: None, + }), + Ok(false) => checks.push(GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Error, + summary: "Accessibility permission is not granted for native GUI input." + .to_string(), + detail: None, + }), + Err(error) => checks.push(GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Warn, + summary: "Could not confirm Accessibility permission state.".to_string(), + detail: Some(error.to_string()), + }), + } + + match run_swift_boolean_check( + r#"import CoreGraphics +print(CGPreflightScreenCaptureAccess() ? "1" : "0")"#, + ) { + Ok(true) => checks.push(GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Ok, + summary: "Screen Recording permission is granted for GUI screenshots.".to_string(), + detail: None, + }), + Ok(false) => checks.push(GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Error, + summary: "Screen Recording permission is not granted for GUI screenshots." + .to_string(), + detail: None, + }), + Err(error) => checks.push(GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Warn, + summary: "Could not confirm Screen Recording permission state.".to_string(), + detail: Some(error.to_string()), + }), + } + + match self.resolve_helper_binary() { + Ok(path) => checks.push(GuiEnvironmentReadinessCheck { + id: "native_helper", + label: "Native GUI Helper", + status: GuiReadinessStatus::Ok, + summary: "Native GUI helper is ready for capture and input execution.".to_string(), + detail: Some(path.display().to_string()), + }), + Err(error) => checks.push(GuiEnvironmentReadinessCheck { + id: "native_helper", + label: "Native GUI Helper", + status: GuiReadinessStatus::Error, + summary: "Native GUI helper is unavailable.".to_string(), + detail: Some(error.to_string()), + }), + } + + let status = if checks + .iter() + .all(|check| check.status == GuiReadinessStatus::Unsupported) + { + "unsupported" + } else if checks + .iter() + .any(|check| check.status == GuiReadinessStatus::Error) + { + "blocked" + } else if checks + .iter() + .any(|check| check.status == GuiReadinessStatus::Warn) + { + "degraded" + } else { + "ready" + }; + GuiEnvironmentReadinessSnapshot { status, checks } + } + + fn resolve_helper_binary(&self) -> Result { + static CACHED_PATH: std::sync::OnceLock = std::sync::OnceLock::new(); + if let Some(path) = CACHED_PATH.get() + && path.exists() + { + return Ok(path.clone()); + } + + let mut hasher = Sha1::new(); + hasher.update(HELPER_SOURCE.as_bytes()); + let hash = format!("{:x}", hasher.finalize()); + let helper_dir = std::env::temp_dir() + .join("codex-gui-native-helper") + .join(&hash[..16]); + let source_path = helper_dir.join("codex-gui-native-helper.swift"); + let binary_path = helper_dir.join("codex-gui-native-helper"); + + if binary_path.exists() { + let _ = CACHED_PATH.set(binary_path.clone()); + return Ok(binary_path); + } + + std::fs::create_dir_all(&helper_dir).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to create native GUI helper directory: {error}" + )) + })?; + std::fs::write(&source_path, HELPER_SOURCE).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to write native GUI helper source: {error}" + )) + })?; + + // Compile to a temporary path and atomically rename to avoid TOCTOU + // races when multiple GUI tool calls resolve concurrently. + let tmp_dir = tempdir().map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to create temp dir for native GUI helper compilation: {error}" + )) + })?; + let tmp_binary = tmp_dir.path().join("codex-gui-native-helper"); + + let output = Command::new("swiftc") + .arg(&source_path) + .arg("-o") + .arg(&tmp_binary) + .output() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to run `swiftc` for native GUI helper: {error}" + )) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "failed to compile native GUI helper. Ensure Xcode Command Line Tools are installed and `swiftc` is available. {}", + stderr.trim() + ))); + } + + // Atomic rename; if another caller raced us, the last rename wins + // with a fully-formed binary. + std::fs::rename(&tmp_binary, &binary_path).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to install native GUI helper binary: {error}" + )) + })?; + + let _ = CACHED_PATH.set(binary_path.clone()); + Ok(binary_path) + } + + fn cleanup_input_state(&self) -> Result<(), FunctionCallError> { + run_helper( + self, + "cleanup", + &[ + ("CODEX_GUI_RELEASE_MOUSE", "1".to_string()), + ("CODEX_GUI_RELEASE_MODIFIERS", "1".to_string()), + ], + ) + .map(|_| ()) + } + + fn hide_other_apps(&self, app: Option<&str>) -> Result, FunctionCallError> { + let mut env = vec![]; + if let Some(app) = app.filter(|a| !a.trim().is_empty()) { + env.push(("CODEX_GUI_APP", app.to_string())); + } + let output = run_helper(self, "hide-other-apps", &env)?; + #[derive(serde::Deserialize)] + struct HideResult { + #[serde(rename = "hiddenPids")] + hidden_pids: Vec, + } + let result: HideResult = parse_json(&output).map_err(|e| { + FunctionCallError::RespondToModel(format!( + "failed to parse hide-other-apps result: {e}" + )) + })?; + Ok(result.hidden_pids) + } + + fn unhide_apps(&self, pids: &[i32]) -> Result<(), FunctionCallError> { + if pids.is_empty() { + return Ok(()); + } + let pids_str = pids + .iter() + .map(|p| p.to_string()) + .collect::>() + .join(","); + run_helper( + self, + "unhide-apps", + &[("CODEX_GUI_HIDDEN_PIDS", pids_str)], + ) + .map(|_| ()) + } + + fn start_emergency_stop_monitor( + &self, + ) -> Result, FunctionCallError> { + let helper_path = self.resolve_helper_binary()?; + let child = Command::new(helper_path) + .arg("monitor-escape") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to start native GUI emergency stop monitor: {error}" + )) + })?; + Ok(Some(GuiEmergencyStopMonitor::from_child(child)?)) + } + + fn capture_context( + &self, + app: Option<&str>, + activate_app: bool, + window_selection: Option<&WindowSelector>, + ) -> Result { + let mut env = vec![( + "CODEX_GUI_ACTIVATE_APP", + if activate_app { "1" } else { "0" }.to_string(), + )]; + if let Some(app) = app.filter(|app| !app.trim().is_empty()) { + env.push(("CODEX_GUI_APP", app.to_string())); + } + env.extend(build_host_window_exclusion_env(app)); + if let Some(window_selection) = window_selection { + if let Some(title) = &window_selection.title { + env.push(("CODEX_GUI_WINDOW_TITLE", title.clone())); + } + if let Some(title_contains) = &window_selection.title_contains { + env.push(("CODEX_GUI_WINDOW_TITLE_CONTAINS", title_contains.clone())); + } + if let Some(index) = window_selection.index { + env.push(("CODEX_GUI_WINDOW_INDEX", index.to_string())); + } + } + + let output = run_helper(self, "capture-context", &env)?; + parse_json::(&output).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode native GUI capture context: {error}" + )) + }) + } + + fn capture_region( + &self, + bounds: &HelperRect, + _target_width: u32, + _target_height: u32, + ) -> Result, FunctionCallError> { + let dir = tempdir().map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to create temporary directory for GUI screenshot: {error}" + )) + })?; + let image_path = dir.path().join("codex-gui-observe.png"); + let region = format!( + "{},{},{},{}", + bounds.x.round(), + bounds.y.round(), + bounds.width.round(), + bounds.height.round() + ); + + let output = Command::new("screencapture") + .args(["-x", "-C", "-R", ®ion]) + .arg(&image_path) + .output() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to execute `screencapture`: {error}" + )) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "macOS screenshot capture failed: {}", + stderr.trim() + ))); + } + + let bytes = std::fs::read(&image_path).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to read captured screenshot: {error}" + )) + })?; + + Ok(bytes) + } + + fn observe( + &self, + app: Option<&str>, + activate_app: bool, + capture_mode: Option<&str>, + window_selection: Option<&WindowSelector>, + prefer_window_when_available: bool, + ) -> Result { + let capture_mode = capture_mode.and_then(normalize_capture_mode_env); + let mut env = vec![ + ( + "CODEX_GUI_ACTIVATE_APP", + if activate_app { "1" } else { "0" }.to_string(), + ), + ( + "CODEX_GUI_PREFER_WINDOW", + if prefer_window_when_available { + "1" + } else { + "0" + } + .to_string(), + ), + ]; + if let Some(app) = app.filter(|app| !app.trim().is_empty()) { + env.push(("CODEX_GUI_APP", app.to_string())); + } + env.extend(build_host_window_exclusion_env(app)); + if let Some(capture_mode) = capture_mode.filter(|mode| !mode.trim().is_empty()) { + env.push(("CODEX_GUI_CAPTURE_MODE", capture_mode.to_string())); + } + if let Some(window_selection) = window_selection { + if let Some(title) = &window_selection.title { + env.push(("CODEX_GUI_WINDOW_TITLE", title.clone())); + } + if let Some(title_contains) = &window_selection.title_contains { + env.push(("CODEX_GUI_WINDOW_TITLE_CONTAINS", title_contains.clone())); + } + if let Some(index) = window_selection.index { + env.push(("CODEX_GUI_WINDOW_INDEX", index.to_string())); + } + } + + let output = run_helper(self, "observe", &env)?; + let observed: ObserveHelperOutput = parse_json(&output).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to decode native GUI observation payload: {error}" + )) + })?; + let host_self_exclude_adjusted = capture_mode.is_none() + && window_selection.is_none() + && !prefer_window_when_available + && observed.host_self_exclude_applied.unwrap_or(false) + && observed.host_frontmost_excluded.unwrap_or(false) + && observed.capture_mode == "window" + && observed.window_title.is_some(); + let mut redaction_count = 0_i64; + if observed.capture_mode == "display" + && observed.host_self_exclude_applied.unwrap_or(false) + && observed.host_frontmost_excluded.unwrap_or(false) + { + redaction_count = + redact_host_windows(self, &observed.image_path, &observed.capture_bounds, app)?; + } + let image_path = PathBuf::from(&observed.image_path); + let bytes = std::fs::read(&image_path).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to read native GUI observation image: {error}" + )) + })?; + let _ = std::fs::remove_file(&image_path); + let image_bytes = bytes; + let (image_width, image_height) = image_dimensions(&image_bytes)?; + let state = ObserveState { + capture: super::super::CaptureArtifact { + origin_x: observed.capture_bounds.x, + origin_y: observed.capture_bounds.y, + width: observed.capture_width, + height: observed.capture_height, + image_width, + image_height, + display_index: observed.display.index, + capture_mode: match observed.capture_mode.as_str() { + "window" => super::super::CaptureMode::Window, + _ => super::super::CaptureMode::Display, + }, + window_title: observed.window_title.clone(), + window_count: observed.window_count, + window_capture_strategy: observed.window_capture_strategy.clone(), + host_exclusion: HostCaptureExclusionState { + applied: observed.host_self_exclude_applied.unwrap_or(false), + frontmost_excluded: observed.host_frontmost_excluded.unwrap_or(false), + adjusted: host_self_exclude_adjusted, + frontmost_app_name: observed.host_frontmost_app_name.clone(), + frontmost_bundle_id: observed.host_frontmost_bundle_id.clone(), + redaction_count, + }, + }, + app_name: observed.app_name.clone(), + }; + Ok(PlatformObservation { state, image_bytes }) + } + + fn run_event( + &self, + event_mode: &str, + app: Option<&str>, + float_env: &[(&str, f64)], + string_env: &[(&str, String)], + ) -> Result<(), FunctionCallError> { + let mut env = vec![("CODEX_GUI_EVENT_MODE", event_mode.to_string())]; + if let Some(app) = app.filter(|app| !app.trim().is_empty()) { + env.push(("CODEX_GUI_APP", app.to_string())); + } + for (key, value) in float_env { + env.push(((*key), value.to_string())); + } + for (key, value) in string_env { + env.push(((*key), value.clone())); + } + run_helper(self, "event", &env).map(|_| ()) + } + + fn run_system_events_type( + &self, + app: Option<&str>, + window_selection: Option<&WindowSelector>, + text: &str, + replace: bool, + submit: bool, + strategy: &str, + ) -> Result<(), FunctionCallError> { + let resolved_window_selection = + resolve_script_window_selection(self, app, window_selection)?; + let mut env = Vec::new(); + if let Some(app) = app.filter(|app| !app.trim().is_empty()) { + env.push(("CODEX_GUI_APP", app.to_string())); + } + if let Some(window_selection) = resolved_window_selection.as_ref() { + if let Some(title) = &window_selection.title { + env.push(("CODEX_GUI_WINDOW_TITLE", title.clone())); + } + if let Some(title_contains) = &window_selection.title_contains { + env.push(("CODEX_GUI_WINDOW_TITLE_CONTAINS", title_contains.clone())); + } + if let Some(index) = window_selection.index { + env.push(("CODEX_GUI_WINDOW_INDEX", index.to_string())); + } + if let Some(bounds) = &window_selection.bounds { + env.push(("CODEX_GUI_WINDOW_BOUNDS_X", bounds.x.to_string())); + env.push(("CODEX_GUI_WINDOW_BOUNDS_Y", bounds.y.to_string())); + env.push(("CODEX_GUI_WINDOW_BOUNDS_WIDTH", bounds.width.to_string())); + env.push(("CODEX_GUI_WINDOW_BOUNDS_HEIGHT", bounds.height.to_string())); + } + } + env.push(("CODEX_GUI_TEXT", text.to_string())); + env.push(( + "CODEX_GUI_REPLACE", + if replace { "1" } else { "0" }.to_string(), + )); + env.push(( + "CODEX_GUI_SUBMIT", + if submit { "1" } else { "0" }.to_string(), + )); + let strategy_env = match strategy { + "system_events_keystroke" => "keystroke", + "system_events_keystroke_chars" => "keystroke_chars", + _ => "paste", + }; + env.push(( + "CODEX_GUI_SYSTEM_EVENTS_TYPE_STRATEGY", + strategy_env.to_string(), + )); + if replace { + env.push(( + "CODEX_GUI_CLEAR_REPEAT", + DEFAULT_NATIVE_TYPE_CLEAR_REPEAT.to_string(), + )); + } + env.push(( + "CODEX_GUI_PASTE_PRE_DELAY_MS", + DEFAULT_SYSTEM_EVENTS_PASTE_PRE_DELAY_MS.to_string(), + )); + env.push(( + "CODEX_GUI_PASTE_POST_DELAY_MS", + DEFAULT_SYSTEM_EVENTS_PASTE_POST_DELAY_MS.to_string(), + )); + if strategy == "system_events_keystroke_chars" { + env.push(( + "CODEX_GUI_KEYSTROKE_CHAR_DELAY_MS", + DEFAULT_SYSTEM_EVENTS_KEYSTROKE_CHAR_DELAY_MS.to_string(), + )); + } + + run_apple_script(TYPE_SYSTEM_EVENTS_SCRIPT, &env).map(|_| ()) + } +} + +fn resolve_script_window_selection( + platform: &MacOSPlatform, + app: Option<&str>, + window_selection: Option<&WindowSelector>, +) -> Result, FunctionCallError> { + let Some(window_selection) = window_selection else { + return Ok(None); + }; + let context = platform.capture_context(app, false, Some(window_selection))?; + if context.window_bounds.is_none() { + return Err(FunctionCallError::RespondToModel( + "requested macOS window could not be found for System Events typing".to_string(), + )); + } + Ok(Some(ScriptWindowSelection { + title: context + .window_title + .clone() + .or_else(|| window_selection.title.clone()), + title_contains: if context.window_title.is_some() { + None + } else { + window_selection.title_contains.clone() + }, + index: window_selection.index, + bounds: context.window_bounds.clone(), + })) +} + +fn run_swift_boolean_check(script: &str) -> Result { + let output = Command::new("swift") + .args(["-e", script]) + .output() + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to run Swift GUI readiness check: {error}" + )) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "Swift GUI readiness check failed: {}", + stderr.trim() + ))); + } + match String::from_utf8_lossy(&output.stdout) + .trim() + .to_lowercase() + .as_str() + { + "1" | "true" | "yes" => Ok(true), + "0" | "false" | "no" => Ok(false), + other => Err(FunctionCallError::RespondToModel(format!( + "unexpected Swift GUI readiness check output `{other}`" + ))), + } +} + +fn image_dimensions(bytes: &[u8]) -> Result<(u32, u32), FunctionCallError> { + let image = image::load_from_memory(bytes).map_err(|error| { + FunctionCallError::RespondToModel(format!("failed to decode captured screenshot: {error}")) + })?; + Ok((image.width(), image.height())) +} + +fn run_helper( + platform: &MacOSPlatform, + command: &str, + env: &[(&str, String)], +) -> Result { + run_helper_persistent(platform, command, env) +} + +fn redact_host_windows( + platform: &MacOSPlatform, + image_path: &str, + capture_bounds: &HelperRect, + app: Option<&str>, +) -> Result { + let mut env = vec![ + ("CODEX_GUI_IMAGE_PATH", image_path.to_string()), + ("CODEX_GUI_CAPTURE_X", capture_bounds.x.to_string()), + ("CODEX_GUI_CAPTURE_Y", capture_bounds.y.to_string()), + ("CODEX_GUI_CAPTURE_WIDTH", capture_bounds.width.to_string()), + ( + "CODEX_GUI_CAPTURE_HEIGHT", + capture_bounds.height.to_string(), + ), + ]; + env.extend(build_host_window_exclusion_env(app)); + let output = run_helper(platform, "redact-host-windows", &env)?; + let parsed: serde_json::Value = parse_json(&output).unwrap_or_else(|_| serde_json::json!({})); + Ok(parsed + .get("redactionCount") + .and_then(serde_json::Value::as_i64) + .or_else(|| output.trim().parse::().ok()) + .unwrap_or(0) + .max(0)) +} + +fn run_apple_script(script: &str, env: &[(&str, String)]) -> Result { + let mut command = Command::new("osascript"); + command.args(["-l", "AppleScript", "-e", script]); + for (key, value) in env { + command.env(key, value); + } + let output = command.output().map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to execute `osascript` for GUI typing: {error}" + )) + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(FunctionCallError::RespondToModel(format!( + "macOS System Events typing failed: {}", + stderr.trim() + ))); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} diff --git a/codex-rs/core/src/tools/handlers/gui/platform_windows.rs b/codex-rs/core/src/tools/handlers/gui/platform_windows.rs new file mode 100644 index 000000000000..c9c307f0164e --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/platform_windows.rs @@ -0,0 +1,120 @@ +use std::path::PathBuf; + +use crate::function_tool::FunctionCallError; + +use super::super::HelperCaptureContext; +use super::super::HelperRect; +use super::super::WindowSelector; +use super::super::readiness::GUI_TOOL_NAMES; +use super::super::readiness::GuiEnvironmentReadinessCheck; +use super::super::readiness::GuiEnvironmentReadinessSnapshot; +use super::super::readiness::GuiReadinessStatus; +use super::super::readiness::GuiToolCapability; +use super::GuiPlatform; +use super::PlatformObservation; + +const WINDOWS_GUI_UNIMPLEMENTED_MESSAGE: &str = + "Windows native GUI backend is not implemented yet."; + +pub(super) struct WindowsPlatform; + +impl GuiPlatform for WindowsPlatform { + fn readiness_snapshot(&self) -> GuiEnvironmentReadinessSnapshot { + GuiEnvironmentReadinessSnapshot { + status: "unsupported", + checks: vec![GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Unsupported, + summary: WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + detail: Some( + "Reserved backend slot for a future Windows implementation built around screenshot capture and native input dispatch." + .to_string(), + ), + }], + } + } + + fn resolve_helper_binary(&self) -> Result { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } + + fn tool_capabilities(&self) -> std::collections::HashMap<&'static str, GuiToolCapability> { + GUI_TOOL_NAMES + .into_iter() + .map(|tool_name| { + ( + tool_name, + GuiToolCapability { + enabled: false, + reason: Some(WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string()), + targetless_only: false, + }, + ) + }) + .collect() + } + + fn capture_context( + &self, + _app: Option<&str>, + _activate_app: bool, + _window_selection: Option<&WindowSelector>, + ) -> Result { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } + + fn capture_region( + &self, + _bounds: &HelperRect, + _target_width: u32, + _target_height: u32, + ) -> Result, FunctionCallError> { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } + + fn run_event( + &self, + _event_mode: &str, + _app: Option<&str>, + _float_env: &[(&str, f64)], + _string_env: &[(&str, String)], + ) -> Result<(), FunctionCallError> { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } + + fn observe( + &self, + _app: Option<&str>, + _activate_app: bool, + _capture_mode: Option<&str>, + _window_selection: Option<&WindowSelector>, + _prefer_window_when_available: bool, + ) -> Result { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } + + fn run_system_events_type( + &self, + _app: Option<&str>, + _window_selection: Option<&WindowSelector>, + _text: &str, + _replace: bool, + _submit: bool, + _strategy: &str, + ) -> Result<(), FunctionCallError> { + Err(FunctionCallError::RespondToModel( + WINDOWS_GUI_UNIMPLEMENTED_MESSAGE.to_string(), + )) + } +} diff --git a/codex-rs/core/src/tools/handlers/gui/provider.rs b/codex-rs/core/src/tools/handlers/gui/provider.rs new file mode 100644 index 000000000000..ca9c0a3e0cc8 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/provider.rs @@ -0,0 +1,40 @@ +use async_trait::async_trait; + +use crate::function_tool::FunctionCallError; +use crate::tools::context::ToolInvocation; + +use super::GuiTargetRequest; +use super::ObserveState; +use super::ResolvedTarget; +use super::grounding::resolve_grounded_target; + +#[async_trait] +pub(super) trait GuiGroundingProvider { + async fn ground( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + image_bytes: &[u8], + ) -> Result, FunctionCallError>; +} + +pub(super) struct ModelGuiGroundingProvider; + +#[async_trait] +impl GuiGroundingProvider for ModelGuiGroundingProvider { + async fn ground( + &self, + invocation: &ToolInvocation, + request: GuiTargetRequest<'_>, + capture_state: &ObserveState, + image_bytes: &[u8], + ) -> Result, FunctionCallError> { + resolve_grounded_target(invocation, request, capture_state, image_bytes).await + } +} + +pub(super) fn default_gui_grounding_provider() -> &'static ModelGuiGroundingProvider { + static PROVIDER: ModelGuiGroundingProvider = ModelGuiGroundingProvider; + &PROVIDER +} diff --git a/codex-rs/core/src/tools/handlers/gui/readiness.rs b/codex-rs/core/src/tools/handlers/gui/readiness.rs new file mode 100644 index 000000000000..e6f280f5561a --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/readiness.rs @@ -0,0 +1,274 @@ +use std::collections::HashMap; +use std::sync::Mutex as StdMutex; +use std::time::Instant; + +use crate::function_tool::FunctionCallError; +use crate::tools::context::ToolInvocation; + +use super::GUI_UNSUPPORTED_MESSAGE; +use super::platform::default_gui_platform; +use super::platform::resolve_gui_platform_tool_capabilities; +use super::supports_image_input; + +static READINESS_CACHE: std::sync::LazyLock< + StdMutex>, +> = std::sync::LazyLock::new(|| StdMutex::new(None)); +const READINESS_CACHE_TTL_SECS: u64 = 30; + +const GUI_ACCESSIBILITY_REQUIRED_REASON: &str = "Accessibility permission is not granted, so GUI input actions (click, type, scroll, drag, etc.) are unavailable. GUI observation tools (gui_observe) may still work. Grant Accessibility permission in System Settings > Privacy & Security > Accessibility."; +const GUI_GROUNDING_REQUIRED_REASON: &str = "Visual grounding is not configured, so grounding-based GUI actions (click, drag, etc.) are unavailable. Keyboard-only tool (gui_key) and targetless gui_type still work."; +const GUI_NATIVE_HELPER_REQUIRED_REASON: &str = "Native GUI helper is unavailable, so GUI tools cannot run. Verify the helper binary is installed and accessible."; +const GUI_SCREEN_CAPTURE_REQUIRED_REASON: &str = "Screen Recording permission is not granted, so screenshot-based GUI actions are unavailable. Keyboard-only tool (gui_key) still works. Grant Screen Recording permission in System Settings > Privacy & Security > Screen Recording."; +const GUI_SCREEN_CAPTURE_TARGETLESS_ONLY_REASON: &str = "Screen Recording permission is not granted, so this tool only supports targetless usage (omit the `target` parameter). Keyboard-only tool (gui_key) still works."; +const GUI_TARGETLESS_ONLY_REASON: &str = "Visual grounding is not configured, so this tool only supports targetless usage (omit the `target` parameter). It will operate on the current surface or focused control."; + +pub(super) const GUI_TOOL_NAMES: [&str; 8] = [ + "gui_observe", + "gui_click", + "gui_drag", + "gui_scroll", + "gui_type", + "gui_key", + "gui_wait", + "gui_move", +]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum GuiReadinessStatus { + Ok, + Warn, + Error, + Unsupported, +} + +#[derive(Clone, Debug)] +pub(super) struct GuiEnvironmentReadinessCheck { + pub(super) id: &'static str, + #[allow(dead_code)] + pub(super) label: &'static str, + pub(super) status: GuiReadinessStatus, + #[allow(dead_code)] + pub(super) summary: String, + #[allow(dead_code)] + pub(super) detail: Option, +} + +#[derive(Clone, Debug)] +pub(super) struct GuiEnvironmentReadinessSnapshot { + #[allow(dead_code)] + pub(super) status: &'static str, + pub(super) checks: Vec, +} + +#[derive(Clone, Debug)] +pub(super) struct GuiToolCapability { + pub(super) enabled: bool, + pub(super) reason: Option, + pub(super) targetless_only: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct GuiRuntimeCapabilitySnapshot { + #[allow(dead_code)] + pub(super) platform_supported: bool, + #[allow(dead_code)] + pub(super) grounding_available: bool, + #[allow(dead_code)] + pub(super) native_helper_available: bool, + #[allow(dead_code)] + pub(super) screen_capture_available: bool, + #[allow(dead_code)] + pub(super) input_available: bool, + #[allow(dead_code)] + pub(super) enabled_tool_names: Vec<&'static str>, + #[allow(dead_code)] + pub(super) disabled_tool_names: Vec<&'static str>, + pub(super) tool_availability: HashMap<&'static str, GuiToolCapability>, +} + +pub(super) fn resolve_gui_readiness_snapshot() -> GuiEnvironmentReadinessSnapshot { + let mut cache = READINESS_CACHE.lock().expect("readiness cache poisoned"); + if let Some((snapshot, created_at)) = cache.as_ref() { + if snapshot.status == "ready" && created_at.elapsed().as_secs() < READINESS_CACHE_TTL_SECS { + return snapshot.clone(); + } + } + let snapshot = default_gui_platform().readiness_snapshot(); + if snapshot.status == "ready" { + *cache = Some((snapshot.clone(), Instant::now())); + } else { + *cache = None; + } + snapshot +} + +fn resolve_readiness_check_status( + snapshot: &GuiEnvironmentReadinessSnapshot, + check_id: &str, +) -> Option { + snapshot + .checks + .iter() + .find(|check| check.id == check_id) + .map(|check| check.status) +} + +pub(super) fn resolve_gui_runtime_capabilities( + grounding_available: bool, + readiness: &GuiEnvironmentReadinessSnapshot, + platform_tool_availability: Option<&HashMap<&'static str, GuiToolCapability>>, +) -> GuiRuntimeCapabilitySnapshot { + let platform_supported = resolve_readiness_check_status(readiness, "platform") + != Some(GuiReadinessStatus::Unsupported); + let native_helper_available = platform_supported + && resolve_readiness_check_status(readiness, "native_helper") + != Some(GuiReadinessStatus::Error); + let input_available = native_helper_available + && resolve_readiness_check_status(readiness, "accessibility") + != Some(GuiReadinessStatus::Error); + let screen_capture_available = native_helper_available + && resolve_readiness_check_status(readiness, "screen_recording") + != Some(GuiReadinessStatus::Error); + + let mut tool_availability = HashMap::new(); + for tool_name in GUI_TOOL_NAMES { + let mut capability = platform_tool_availability + .and_then(|tool_support| tool_support.get(tool_name)) + .cloned() + .unwrap_or(GuiToolCapability { + enabled: true, + reason: None, + targetless_only: false, + }); + if !capability.enabled { + tool_availability.insert(tool_name, capability); + continue; + } + if !platform_supported { + capability = GuiToolCapability { + enabled: false, + reason: Some(GUI_UNSUPPORTED_MESSAGE.to_string()), + targetless_only: false, + }; + } else if !native_helper_available { + capability = GuiToolCapability { + enabled: false, + reason: Some(GUI_NATIVE_HELPER_REQUIRED_REASON.to_string()), + targetless_only: false, + }; + } else if matches!( + tool_name, + "gui_click" | "gui_drag" | "gui_scroll" | "gui_type" | "gui_key" | "gui_move" + ) && !input_available + { + capability = GuiToolCapability { + enabled: false, + reason: Some(GUI_ACCESSIBILITY_REQUIRED_REASON.to_string()), + targetless_only: false, + }; + } else if matches!( + tool_name, + "gui_observe" | "gui_click" | "gui_drag" | "gui_wait" + ) && !screen_capture_available + { + capability = GuiToolCapability { + enabled: false, + reason: Some(GUI_SCREEN_CAPTURE_REQUIRED_REASON.to_string()), + targetless_only: false, + }; + } else if !screen_capture_available && matches!(tool_name, "gui_scroll" | "gui_type") { + if !capability.targetless_only { + capability = GuiToolCapability { + enabled: true, + reason: Some(GUI_SCREEN_CAPTURE_TARGETLESS_ONLY_REASON.to_string()), + targetless_only: true, + }; + } + } else if matches!(tool_name, "gui_key" | "gui_move") { + capability = GuiToolCapability { + enabled: true, + reason: capability.reason.clone(), + targetless_only: capability.targetless_only, + }; + } else if grounding_available { + capability = GuiToolCapability { + enabled: true, + reason: capability.reason.clone(), + targetless_only: capability.targetless_only, + }; + } else if matches!(tool_name, "gui_observe" | "gui_scroll" | "gui_type") { + if !capability.targetless_only { + capability = GuiToolCapability { + enabled: true, + reason: Some(GUI_TARGETLESS_ONLY_REASON.to_string()), + targetless_only: true, + }; + } + } else if matches!(tool_name, "gui_click" | "gui_drag" | "gui_wait") { + capability = GuiToolCapability { + enabled: false, + reason: Some(GUI_GROUNDING_REQUIRED_REASON.to_string()), + targetless_only: false, + }; + } + tool_availability.insert(tool_name, capability); + } + let enabled_tool_names = GUI_TOOL_NAMES + .into_iter() + .filter(|tool_name| tool_availability[*tool_name].enabled) + .collect(); + let disabled_tool_names = GUI_TOOL_NAMES + .into_iter() + .filter(|tool_name| !tool_availability[*tool_name].enabled) + .collect(); + GuiRuntimeCapabilitySnapshot { + platform_supported, + grounding_available, + native_helper_available, + screen_capture_available, + input_available, + enabled_tool_names, + disabled_tool_names, + tool_availability, + } +} + +pub(super) async fn enforce_gui_tool_capability( + invocation: &ToolInvocation, + tool_name: &'static str, + targeted: bool, +) -> Result<(), FunctionCallError> { + let has_image_input = supports_image_input(invocation); + let capabilities = tokio::task::spawn_blocking(move || { + let readiness = resolve_gui_readiness_snapshot(); + let platform_tool_availability = resolve_gui_platform_tool_capabilities(); + resolve_gui_runtime_capabilities( + has_image_input, + &readiness, + Some(&platform_tool_availability), + ) + }) + .await + .map_err(|error| { + FunctionCallError::RespondToModel(format!("GUI readiness check failed: {error}")) + })?; + let Some(capability) = capabilities.tool_availability.get(tool_name) else { + return Ok(()); + }; + if !capability.enabled { + return Err(FunctionCallError::RespondToModel( + capability + .reason + .clone() + .unwrap_or_else(|| format!("`{tool_name}` is currently unavailable.")), + )); + } + if capability.targetless_only && targeted { + return Err(FunctionCallError::RespondToModel( + capability.reason.clone().unwrap_or_else(|| { + format!("`{tool_name}` currently supports only targetless usage.") + }), + )); + } + Ok(()) +} diff --git a/codex-rs/core/src/tools/handlers/gui/session.rs b/codex-rs/core/src/tools/handlers/gui/session.rs new file mode 100644 index 000000000000..e55af8d199d8 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/session.rs @@ -0,0 +1,305 @@ +use serde::Deserialize; +use serde::Serialize; +use std::fs::OpenOptions; +use std::io::ErrorKind; +use std::path::PathBuf; + +use crate::function_tool::FunctionCallError; +use crate::tools::context::ToolInvocation; + +use super::platform::GuiEmergencyStopMonitor; +use super::platform::default_gui_platform; + +pub(super) struct GuiActionSession { + conversation_id: String, + process_id: u32, + lock: Option, + emergency_stop_monitor: Option, + /// PIDs of applications hidden at the start of the action via + /// [`hide_other_apps`]. Restored in [`Drop`]. + hidden_pids: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GuiPhysicalResourceLockHolder { + session_id: Option, + pid: Option, + acquired_at: Option, + tool_name: Option, +} + +#[derive(Clone, Debug, Serialize)] +struct GuiPhysicalResourceLockPayload<'a> { + session_id: &'a str, + pid: u32, + acquired_at: u64, + tool_name: &'a str, +} + +struct FileGuiPhysicalResourceLock { + path: PathBuf, +} + +impl FileGuiPhysicalResourceLock { + fn acquire( + path: PathBuf, + session_id: &str, + process_id: u32, + tool_name: &str, + ) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to prepare GUI lock directory: {error}" + )) + })?; + } + + let payload = serde_json::to_vec(&GuiPhysicalResourceLockPayload { + session_id, + pid: process_id, + acquired_at: unix_timestamp_ms(), + tool_name, + }) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to encode GUI physical lock payload: {error}" + )) + })?; + + for attempt in 0..4 { + if attempt > 0 { + std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt.min(3)))); + } + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(mut file) => { + use std::io::Write; + file.write_all(&payload).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "failed to write GUI physical lock: {error}" + )) + })?; + return Ok(Self { path }); + } + Err(error) if error.kind() == ErrorKind::AlreadyExists => { + let holder = read_lock_holder(&path); + if holder + .as_ref() + .and_then(|holder| holder.session_id.as_deref()) + == Some(session_id) + && holder.as_ref().and_then(|holder| holder.pid) == Some(process_id) + { + return Ok(Self { path }); + } + if !is_process_alive(holder.as_ref().and_then(|holder| holder.pid)) { + let _ = std::fs::remove_file(&path); + continue; + } + return Err(FunctionCallError::RespondToModel(format!( + "GUI physical resources are currently locked by {}.", + format_lock_holder(holder.as_ref()) + ))); + } + Err(error) => { + return Err(FunctionCallError::RespondToModel(format!( + "failed to acquire GUI physical lock: {error}" + ))); + } + } + } + + Err(FunctionCallError::RespondToModel( + "GUI physical resources are currently locked by another GUI session.".to_string(), + )) + } + + fn release(&self, session_id: &str, process_id: u32) { + let holder = read_lock_holder(&self.path); + if holder + .as_ref() + .and_then(|holder| holder.session_id.as_deref()) + != Some(session_id) + || holder.as_ref().and_then(|holder| holder.pid) != Some(process_id) + { + return; + } + let _ = std::fs::remove_file(&self.path); + } +} + +impl GuiActionSession { + pub(super) fn throw_if_emergency_stopped(&self) -> Result<(), FunctionCallError> { + if self + .emergency_stop_monitor + .as_ref() + .is_some_and(GuiEmergencyStopMonitor::triggered) + { + return Err(FunctionCallError::RespondToModel( + "Stopped the GUI action after Escape was pressed.".to_string(), + )); + } + Ok(()) + } + + /// Hide all visible applications except the target app and the host + /// terminal so that the GUI action operates on a clean screen. Hidden + /// PIDs are recorded and automatically restored when this session is + /// dropped. + pub(super) fn hide_other_apps(&mut self, app: Option<&str>) { + if let Ok(pids) = default_gui_platform().hide_other_apps(app) { + self.hidden_pids = pids; + } + } + + /// Notify the emergency stop monitor that the next Escape keypress is + /// an expected programmatic event (e.g. `gui_key("Escape")`), not a + /// user abort. Must be called *before* injecting the Escape key. + pub(super) fn expect_escape(&self) { + if let Some(monitor) = &self.emergency_stop_monitor { + monitor.expect_escape(); + } + } +} + +impl Drop for GuiActionSession { + fn drop(&mut self) { + if let Some(monitor) = &mut self.emergency_stop_monitor { + monitor.stop(); + } + let _ = default_gui_platform().cleanup_input_state(); + if !self.hidden_pids.is_empty() { + let _ = default_gui_platform().unhide_apps(&self.hidden_pids); + } + if let Some(lock) = &self.lock { + lock.release(&self.conversation_id, self.process_id); + } + } +} + +pub(super) fn begin_gui_action_session( + invocation: &ToolInvocation, + tool_name: &'static str, + acquire_lock: bool, +) -> Result { + let conversation_id = invocation.session.conversation_id.to_string(); + let process_id = std::process::id(); + let lock = if acquire_lock { + Some(FileGuiPhysicalResourceLock::acquire( + gui_lock_path(invocation), + &conversation_id, + process_id, + tool_name, + )?) + } else { + None + }; + let emergency_stop_monitor = if acquire_lock { + match default_gui_platform().start_emergency_stop_monitor() { + Ok(monitor) => monitor, + Err(error) => { + if let Some(lock) = &lock { + lock.release(&conversation_id, process_id); + } + return Err(error); + } + } + } else { + None + }; + Ok(GuiActionSession { + conversation_id, + process_id, + lock, + emergency_stop_monitor, + hidden_pids: Vec::new(), + }) +} + +fn gui_lock_path(invocation: &ToolInvocation) -> PathBuf { + std::env::var_os("CODEX_GUI_LOCK_PATH") + .map(PathBuf::from) + .unwrap_or_else(|| { + invocation + .turn + .config + .codex_home + .join("gui") + .join("physical-resource.lock") + }) +} + +fn read_lock_holder(path: &PathBuf) -> Option { + let raw = std::fs::read_to_string(path).ok()?; + serde_json::from_str(&raw).ok() +} + +fn format_lock_holder(holder: Option<&GuiPhysicalResourceLockHolder>) -> String { + let Some(holder) = holder else { + return "another GUI session".to_string(); + }; + let mut parts = Vec::new(); + if let Some(tool_name) = holder.tool_name.as_deref() { + parts.push(format!("tool {tool_name}")); + } + if let Some(pid) = holder.pid { + parts.push(format!("pid {pid}")); + } + if let Some(acquired_at) = holder.acquired_at { + parts.push(format!("acquired {acquired_at}")); + } + if parts.is_empty() { + "another GUI session".to_string() + } else { + parts.join(", ") + } +} + +#[cfg(unix)] +fn is_process_alive(pid: Option) -> bool { + let Some(pid) = pid else { + return false; + }; + if pid == 0 { + return false; + } + let result = unsafe { libc::kill(pid as libc::pid_t, 0) }; + if result == 0 { + return true; + } + std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +#[cfg(not(unix))] +fn is_process_alive(pid: Option) -> bool { + pid.is_some() +} + +fn unix_timestamp_ms() -> u64 { + use std::time::SystemTime; + use std::time::UNIX_EPOCH; + + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_lock_holder_matches_user_facing_shape() { + let holder = GuiPhysicalResourceLockHolder { + session_id: Some("other-thread".to_string()), + pid: Some(4242), + acquired_at: None, + tool_name: Some("gui_click".to_string()), + }; + + assert_eq!( + format_lock_holder(Some(&holder)), + "tool gui_click, pid 4242" + ); + } +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/benchmark.rs b/codex-rs/core/src/tools/handlers/gui/tests/benchmark.rs new file mode 100644 index 000000000000..c4329820a53e --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/benchmark.rs @@ -0,0 +1,1344 @@ +use super::*; + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct GroundingBenchmarkCase { + id: String, + element_id: Option, + target: String, + scope: Option, + action: String, + location_hint: Option, + difficulty: String, + prompt_clarity: String, + kind: String, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize)] +struct GroundingBenchmarkArtifacts { + truths: Vec, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct GroundingBenchmarkTruth { + id: String, + #[allow(dead_code)] + element_id: Option, + target: String, + scope: Option, + action: String, + location_hint: Option, + difficulty: String, + prompt_clarity: String, + kind: String, + #[serde(rename = "box")] + box_: GroundingBenchmarkRect, + point: HelperPoint, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize)] +struct GroundingBenchmarkRect { + x: i64, + y: i64, + width: i64, + height: i64, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug)] +struct GroundingBenchmarkMeasurement { + strategy: &'static str, + case_id: String, + kind: String, + prompt_clarity: String, + difficulty: String, + found: bool, + inside: bool, + distance_px: f64, + elapsed_ms: u128, + error: Option, +} + +#[cfg(target_os = "macos")] +struct RenderedGroundingBenchmark { + _tempdir: tempfile::TempDir, + screenshot_path: std::path::PathBuf, + screenshot_bytes: Vec, + truths: Vec, + image_width: u32, + image_height: u32, + logical_width: u32, + logical_height: u32, +} + +#[cfg(target_os = "macos")] +const GUI_DIRECT_COORDINATE_SYSTEM_PROMPT: &str = concat!( + "You are choosing a direct GUI interaction coordinate inside a screenshot. ", + "Return JSON only, following the provided schema exactly. ", + "Choose one best click_point on the visible actionable or editable surface that matches the request. ", + "Do not rely on hidden DOM or implementation details. ", + "If the target is not confidently visible, return `status` = `not_found`, `found` = false, and null coordinates." +); + +#[cfg(target_os = "macos")] +const GUI_DIRECT_COORDINATE_VALIDATION_SYSTEM_PROMPT: &str = concat!( + "You are validating a direct GUI coordinate prediction inside a screenshot. ", + "A highlighted marker indicates the proposed click point and target region. ", + "Return JSON only, following the provided schema exactly. ", + "Approve the prediction only when the marked point clearly lands on the requested target." +); + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Deserialize)] +struct DirectCoordinateValidationResponse { + status: String, + approved: bool, + #[allow(dead_code)] + confidence: Option, + reason: Option, + failure_kind: Option, + retry_hint: Option, +} + +#[cfg(target_os = "macos")] +fn understudy_grounding_fixture_source_path() -> std::path::PathBuf { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../understudy/apps/cli/src/commands/__tests__/gui-benchmark-fixture.ts"); + assert!( + path.exists(), + "expected Understudy grounding fixture at {}", + path.display() + ); + path +} + +#[cfg(target_os = "macos")] +fn extract_understudy_grounding_fixture_html(source: &str) -> String { + let marker = "export const GUI_GROUNDING_BENCHMARK_HTML = String.raw`"; + let start = source + .find(marker) + .map(|index| index + marker.len()) + .expect("fixture html marker should exist"); + let end_marker = "`;\n\nexport async function prepareGuiGroundingBenchmarkPage"; + let end = source[start..] + .find(end_marker) + .map(|index| start + index) + .expect("fixture html closing marker should exist"); + source[start..end].to_string() +} + +#[cfg(target_os = "macos")] +fn extract_understudy_grounding_benchmark_cases(source: &str) -> Vec { + let marker = "export const GUI_GROUNDING_BENCHMARK_CASES: GuiGroundingBenchmarkCase[] = ["; + let start = source + .find(marker) + .map(|index| index + marker.len()) + .expect("benchmark cases marker should exist"); + let end = source[start..] + .find("\n];\n\nexport const GUI_GROUNDING_BENCHMARK_HTML") + .map(|index| start + index) + .expect("benchmark cases closing marker should exist"); + let body = &source[start..end]; + + let mut cases = Vec::new(); + let mut current: HashMap = HashMap::new(); + let mut in_object = false; + + for line in body.lines() { + let trimmed = line.trim(); + if trimmed == "{" { + in_object = true; + current.clear(); + continue; + } + if trimmed == "}," || trimmed == "}" { + if in_object { + let action = current + .remove("action") + .expect("benchmark case should define action"); + let parsed = GroundingBenchmarkCase { + id: current + .remove("id") + .expect("benchmark case should define id"), + element_id: current.remove("elementId"), + target: current + .remove("target") + .expect("benchmark case should define target"), + scope: current.remove("scope"), + action, + location_hint: current.remove("locationHint"), + difficulty: current + .remove("difficulty") + .expect("benchmark case should define difficulty"), + prompt_clarity: current + .remove("promptClarity") + .expect("benchmark case should define promptClarity"), + kind: current + .remove("kind") + .expect("benchmark case should define kind"), + }; + cases.push(parsed); + } + current.clear(); + in_object = false; + continue; + } + if !in_object || trimmed.is_empty() { + continue; + } + let Some((raw_key, raw_value)) = trimmed.split_once(':') else { + continue; + }; + let key = raw_key.trim().to_string(); + let value = raw_value + .trim() + .trim_end_matches(',') + .trim() + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + .expect("benchmark case values should be quoted strings") + .to_string(); + current.insert(key, value); + } + + cases +} + +#[cfg(target_os = "macos")] +fn resolve_grounding_benchmark_renderer_binary() -> std::path::PathBuf { + let source_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("src/tools/handlers/gui/benchmark_renderer.swift"); + let output_path = std::env::temp_dir().join("codex-gui-benchmark-renderer"); + let should_compile = std::fs::metadata(&output_path) + .and_then(|binary| { + let binary_mtime = binary.modified()?; + let source_mtime = std::fs::metadata(&source_path)?.modified()?; + Ok(source_mtime > binary_mtime) + }) + .unwrap_or(true); + if should_compile { + let output = std::process::Command::new("swiftc") + .arg(&source_path) + .arg("-o") + .arg(&output_path) + .output() + .expect("swiftc should launch for benchmark renderer"); + if !output.status.success() { + panic!( + "failed to compile benchmark renderer: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + } + output_path +} + +#[cfg(target_os = "macos")] +fn render_understudy_grounding_benchmark() -> RenderedGroundingBenchmark { + let fixture_source = std::fs::read_to_string(understudy_grounding_fixture_source_path()) + .expect("should read Understudy grounding fixture source"); + let html = extract_understudy_grounding_fixture_html(&fixture_source); + let cases = extract_understudy_grounding_benchmark_cases(&fixture_source); + + let tempdir = tempfile::tempdir().expect("benchmark tempdir"); + let html_path = tempdir.path().join("grounding-benchmark.html"); + let cases_path = tempdir.path().join("grounding-benchmark-cases.json"); + let screenshot_path = tempdir.path().join("grounding-benchmark.png"); + let truths_path = tempdir.path().join("grounding-benchmark-truths.json"); + + std::fs::write(&html_path, html).expect("write benchmark html"); + std::fs::write( + &cases_path, + serde_json::to_vec(&cases).expect("serialize benchmark cases"), + ) + .expect("write benchmark cases json"); + + let renderer = resolve_grounding_benchmark_renderer_binary(); + let output = std::process::Command::new(renderer) + .arg(&html_path) + .arg(&cases_path) + .arg(&screenshot_path) + .arg(&truths_path) + .output() + .expect("benchmark renderer should launch"); + assert!( + output.status.success(), + "benchmark renderer failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + + let screenshot_bytes = std::fs::read(&screenshot_path).expect("read benchmark screenshot"); + let screenshot = + image::load_from_memory(&screenshot_bytes).expect("benchmark screenshot should decode"); + let artifacts: GroundingBenchmarkArtifacts = + serde_json::from_slice(&std::fs::read(&truths_path).expect("read benchmark truths")) + .expect("benchmark truths should deserialize"); + let logical_width = 1280_u32; + let uniform_scale = screenshot.width() as f64 / logical_width as f64; + let logical_height = (screenshot.height() as f64 / uniform_scale).round() as u32; + + RenderedGroundingBenchmark { + _tempdir: tempdir, + screenshot_path, + screenshot_bytes, + truths: artifacts.truths, + image_width: screenshot.width(), + image_height: screenshot.height(), + logical_width, + logical_height, + } +} + +#[cfg(target_os = "macos")] +fn developer_codex_home() -> std::path::PathBuf { + if let Some(path) = std::env::var_os("CODEX_HOME") { + return std::path::PathBuf::from(path); + } + std::env::var_os("HOME") + .map(std::path::PathBuf::from) + .map(|path| path.join(".codex")) + .expect("HOME should be set") +} + +#[cfg(target_os = "macos")] +fn load_live_codex_auth() -> codex_login::CodexAuth { + let auth_manager = + codex_login::AuthManager::shared(developer_codex_home(), true, Default::default()); + auth_manager + .auth_cached() + .expect("Codex auth is required for the live grounding benchmark") +} + +#[cfg(target_os = "macos")] +pub(super) async fn live_grounding_benchmark_session() +-> (Arc, Arc) { + let codex_home = tempfile::tempdir().expect("codex home tempdir"); + let codex_home_path = codex_home.keep(); + let mut config = crate::config::ConfigBuilder::default() + .codex_home(codex_home_path) + .build() + .await + .expect("build benchmark config"); + config.model = Some("gpt-5.4".to_string()); + let _ = config.features.enable(codex_features::Feature::GuiTools); + + let thread_manager = crate::ThreadManager::with_models_provider_for_tests( + load_live_codex_auth(), + crate::ModelProviderInfo::create_openai_provider(None), + ); + let thread = thread_manager + .start_thread(config) + .await + .expect("start live benchmark thread"); + let session = thread.thread.codex.session.clone(); + let turn = session.new_default_turn().await; + (session, turn) +} + +#[cfg(target_os = "macos")] +fn point_inside_benchmark_box(point: &HelperPoint, box_: &GroundingBenchmarkRect) -> bool { + let left = box_.x as f64 - 4.0; + let right = (box_.x + box_.width) as f64 + 4.0; + let top = box_.y as f64 - 4.0; + let bottom = (box_.y + box_.height) as f64 + 4.0; + point.x >= left && point.x <= right && point.y >= top && point.y <= bottom +} + +#[cfg(target_os = "macos")] +fn allowed_point_distance_px(truth: &GroundingBenchmarkTruth) -> f64 { + 160.0_f64.min(24.0_f64.max((truth.box_.width.max(truth.box_.height) as f64) * 0.45)) +} + +#[cfg(target_os = "macos")] +fn summarize_grounding_bucket( + measurements: &[GroundingBenchmarkMeasurement], + label: &str, + filter: impl Fn(&GroundingBenchmarkMeasurement) -> bool, +) { + let bucket: Vec<_> = measurements + .iter() + .filter(|measurement| filter(measurement)) + .collect(); + if bucket.is_empty() { + return; + } + let inside = bucket + .iter() + .filter(|measurement| measurement.inside) + .count(); + let found = bucket + .iter() + .filter(|measurement| measurement.found) + .count(); + let avg_latency_ms = bucket + .iter() + .map(|measurement| measurement.elapsed_ms as f64) + .sum::() + / bucket.len() as f64; + println!( + "[codex-grounding-benchmark] bucket={} total={} found={} inside={} avg={}ms", + label, + bucket.len(), + found, + inside, + avg_latency_ms.round() + ); +} + +#[cfg(target_os = "macos")] +fn summarize_grounding_bucket_for_strategy( + measurements: &[GroundingBenchmarkMeasurement], + strategy: &'static str, + label: &str, + filter: impl Fn(&GroundingBenchmarkMeasurement) -> bool, +) { + summarize_grounding_bucket( + measurements, + &format!("{strategy}:{label}"), + |measurement| measurement.strategy == strategy && filter(measurement), + ); +} + +#[cfg(target_os = "macos")] +fn requested_grounding_case_ids() -> Option> { + let raw = std::env::var("CODEX_GUI_GROUNDING_CASE_IDS").ok()?; + let ids = raw + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + (!ids.is_empty()).then_some(ids) +} + +#[cfg(target_os = "macos")] +fn direct_coordinate_validation_output_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "status": { "type": "string" }, + "approved": { "type": "boolean" }, + "confidence": { "type": ["number", "null"] }, + "reason": { "type": ["string", "null"] }, + "failure_kind": { "type": ["string", "null"] }, + "retry_hint": { "type": ["string", "null"] } + }, + "required": [ + "status", + "approved", + "confidence", + "reason", + "failure_kind", + "retry_hint" + ], + "additionalProperties": false + }) +} + +#[cfg(target_os = "macos")] +fn build_gui_direct_coordinate_prompt( + truth: &GroundingBenchmarkTruth, + capture_state: &ObserveState, + retry_notes: &[String], +) -> String { + let mut prompt = build_gui_grounding_prompt( + benchmark_request_for_truth(truth), + capture_state, + grounding_mode_for_truth(truth), + retry_notes, + false, + ); + prompt.push_str( + "\nDirect-coordinate mode: choose the final tool coordinate yourself instead of delegating to a separate grounding service.", + ); + prompt.push_str( + "\nFirst identify the exact visible target, then place click_point inside its actionable or editable surface.", + ); + prompt.push_str( + "\nDo not return a coordinate from a visually salient but semantically different panel, row, dialog, toolbar, or card.", + ); + prompt.push_str( + "\nFor buttons, tabs, icon controls, and checkboxes, prefer the visible hit target itself over nearby labels, card bodies, or whitespace.", + ); + prompt.push_str( + "\nFor fields, place click_point inside the editable text-entry area, not on the label, icon, or surrounding panel.", + ); + prompt.push_str( + "\nIf the request is ambiguous and several candidates partially match, prefer the one whose visible scope and local context best fit the request; otherwise return not_found.", + ); + prompt +} + +#[cfg(target_os = "macos")] +fn append_direct_coordinate_retry_notes( + retry_notes: &mut Vec, + truth: &GroundingBenchmarkTruth, + round: usize, + predictor_reason: Option<&str>, + validation: Option<&DirectCoordinateValidationResponse>, + model_point: Option<&HelperPoint>, + model_bbox: Option<&GroundingBoundingBox>, +) { + if let Some(reason) = predictor_reason { + let note = format!("Round {round} predictor rationale: {reason}"); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } + if let Some(validation) = validation { + if let Some(reason) = validation.reason.as_deref() { + let note = format!("Round {round} validator rejected the candidate: {reason}"); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } + if let Some(failure_kind) = validation.failure_kind.as_deref() { + let note = format!("Round {round} validator failure kind: {failure_kind}"); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + let guidance = match failure_kind { + "wrong_region" | "scope_mismatch" => Some( + "Search a different visible area or panel instead of staying near the previous candidate.", + ), + "wrong_control" => Some( + "Keep the same semantic goal, but choose a different visible control that serves it.", + ), + "wrong_point" => Some( + "Stay on the matched control, but move the click point onto the inner actionable or editable surface.", + ), + "state_mismatch" => Some( + "Re-check the visible control state and pick the candidate whose current state best matches the request.", + ), + "partial_visibility" => Some( + "Only resolve a partially visible target when a safe interaction point is clearly visible inside the visible portion.", + ), + _ => None, + }; + if let Some(guidance) = guidance + && !retry_notes.iter().any(|existing| existing == guidance) + { + retry_notes.push(guidance.to_string()); + } + } + if let Some(retry_hint) = validation.retry_hint.as_deref() { + let note = format!("Round {round} retry hint: {retry_hint}"); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } + } + if let Some(model_point) = model_point { + let note = format!( + "Round {round} rejected_point_image_pixels: ({}, {})", + model_point.x.round(), + model_point.y.round() + ); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } + if let Some(model_bbox) = model_bbox { + let note = format!( + "Round {round} rejected_bbox_image_pixels: ({}, {}, {}, {})", + model_bbox.x1.round(), + model_bbox.y1.round(), + model_bbox.x2.round(), + model_bbox.y2.round() + ); + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } + for note in build_not_found_retry_notes(benchmark_request_for_truth(truth), round) { + if !retry_notes.iter().any(|existing| existing == ¬e) { + retry_notes.push(note); + } + } +} + +#[cfg(target_os = "macos")] +fn grounding_mode_for_truth(truth: &GroundingBenchmarkTruth) -> &'static str { + if truth.difficulty == "complex" { + "complex" + } else { + "single" + } +} + +#[cfg(target_os = "macos")] +fn tool_name_for_truth(truth: &GroundingBenchmarkTruth) -> &'static str { + if truth.action == "type" { + "gui_type" + } else { + "gui_click" + } +} + +#[cfg(target_os = "macos")] +fn benchmark_request_for_truth<'a>(truth: &'a GroundingBenchmarkTruth) -> GuiTargetRequest<'a> { + GuiTargetRequest { + app: Some("Understudy GUI benchmark"), + capture_mode: Some("window"), + window_selection: None, + target: truth.target.as_str(), + location_hint: truth.location_hint.as_deref(), + scope: truth.scope.as_deref(), + grounding_mode: Some(grounding_mode_for_truth(truth)), + action: if truth.action == "type" { + "type" + } else { + "click" + }, + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + } +} + +#[cfg(target_os = "macos")] +async fn benchmark_measure_separate_grounding_strategy( + invocation: &ToolInvocation, + truth: &GroundingBenchmarkTruth, + capture_state: &ObserveState, + screenshot_bytes: &[u8], +) -> GroundingBenchmarkMeasurement { + let started_at = std::time::Instant::now(); + let outcome = default_gui_grounding_provider() + .ground( + invocation, + benchmark_request_for_truth(truth), + capture_state, + screenshot_bytes, + ) + .await; + let elapsed_ms = started_at.elapsed().as_millis(); + + match outcome { + Ok(Some(resolved)) => { + let dx = resolved.point.x - truth.point.x; + let dy = resolved.point.y - truth.point.y; + let distance_px = (dx * dx + dy * dy).sqrt(); + let inside = point_inside_benchmark_box(&resolved.point, &truth.box_); + println!( + "[codex-grounding-benchmark] strategy=separate_grounding case={} found=true inside={} distance={:.1}px elapsed={}ms target={}", + truth.id, inside, distance_px, elapsed_ms, truth.target + ); + GroundingBenchmarkMeasurement { + strategy: "separate_grounding", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: true, + inside, + distance_px, + elapsed_ms, + error: None, + } + } + Ok(None) => { + println!( + "[codex-grounding-benchmark] strategy=separate_grounding case={} found=false inside=false elapsed={}ms target={}", + truth.id, elapsed_ms, truth.target + ); + GroundingBenchmarkMeasurement { + strategy: "separate_grounding", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms, + error: None, + } + } + Err(error) => { + println!( + "[codex-grounding-benchmark] strategy=separate_grounding case={} error={} elapsed={}ms target={}", + truth.id, error, elapsed_ms, truth.target + ); + GroundingBenchmarkMeasurement { + strategy: "separate_grounding", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms, + error: Some(error.to_string()), + } + } + } +} + +#[cfg(target_os = "macos")] +async fn benchmark_measure_main_model_coordinate_strategy( + invocation: &ToolInvocation, + truth: &GroundingBenchmarkTruth, + capture_state: &ObserveState, + screenshot_bytes: &[u8], +) -> GroundingBenchmarkMeasurement { + let prepared_image = match prepare_grounding_model_image( + screenshot_bytes, + GroundingModelImageConfig { + logical_width: Some(capture_state.capture.width), + logical_height: Some(capture_state.capture.height), + scale_x: Some(capture_state.capture.scale_x()), + scale_y: Some(capture_state.capture.scale_y()), + allow_logical_normalization: true, + }, + "direct coordinate benchmark image", + ) { + Ok(image) => image, + Err(error) => { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: 0, + error: Some(error.to_string()), + }; + } + }; + let mut model_capture_state = capture_state.clone(); + model_capture_state.capture.image_width = prepared_image.model_width; + model_capture_state.capture.image_height = prepared_image.model_height; + let max_rounds = if truth.difficulty == "complex" { 3 } else { 2 }; + let started_at = std::time::Instant::now(); + let mut retry_notes = Vec::new(); + + for round in 1..=max_rounds { + let outcome = request_model_json_from_image::( + invocation, + build_gui_direct_coordinate_prompt(truth, &model_capture_state, &retry_notes), + prepared_image.bytes.as_slice(), + prepared_image.mime_type, + GUI_DIRECT_COORDINATE_SYSTEM_PROMPT, + gui_grounding_output_schema(), + "GUI direct coordinate benchmark", + ) + .await; + + match outcome { + Ok((decision, _, _)) => { + if decision.status == "not_found" || !decision.found { + if round < max_rounds { + append_direct_coordinate_retry_notes( + &mut retry_notes, + truth, + round, + decision.reason.as_deref(), + None, + None, + None, + ); + continue; + } + let elapsed_ms = started_at.elapsed().as_millis(); + println!( + "[codex-grounding-benchmark] strategy=main_model_coordinates case={} found=false inside=false elapsed={}ms target={}", + truth.id, elapsed_ms, truth.target + ); + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms, + error: None, + }; + } + if decision.status != "resolved" { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some(format!( + "unsupported direct coordinate status `{}`", + decision.status + )), + }; + } + let coordinate_space = decision + .coordinate_space + .as_deref() + .unwrap_or("image_pixels"); + if coordinate_space != "image_pixels" { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some(format!("unsupported coordinate space `{coordinate_space}`")), + }; + } + let Some(model_point) = decision.click_point.as_ref() else { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some("resolved without click_point".to_string()), + }; + }; + + let validation_image = match render_validation_overlay( + prepared_image.bytes.as_slice(), + model_point, + decision.bbox.as_ref(), + ) { + Ok(image) => image, + Err(error) => { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some(error.to_string()), + }; + } + }; + let validation = + request_model_json_from_image::( + invocation, + build_gui_grounding_validation_prompt( + benchmark_request_for_truth(truth), + &model_capture_state, + model_point, + decision.bbox.as_ref(), + false, + ), + validation_image.as_slice(), + "image/png", + GUI_DIRECT_COORDINATE_VALIDATION_SYSTEM_PROMPT, + direct_coordinate_validation_output_schema(), + "GUI direct coordinate validation", + ) + .await; + + match validation { + Ok((validation, _, _)) + if validation.status == "approved" && validation.approved => + { + let point = translate_model_point_to_original(&prepared_image, model_point); + let dx = point.x - truth.point.x; + let dy = point.y - truth.point.y; + let distance_px = (dx * dx + dy * dy).sqrt(); + let inside = point_inside_benchmark_box(&point, &truth.box_); + let elapsed_ms = started_at.elapsed().as_millis(); + println!( + "[codex-grounding-benchmark] strategy=main_model_coordinates case={} found=true inside={} distance={:.1}px elapsed={}ms target={}", + truth.id, inside, distance_px, elapsed_ms, truth.target + ); + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: true, + inside, + distance_px, + elapsed_ms, + error: None, + }; + } + Ok((validation, _, _)) => { + if round < max_rounds { + append_direct_coordinate_retry_notes( + &mut retry_notes, + truth, + round, + decision.reason.as_deref(), + Some(&validation), + Some(model_point), + decision.bbox.as_ref(), + ); + continue; + } + let elapsed_ms = started_at.elapsed().as_millis(); + println!( + "[codex-grounding-benchmark] strategy=main_model_coordinates case={} found=false inside=false elapsed={}ms target={}", + truth.id, elapsed_ms, truth.target + ); + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms, + error: None, + }; + } + Err(error) => { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some(error.to_string()), + }; + } + } + } + Err(error) => { + return GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some(error.to_string()), + }; + } + } + } + + GroundingBenchmarkMeasurement { + strategy: "main_model_coordinates", + case_id: truth.id.clone(), + kind: truth.kind.clone(), + prompt_clarity: truth.prompt_clarity.clone(), + difficulty: truth.difficulty.clone(), + found: false, + inside: false, + distance_px: f64::INFINITY, + elapsed_ms: started_at.elapsed().as_millis(), + error: Some("direct coordinate benchmark exhausted without returning".to_string()), + } +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual real grounding benchmark requiring local Codex auth and swiftc"] +async fn macos_gui_grounding_benchmark_matches_understudy_thresholds() { + const MIN_TOTAL_INSIDE_HIT_RATE: f64 = 0.78; + const MIN_EXPLICIT_INSIDE_HIT_RATE: f64 = 0.90; + const MIN_AMBIGUOUS_INSIDE_HIT_RATE: f64 = 0.60; + const MIN_COMPLEX_EXPLICIT_INSIDE_HIT_RATE: f64 = 0.85; + + let rendered = render_understudy_grounding_benchmark(); + let requested_case_ids = requested_grounding_case_ids(); + let truths = rendered + .truths + .iter() + .filter(|truth| match requested_case_ids.as_ref() { + Some(ids) => ids.contains(&truth.id), + None => true, + }) + .cloned() + .collect::>(); + let capture_state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: rendered.logical_width, + height: rendered.logical_height, + image_width: rendered.image_width, + image_height: rendered.image_height, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Understudy GUI benchmark".to_string()), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Understudy GUI benchmark".to_string()), + }; + let (session, turn) = live_grounding_benchmark_session().await; + + let mut measurements = Vec::new(); + for truth in &truths { + let invocation = gui_invocation( + session.clone(), + turn.clone(), + tool_name_for_truth(truth), + serde_json::json!({}), + ); + measurements.push( + benchmark_measure_separate_grounding_strategy( + &invocation, + truth, + &capture_state, + &rendered.screenshot_bytes, + ) + .await, + ); + } + + println!( + "[codex-grounding-benchmark] screenshot={} cases={}", + rendered.screenshot_path.display(), + measurements.len() + ); + summarize_grounding_bucket(&measurements, "explicit", |measurement| { + measurement.prompt_clarity == "explicit" + }); + summarize_grounding_bucket(&measurements, "ambiguous", |measurement| { + measurement.prompt_clarity == "ambiguous" + }); + summarize_grounding_bucket(&measurements, "complex", |measurement| { + measurement.difficulty == "complex" + }); + summarize_grounding_bucket(&measurements, "type", |measurement| { + measurement.kind == "text_field" + }); + + let total_hit_rate = measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / measurements.len() as f64; + let explicit_measurements: Vec<_> = measurements + .iter() + .filter(|measurement| measurement.prompt_clarity == "explicit") + .collect(); + let ambiguous_measurements: Vec<_> = measurements + .iter() + .filter(|measurement| measurement.prompt_clarity == "ambiguous") + .collect(); + let complex_explicit_measurements: Vec<_> = measurements + .iter() + .filter(|measurement| { + measurement.prompt_clarity == "explicit" && measurement.difficulty == "complex" + }) + .collect(); + let explicit_hit_rate = explicit_measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / explicit_measurements.len() as f64; + let ambiguous_hit_rate = ambiguous_measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / ambiguous_measurements.len() as f64; + let complex_explicit_hit_rate = complex_explicit_measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / complex_explicit_measurements.len() as f64; + + let provider_errors: Vec<_> = measurements + .iter() + .filter(|measurement| measurement.error.is_some()) + .map(|measurement| measurement.case_id.clone()) + .collect(); + let explicit_missing_cases: Vec<_> = explicit_measurements + .iter() + .filter(|measurement| !measurement.found) + .map(|measurement| measurement.case_id.clone()) + .collect(); + let point_distance_outliers: Vec<_> = measurements + .iter() + .filter_map(|measurement| { + let truth = truths + .iter() + .find(|truth| truth.id == measurement.case_id)?; + if measurement.found && measurement.distance_px > allowed_point_distance_px(truth) { + Some(measurement.case_id.clone()) + } else { + None + } + }) + .collect(); + + println!( + "[codex-grounding-benchmark] total={:.2} explicit={:.2} ambiguous={:.2} complex_explicit={:.2}", + total_hit_rate, explicit_hit_rate, ambiguous_hit_rate, complex_explicit_hit_rate + ); + + let mut failures = Vec::new(); + if !explicit_missing_cases.is_empty() { + failures.push(format!( + "missing explicit cases: {}", + explicit_missing_cases.join(", ") + )); + } + if total_hit_rate < MIN_TOTAL_INSIDE_HIT_RATE { + failures.push(format!( + "total inside hit rate {:.2} is below {:.2}", + total_hit_rate, MIN_TOTAL_INSIDE_HIT_RATE + )); + } + if explicit_hit_rate < MIN_EXPLICIT_INSIDE_HIT_RATE { + failures.push(format!( + "explicit inside hit rate {:.2} is below {:.2}", + explicit_hit_rate, MIN_EXPLICIT_INSIDE_HIT_RATE + )); + } + if ambiguous_hit_rate < MIN_AMBIGUOUS_INSIDE_HIT_RATE { + failures.push(format!( + "ambiguous inside hit rate {:.2} is below {:.2}", + ambiguous_hit_rate, MIN_AMBIGUOUS_INSIDE_HIT_RATE + )); + } + if complex_explicit_hit_rate < MIN_COMPLEX_EXPLICIT_INSIDE_HIT_RATE { + failures.push(format!( + "complex explicit hit rate {:.2} is below {:.2}", + complex_explicit_hit_rate, MIN_COMPLEX_EXPLICIT_INSIDE_HIT_RATE + )); + } + if !provider_errors.is_empty() { + failures.push(format!("provider errors: {}", provider_errors.join(", "))); + } + if !point_distance_outliers.is_empty() { + failures.push(format!( + "distance outliers: {}", + point_distance_outliers.join(", ") + )); + } + + assert!( + failures.is_empty(), + "grounding benchmark failures:\n{}", + failures.join("\n") + ); +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual real grounding comparison benchmark requiring local Codex auth and swiftc"] +async fn macos_gui_grounding_benchmark_compares_coordinate_strategies() { + let rendered = render_understudy_grounding_benchmark(); + let requested_case_ids = requested_grounding_case_ids(); + let truths = rendered + .truths + .iter() + .filter(|truth| match requested_case_ids.as_ref() { + Some(ids) => ids.contains(&truth.id), + None => true, + }) + .cloned() + .collect::>(); + let capture_state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: rendered.logical_width, + height: rendered.logical_height, + image_width: rendered.image_width, + image_height: rendered.image_height, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Understudy GUI benchmark".to_string()), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Understudy GUI benchmark".to_string()), + }; + let (session, turn) = live_grounding_benchmark_session().await; + + let mut measurements = Vec::new(); + for truth in &truths { + let invocation = gui_invocation( + session.clone(), + turn.clone(), + tool_name_for_truth(truth), + serde_json::json!({}), + ); + measurements.push( + benchmark_measure_separate_grounding_strategy( + &invocation, + truth, + &capture_state, + &rendered.screenshot_bytes, + ) + .await, + ); + measurements.push( + benchmark_measure_main_model_coordinate_strategy( + &invocation, + truth, + &capture_state, + &rendered.screenshot_bytes, + ) + .await, + ); + } + + let separate_measurements = measurements + .iter() + .filter(|measurement| measurement.strategy == "separate_grounding") + .collect::>(); + let direct_measurements = measurements + .iter() + .filter(|measurement| measurement.strategy == "main_model_coordinates") + .collect::>(); + + println!( + "[codex-grounding-benchmark] screenshot={} cases={} strategies=2", + rendered.screenshot_path.display(), + truths.len() + ); + for strategy in ["separate_grounding", "main_model_coordinates"] { + summarize_grounding_bucket_for_strategy( + &measurements, + strategy, + "explicit", + |measurement| measurement.prompt_clarity == "explicit", + ); + summarize_grounding_bucket_for_strategy( + &measurements, + strategy, + "ambiguous", + |measurement| measurement.prompt_clarity == "ambiguous", + ); + summarize_grounding_bucket_for_strategy( + &measurements, + strategy, + "complex", + |measurement| measurement.difficulty == "complex", + ); + summarize_grounding_bucket_for_strategy(&measurements, strategy, "type", |measurement| { + measurement.kind == "text_field" + }); + } + + let separate_hit_rate = separate_measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / separate_measurements.len() as f64; + let direct_hit_rate = direct_measurements + .iter() + .filter(|measurement| measurement.inside) + .count() as f64 + / direct_measurements.len() as f64; + let separate_avg_latency_ms = separate_measurements + .iter() + .map(|measurement| measurement.elapsed_ms as f64) + .sum::() + / separate_measurements.len() as f64; + let direct_avg_latency_ms = direct_measurements + .iter() + .map(|measurement| measurement.elapsed_ms as f64) + .sum::() + / direct_measurements.len() as f64; + + let separate_errors = separate_measurements + .iter() + .filter(|measurement| measurement.error.is_some()) + .count(); + let direct_errors = direct_measurements + .iter() + .filter(|measurement| measurement.error.is_some()) + .count(); + let direct_only_hits = truths + .iter() + .filter(|truth| { + let separate = separate_measurements + .iter() + .find(|measurement| measurement.case_id == truth.id) + .expect("separate measurement should exist"); + let direct = direct_measurements + .iter() + .find(|measurement| measurement.case_id == truth.id) + .expect("direct measurement should exist"); + !separate.inside && direct.inside + }) + .map(|truth| truth.id.clone()) + .collect::>(); + let separate_only_hits = truths + .iter() + .filter(|truth| { + let separate = separate_measurements + .iter() + .find(|measurement| measurement.case_id == truth.id) + .expect("separate measurement should exist"); + let direct = direct_measurements + .iter() + .find(|measurement| measurement.case_id == truth.id) + .expect("direct measurement should exist"); + separate.inside && !direct.inside + }) + .map(|truth| truth.id.clone()) + .collect::>(); + + println!( + "[codex-grounding-benchmark] compare separate_hit_rate={:.2} direct_hit_rate={:.2} hit_rate_delta={:.2} separate_avg={}ms direct_avg={}ms latency_delta={}ms separate_errors={} direct_errors={}", + separate_hit_rate, + direct_hit_rate, + direct_hit_rate - separate_hit_rate, + separate_avg_latency_ms.round(), + direct_avg_latency_ms.round(), + (direct_avg_latency_ms - separate_avg_latency_ms).round(), + separate_errors, + direct_errors + ); + if !direct_only_hits.is_empty() { + println!( + "[codex-grounding-benchmark] direct_only_hits={}", + direct_only_hits.join(", ") + ); + } + if !separate_only_hits.is_empty() { + println!( + "[codex-grounding-benchmark] separate_only_hits={}", + separate_only_hits.join(", ") + ); + } + + assert!( + !separate_measurements.is_empty() + && separate_measurements.len() == direct_measurements.len(), + "expected both strategies to produce one measurement per case" + ); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/coordinate_space.rs b/codex-rs/core/src/tools/handlers/gui/tests/coordinate_space.rs new file mode 100644 index 000000000000..cecc0365381a --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/coordinate_space.rs @@ -0,0 +1,273 @@ +use super::*; + +#[test] +fn local_point_within_state_reports_only_in_bounds_targets() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 100.0, + origin_y: 200.0, + width: 400, + height: 300, + image_width: 800, + image_height: 600, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let in_bounds = local_point_within_state(&state, &HelperPoint { x: 125.0, y: 250.0 }) + .expect("point should be within capture"); + assert_eq!(in_bounds.x, 25.0); + assert_eq!(in_bounds.y, 50.0); + assert!(local_point_within_state(&state, &HelperPoint { x: 50.0, y: 250.0 }).is_none()); +} + +#[test] +fn image_space_helpers_validate_capture_relative_geometry() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 100.0, + origin_y: 200.0, + width: 400, + height: 300, + image_width: 800, + image_height: 600, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let point = image_point_within_capture(&state, &HelperPoint { x: 125.0, y: 250.0 }) + .expect("point should remain in image space"); + assert_eq!(point.x, 125.0); + assert_eq!(point.y, 250.0); + assert!(image_point_within_capture(&state, &HelperPoint { x: 850.0, y: 250.0 }).is_none()); + + let rect = local_rect_within_state( + &state, + &HelperRect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 80.0, + }, + ) + .expect("rect should fit inside capture"); + assert_eq!(rect.width, 100.0); + assert_eq!(rect.height, 80.0); + // Rect that extends past the right edge should be clamped, not rejected. + let clamped = local_rect_within_state( + &state, + &HelperRect { + x: 750.0, + y: 20.0, + width: 100.0, + height: 80.0, + }, + ) + .expect("rect with origin inside image should be clamped"); + assert_eq!(clamped.x, 750.0); + assert_eq!(clamped.width, 50.0); // 800 - 750 + assert_eq!(clamped.height, 80.0); + + // Rect with origin completely outside the image should still be rejected. + assert!( + local_rect_within_state( + &state, + &HelperRect { + x: 850.0, + y: 20.0, + width: 100.0, + height: 80.0, + }, + ) + .is_none() + ); +} + +#[test] +fn coordinate_helpers_require_complete_pairs_and_valid_spaces() { + assert_eq!( + normalize_coordinate_space(None).expect("default coordinate space"), + GuiCoordinateSpace::ImagePixels, + ); + assert_eq!( + normalize_coordinate_space(Some("display_points")).expect("display space"), + GuiCoordinateSpace::DisplayPoints, + ); + assert!(normalize_coordinate_space(Some("screen_pixels")).is_err()); + + let point = normalize_optional_coordinate_point(Some(12.0), Some(34.0), "x", "y") + .expect("coordinate pair") + .expect("point"); + assert_eq!(point.x, 12.0); + assert_eq!(point.y, 34.0); + assert!(normalize_optional_coordinate_point(Some(12.0), None, "x", "y").is_err()); +} + +#[test] +fn image_space_geometry_maps_back_to_display_space() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 100.0, + origin_y: 200.0, + width: 400, + height: 300, + image_width: 800, + image_height: 600, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let display_point = image_point_to_display(&state, &HelperPoint { x: 300.0, y: 120.0 }); + assert_eq!(display_point.x, 250.0); + assert_eq!(display_point.y, 260.0); + + let display_rect = image_rect_to_display( + &state, + &HelperRect { + x: 200.0, + y: 100.0, + width: 160.0, + height: 80.0, + }, + ); + assert_eq!(display_rect.x, 200.0); + assert_eq!(display_rect.y, 250.0); + assert_eq!(display_rect.width, 80.0); + assert_eq!(display_rect.height, 40.0); +} + +#[test] +fn grounding_model_image_normalizes_hidpi_capture_dimensions() { + let base = image::RgbaImage::from_pixel(1600, 1200, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + + let prepared = prepare_grounding_model_image( + &encoded.into_inner(), + GroundingModelImageConfig { + logical_width: Some(800), + logical_height: Some(600), + scale_x: Some(2.0), + scale_y: Some(2.0), + allow_logical_normalization: true, + }, + "test image", + ) + .expect("prepared image should build"); + + assert_eq!(prepared.original_width, 1600); + assert_eq!(prepared.original_height, 1200); + assert_eq!(prepared.working_width, 800); + assert_eq!(prepared.working_height, 600); + assert_eq!(prepared.model_width, 800); + assert_eq!(prepared.model_height, 600); + assert_eq!(prepared.mime_type, "image/png"); + assert!(prepared.logical_normalization_applied); + assert!((prepared.model_to_original_scale_x - 2.0).abs() < f64::EPSILON); + assert!((prepared.model_to_original_scale_y - 2.0).abs() < f64::EPSILON); +} + +#[test] +fn grounding_model_image_roundtrips_between_model_and_original_space() { + let base = image::RgbaImage::from_pixel(1600, 1200, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + let prepared = prepare_grounding_model_image( + &encoded.into_inner(), + GroundingModelImageConfig { + logical_width: Some(800), + logical_height: Some(600), + scale_x: Some(2.0), + scale_y: Some(2.0), + allow_logical_normalization: true, + }, + "test image", + ) + .expect("prepared image should build"); + let original_point = HelperPoint { x: 640.0, y: 420.0 }; + + let model_point = translate_original_point_to_model(&prepared, &original_point); + let roundtrip_point = translate_model_point_to_original(&prepared, &model_point); + + assert!((model_point.x - 320.0).abs() < 0.01); + assert!((model_point.y - 210.0).abs() < 0.01); + assert!((roundtrip_point.x - original_point.x).abs() < 0.01); + assert!((roundtrip_point.y - original_point.y).abs() < 0.01); +} + +#[test] +fn refinement_is_used_for_tiny_targets_and_maps_back_to_original_space() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + let original_point = HelperPoint { x: 640.0, y: 420.0 }; + let tiny_bbox = GroundingBoundingBox { + x1: 628.0, + y1: 408.0, + x2: 652.0, + y2: 432.0, + }; + + assert!(should_use_high_resolution_refinement( + &state, + &original_point, + Some(&tiny_bbox) + )); + + let base = image::RgbaImage::from_pixel(1600, 1200, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + let crop = create_refinement_crop(&encoded.into_inner(), &original_point, Some(&tiny_bbox)) + .expect("refinement crop should build") + .expect("refinement crop should exist"); + + let refinement_point = translate_original_point_to_refinement(&crop, &original_point); + let roundtrip_point = translate_refinement_point_to_original(&crop, &refinement_point); + + assert!((roundtrip_point.x - original_point.x).abs() < 0.51); + assert!((roundtrip_point.y - original_point.y).abs() < 0.51); + assert!(crop.model_width >= crop.crop_width); + assert!(crop.model_height >= crop.crop_height); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/drag.rs b/codex-rs/core/src/tools/handlers/gui/tests/drag.rs new file mode 100644 index 000000000000..30149cfe2fef --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/drag.rs @@ -0,0 +1,55 @@ +use super::*; + +#[test] +fn normalize_drag_endpoint_accepts_semantic_targets() { + let endpoint = normalize_drag_endpoint( + "source", + "from_target", + Some("Save button"), + Some("top right"), + Some("toolbar"), + ) + .expect("target endpoint should normalize"); + + match endpoint { + DragEndpoint::Target { + target, + location_hint, + scope, + } => { + assert_eq!(target, "Save button"); + assert_eq!(location_hint, Some("top right")); + assert_eq!(scope, Some("toolbar")); + } + } +} + +#[test] +fn normalize_drag_endpoint_requires_semantic_targets() { + let error = normalize_drag_endpoint("destination", "to_target", None, None, None) + .expect_err("missing target endpoint should fail"); + + assert!( + error + .to_string() + .contains("requires `to_target` for the destination") + ); +} + +#[test] +fn normalize_grounding_mode_defaults_and_validates() { + assert_eq!( + normalize_grounding_mode(None, "gui_click").unwrap(), + "single" + ); + assert_eq!(normalize_grounding_mode(None, "type").unwrap(), "complex"); + assert_eq!( + normalize_grounding_mode(None, "drag_source").unwrap(), + "complex" + ); + assert_eq!( + normalize_grounding_mode(Some("complex"), "gui_click").unwrap(), + "complex" + ); + assert!(normalize_grounding_mode(Some("dense"), "gui_click").is_err()); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/grounding.rs b/codex-rs/core/src/tools/handlers/gui/tests/grounding.rs new file mode 100644 index 000000000000..e9d10b58c10a --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/grounding.rs @@ -0,0 +1,555 @@ +use super::*; + +#[test] +fn grounding_prompt_includes_retry_context_and_guide_hint() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let prompt = build_gui_grounding_prompt( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Save button", + location_hint: Some("top right"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + &state, + "complex", + &[ + "Round 1 validator rejected the candidate".to_string(), + "Move away from the highlighted point".to_string(), + ], + true, + ); + + assert!(prompt.contains("Retry context:")); + assert!(prompt.contains("Round 1 validator rejected the candidate")); + assert!(prompt.contains("additional guide image")); + assert!(prompt.contains("Match the target by visible meaning")); + assert!(prompt.contains("Match subtle or weakly labeled controls")); + assert!(prompt.contains("button, icon button, toolbar item")); + assert!(prompt.contains("individual icon-bearing control")); + assert!(prompt.contains("visible button label inside a dialog")); +} + +#[test] +fn grounding_prompt_adds_action_specific_editable_surface_guidance() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let prompt = build_gui_grounding_prompt( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Search field", + location_hint: Some("top right"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "type", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + &state, + "complex", + &[], + false, + ); + + assert!(prompt.contains("editable surface itself")); + assert!(prompt.contains("icon-only affordance")); +} + +#[test] +fn not_found_retry_notes_encourage_semantic_role_flexibility() { + let notes = build_not_found_retry_notes( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Search field", + location_hint: Some("top right"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + 1, + ); + + assert!( + notes + .iter() + .any(|note| note.contains("Broaden the search while keeping the same semantic goal")) + ); + assert!(notes.iter().any(|note| note.contains("visible meaning"))); + assert!(notes.iter().any(|note| note.contains("toolbar items"))); +} + +#[test] +fn validation_prompt_requests_failure_kind_and_retry_hint() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let prompt = build_gui_grounding_validation_prompt( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Save button", + location_hint: Some("top right"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + &state, + &HelperPoint { x: 320.0, y: 180.0 }, + Some(&GroundingBoundingBox { + x1: 280.0, + y1: 140.0, + x2: 360.0, + y2: 220.0, + }), + false, + ); + + assert!(prompt.contains("failure_kind")); + assert!(prompt.contains("retry_hint")); + assert!(prompt.contains("wrong_region")); + assert!(prompt.contains("whitespace, padding, decoration")); + assert!(prompt.contains("subtle, tightly packed, or low-contrast controls")); + assert!(prompt.contains("strongest visible semantic match")); +} + +#[test] +fn validation_prompt_mentions_zoomed_crop_when_requested() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 100.0, + origin_y: 200.0, + width: 360, + height: 320, + image_width: 1200, + image_height: 1067, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let prompt = build_gui_grounding_validation_prompt( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Pin icon button", + location_hint: Some("toolbar row"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + &state, + &HelperPoint { x: 640.0, y: 400.0 }, + None, + true, + ); + + assert!(prompt.contains("zoomed crop")); + assert!(prompt.contains("original request")); +} + +#[test] +fn retry_guide_is_suppressed_for_wrong_region_and_scope_mismatch() { + assert!(!should_generate_retry_guide(Some("wrong_region"))); + assert!(!should_generate_retry_guide(Some("scope_mismatch"))); + assert!(should_generate_retry_guide(Some("wrong_control"))); + assert!(should_generate_retry_guide(None)); +} + +#[test] +fn refinement_prompt_mentions_zoomed_crop_context() { + let state = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("selected_window".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + let crop = create_refinement_crop( + &{ + let base = image::RgbaImage::from_pixel(1200, 900, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + encoded.into_inner() + }, + &HelperPoint { x: 420.0, y: 360.0 }, + Some(&GroundingBoundingBox { + x1: 400.0, + y1: 340.0, + x2: 440.0, + y2: 380.0, + }), + ) + .expect("refinement crop should build") + .expect("refinement crop should exist"); + + let prompt = build_gui_grounding_refinement_prompt( + GuiTargetRequest { + app: Some("Notes"), + capture_mode: Some("window"), + window_selection: None, + target: "Save button", + location_hint: Some("top right"), + scope: Some("toolbar"), + grounding_mode: Some("complex"), + action: "click", + related_target: None, + related_scope: None, + related_location_hint: None, + related_point: None, + }, + &state, + &crop, + Some(&HelperPoint { x: 100.0, y: 120.0 }), + Some(&GroundingBoundingBox { + x1: 80.0, + y1: 100.0, + x2: 140.0, + y2: 160.0, + }), + ); + + assert!(prompt.contains("zoomed crop around a previous candidate")); + assert!(prompt.contains("Previous crop-relative point")); + assert!(prompt.contains("Refine the target inside this crop")); +} + +#[test] +fn grounding_helpers_extract_json_and_convert_bounding_boxes() { + let wrapped = "```json\n{\"status\":\"resolved\",\"found\":true}\n```"; + assert_eq!( + extract_grounding_json(wrapped), + Some("{\"status\":\"resolved\",\"found\":true}") + ); + + let rect = grounding_bbox_to_rect(&GroundingBoundingBox { + x1: 10.0, + y1: 20.0, + x2: 30.0, + y2: 60.0, + }) + .expect("bbox should convert"); + assert_eq!(rect.x, 10.0); + assert_eq!(rect.y, 20.0); + assert_eq!(rect.width, 20.0); + assert_eq!(rect.height, 40.0); + assert!( + grounding_bbox_to_rect(&GroundingBoundingBox { + x1: 30.0, + y1: 20.0, + x2: 10.0, + y2: 60.0, + }) + .is_none() + ); +} + +#[test] +fn grounding_validation_overlay_preserves_image_dimensions() { + let base = image::RgbaImage::from_pixel(32, 24, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base.clone()) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + let base_png = encoded.into_inner(); + + let overlay = render_validation_overlay( + &base_png, + &HelperPoint { x: 12.0, y: 8.0 }, + Some(&GroundingBoundingBox { + x1: 4.0, + y1: 5.0, + x2: 20.0, + y2: 16.0, + }), + ) + .expect("overlay should render"); + let rendered = image::load_from_memory(&overlay).expect("overlay should decode"); + + assert_eq!(rendered.width(), 32); + assert_eq!(rendered.height(), 24); + assert_ne!(overlay, base_png); +} + +#[test] +fn grounding_guide_overlay_preserves_image_dimensions() { + let base = image::RgbaImage::from_pixel(32, 24, image::Rgba([240, 240, 240, 255])); + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(base.clone()) + .write_to(&mut encoded, ImageFormat::Png) + .expect("base image should encode"); + let base_png = encoded.into_inner(); + + let overlay = render_guide_overlay( + &base_png, + &HelperPoint { x: 14.0, y: 10.0 }, + Some(&GroundingBoundingBox { + x1: 6.0, + y1: 4.0, + x2: 18.0, + y2: 20.0, + }), + ) + .expect("guide overlay should render"); + let rendered = image::load_from_memory(&overlay).expect("guide overlay should decode"); + + assert_eq!(rendered.width(), 32); + assert_eq!(rendered.height(), 24); + assert_ne!(overlay, base_png); +} + +#[test] +fn annotate_grounding_raw_adds_validation_metadata() { + let mut raw = serde_json::json!({ + "status": "resolved", + "found": true, + }); + + annotate_grounding_raw( + &mut raw, + "complex", + "validated", + true, + "approved", + Some("marker aligned with the target"), + Some(0.92), + 2, + ); + + assert_eq!(raw["grounding_mode_requested"], "complex"); + assert_eq!(raw["grounding_mode_effective"], "complex"); + assert_eq!(raw["selected_attempt"], "validated"); + assert_eq!(raw["grounding_validation_triggered"], true); + assert_eq!(raw["grounding_rounds_attempted"], 2); + assert_eq!(raw["validation"]["status"], "approved"); + assert_eq!( + raw["validation"]["reason"], + "marker aligned with the target" + ); +} + +#[test] +fn annotate_grounding_round_artifacts_adds_structured_attempts() { + let mut raw = serde_json::json!({ + "status": "resolved", + "found": true, + }); + + annotate_grounding_round_artifacts( + &mut raw, + &[ + serde_json::json!({ + "round": 1, + "terminal_state": "validator_rejected_retry", + "predictor": { + "outcome": { + "status": "resolved", + }, + }, + }), + serde_json::json!({ + "round": 2, + "terminal_state": "accepted", + "validation": { + "triggered": true, + "outcome": { + "status": "approved", + }, + }, + }), + ], + ); + + assert_eq!(raw["grounding_round_artifacts"][0]["round"], 1); + assert_eq!( + raw["grounding_round_artifacts"][0]["terminal_state"], + "validator_rejected_retry" + ); + assert_eq!(raw["grounding_round_artifacts"][1]["round"], 2); + assert_eq!( + raw["grounding_round_artifacts"][1]["validation"]["triggered"], + true + ); +} + +#[test] +fn target_resolution_details_surface_grounding_diagnostics_summary() { + let grounded = GroundedGuiTarget { + grounding_method: "grounding", + resolved: ResolvedTarget { + window_title: Some("Finder".to_string()), + provider: "openai:gpt".to_string(), + confidence: 0.92, + reason: Some("matched visible search affordance".to_string()), + grounding_mode_requested: "complex".to_string(), + grounding_mode_effective: "complex".to_string(), + scope: Some("toolbar".to_string()), + point: HelperPoint { x: 1200.0, y: 42.0 }, + bounds: HelperRect { + x: 1180.0, + y: 22.0, + width: 48.0, + height: 32.0, + }, + local_point: Some(HelperPoint { x: 600.0, y: 21.0 }), + local_bounds: Some(HelperRect { + x: 590.0, + y: 11.0, + width: 24.0, + height: 16.0, + }), + raw: Some(serde_json::json!({ + "selected_attempt": "validated_retry", + "grounding_rounds_attempted": 2, + "grounding_validation_triggered": true, + "grounding_model_image": { + "model_width": 800, + "model_height": 600, + }, + "validation": { + "status": "approved", + }, + "grounding_round_artifacts": [ + { + "round": 1, + "terminal_state": "validator_rejected_retry", + }, + { + "round": 2, + "terminal_state": "accepted", + }, + ], + })), + capture_state: ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 1440, + height: 900, + image_width: 1440, + image_height: 900, + + display_index: 1, + capture_mode: CaptureMode::Display, + window_title: Some("Finder".to_string()), + window_count: Some(1), + window_capture_strategy: Some("display".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Finder".to_string()), + }, + }, + }; + + let details = build_target_resolution_details("Search", &grounded); + + assert_eq!( + details["grounding_diagnostics"]["selected_attempt"], + "validated_retry" + ); + assert_eq!(details["grounding_diagnostics"]["rounds_attempted"], 2); + assert_eq!( + details["grounding_diagnostics"]["round_artifacts"][0]["terminal_state"], + "validator_rejected_retry" + ); + assert_eq!( + details["grounding_diagnostics"]["model_image"]["model_width"], + 800 + ); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/mod.rs b/codex-rs/core/src/tools/handlers/gui/tests/mod.rs new file mode 100644 index 000000000000..f6c4963e7dff --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/mod.rs @@ -0,0 +1,71 @@ +use super::grounding::GroundingModelImageConfig; +use super::grounding::annotate_grounding_raw; +use super::grounding::annotate_grounding_round_artifacts; +use super::grounding::build_gui_grounding_prompt; +use super::grounding::build_gui_grounding_refinement_prompt; +use super::grounding::build_gui_grounding_validation_prompt; +use super::grounding::build_not_found_retry_notes; +use super::grounding::create_refinement_crop; +use super::grounding::extract_grounding_json; +use super::grounding::grounding_bbox_to_rect; +use super::grounding::gui_grounding_output_schema; +use super::grounding::image_point_to_display; +use super::grounding::image_rect_to_display; +use super::grounding::prepare_grounding_model_image; +use super::grounding::render_guide_overlay; +use super::grounding::render_validation_overlay; +use super::grounding::request_model_json_from_image; +use super::grounding::should_generate_retry_guide; +use super::grounding::should_use_high_resolution_refinement; +use super::grounding::translate_model_point_to_original; +use super::grounding::translate_original_point_to_model; +use super::grounding::translate_original_point_to_refinement; +use super::grounding::translate_refinement_point_to_original; +#[cfg(target_os = "macos")] +use super::platform::platform_macos::normalize_capture_mode_env; +use super::readiness::GuiEnvironmentReadinessCheck; +use super::readiness::GuiEnvironmentReadinessSnapshot; +use super::readiness::GuiReadinessStatus; +use super::readiness::GuiToolCapability; +use super::readiness::resolve_gui_runtime_capabilities; +use super::*; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::registry::ToolHandler; +use crate::turn_diff_tracker::TurnDiffTracker; +use image::ImageFormat; +use serde::Deserialize; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::Mutex; + +mod benchmark; +mod coordinate_space; +mod drag; +mod grounding; +mod readiness; +mod scroll; +mod smoke_tests; +mod type_and_input; +mod wait; +mod window_selection; + +pub(super) fn gui_invocation( + session: Arc, + turn: Arc, + tool_name: &str, + args: serde_json::Value, +) -> ToolInvocation { + ToolInvocation { + session, + turn, + tracker: Arc::new(Mutex::new(TurnDiffTracker::default())), + call_id: "gui-test-call".to_string(), + tool_name: tool_name.to_string(), + tool_namespace: None, + payload: ToolPayload::Function { + arguments: args.to_string(), + }, + } +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/readiness.rs b/codex-rs/core/src/tools/handlers/gui/tests/readiness.rs new file mode 100644 index 000000000000..07d40298cd2e --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/readiness.rs @@ -0,0 +1,213 @@ +use super::*; + +#[test] +fn runtime_capabilities_disable_grounded_actions_without_grounding() { + let readiness = GuiEnvironmentReadinessSnapshot { + status: "ready", + checks: vec![ + GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "native_helper", + label: "Native GUI Helper", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + ], + }; + let capabilities = resolve_gui_runtime_capabilities(false, &readiness, None); + assert!(capabilities.platform_supported); + assert!(!capabilities.grounding_available); + assert!(capabilities.native_helper_available); + assert!(capabilities.screen_capture_available); + assert!(capabilities.input_available); + assert!(capabilities.enabled_tool_names.contains(&"gui_scroll")); + assert!(capabilities.disabled_tool_names.contains(&"gui_click")); + assert!(!capabilities.tool_availability["gui_click"].enabled); + assert!(capabilities.tool_availability["gui_scroll"].enabled); + assert!(capabilities.tool_availability["gui_scroll"].targetless_only); + assert!(capabilities.tool_availability["gui_type"].enabled); + assert!(capabilities.tool_availability["gui_type"].targetless_only); +} + +#[tokio::test] +async fn gui_observe_targetless_request_skips_image_attachment_without_image_input() { + let (session, mut turn) = crate::codex::make_session_and_context().await; + turn.model_info.input_modalities = vec![codex_protocol::openai_models::InputModality::Text]; + let invocation = gui_invocation( + Arc::new(session), + Arc::new(turn), + "gui_observe", + serde_json::json!({}), + ); + + let attach_image = prepare_gui_observe_request(&invocation, false, None) + .await + .expect("targetless gui_observe should remain available without image input"); + + assert!(!attach_image); +} + +#[tokio::test] +async fn gui_observe_targeted_request_stays_blocked_without_image_input() { + let (session, mut turn) = crate::codex::make_session_and_context().await; + turn.model_info.input_modalities = vec![codex_protocol::openai_models::InputModality::Text]; + let invocation = gui_invocation( + Arc::new(session), + Arc::new(turn), + "gui_observe", + serde_json::json!({ + "target": "Search field", + }), + ); + + let error = prepare_gui_observe_request(&invocation, true, None) + .await + .expect_err("targeted gui_observe should still be blocked without image input"); + + assert!( + error + .to_string() + .contains("Keyboard-only tool (gui_key) and targetless gui_type still work.") + || error.to_string().contains("targetless"), + "unexpected capability error: {error}" + ); +} + +#[test] +fn runtime_capabilities_disable_input_actions_without_accessibility() { + let readiness = GuiEnvironmentReadinessSnapshot { + status: "blocked", + checks: vec![ + GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Error, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "native_helper", + label: "Native GUI Helper", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + ], + }; + let capabilities = resolve_gui_runtime_capabilities(true, &readiness, None); + assert!(capabilities.platform_supported); + assert!(capabilities.grounding_available); + assert!(capabilities.native_helper_available); + assert!(capabilities.screen_capture_available); + assert!(!capabilities.input_available); + assert!(!capabilities.tool_availability["gui_click"].enabled); + assert!(!capabilities.tool_availability["gui_key"].enabled); + assert!(capabilities.tool_availability["gui_observe"].enabled); + assert!(capabilities.tool_availability["gui_wait"].enabled); +} + +#[test] +fn runtime_capabilities_respect_platform_tool_contract() { + let readiness = GuiEnvironmentReadinessSnapshot { + status: "ready", + checks: vec![ + GuiEnvironmentReadinessCheck { + id: "platform", + label: "Platform", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "accessibility", + label: "Accessibility", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "screen_recording", + label: "Screen Recording", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + GuiEnvironmentReadinessCheck { + id: "native_helper", + label: "Native GUI Helper", + status: GuiReadinessStatus::Ok, + summary: String::new(), + detail: None, + }, + ], + }; + let platform_support = HashMap::from([ + ( + "gui_move", + GuiToolCapability { + enabled: false, + reason: Some("platform backend does not support pointer movement".to_string()), + targetless_only: false, + }, + ), + ( + "gui_type", + GuiToolCapability { + enabled: true, + reason: Some( + "platform backend only supports typing into the focused control".to_string(), + ), + targetless_only: true, + }, + ), + ]); + + let capabilities = resolve_gui_runtime_capabilities(true, &readiness, Some(&platform_support)); + + assert!(!capabilities.tool_availability["gui_move"].enabled); + assert_eq!( + capabilities.tool_availability["gui_move"].reason.as_deref(), + Some("platform backend does not support pointer movement") + ); + assert!(capabilities.tool_availability["gui_type"].enabled); + assert!(capabilities.tool_availability["gui_type"].targetless_only); + assert_eq!( + capabilities.tool_availability["gui_type"].reason.as_deref(), + Some("platform backend only supports typing into the focused control") + ); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/scroll.rs b/codex-rs/core/src/tools/handlers/gui/tests/scroll.rs new file mode 100644 index 000000000000..ca5a5a8cba41 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/scroll.rs @@ -0,0 +1,71 @@ +use super::*; + +#[test] +fn normalize_scroll_direction_defaults_and_validates() { + assert!(matches!( + normalize_scroll_direction(None).unwrap(), + ScrollDirection::Down + )); + assert!(matches!( + normalize_scroll_direction(Some("left")).unwrap(), + ScrollDirection::Left + )); + assert!(normalize_scroll_direction(Some("sideways")).is_err()); +} + +#[test] +fn resolve_scroll_plan_uses_semantic_distance_defaults() { + let capture_bounds = HelperRect { + x: 0.0, + y: 0.0, + width: 1200.0, + height: 800.0, + }; + + let targetless = resolve_scroll_plan( + None, + None, + false, + ScrollDirection::Down, + None, + Some(&capture_bounds), + ); + assert_eq!(targetless.distance_preset, "page"); + assert_eq!(targetless.unit, "pixel"); + assert_eq!(targetless.amount, 600); + + let target_bounds = HelperRect { + x: 0.0, + y: 0.0, + width: 400.0, + height: 200.0, + }; + let targeted = resolve_scroll_plan( + None, + None, + true, + ScrollDirection::Down, + Some(&target_bounds), + Some(&capture_bounds), + ); + assert_eq!(targeted.distance_preset, "medium"); + assert_eq!(targeted.unit, "pixel"); + assert_eq!(targeted.amount, 100); +} + +#[test] +fn scroll_delta_components_match_understudy_scroll_direction_convention() { + assert_eq!( + scroll_delta_components(ScrollDirection::Down, 240), + (0, -240) + ); + assert_eq!(scroll_delta_components(ScrollDirection::Up, 240), (0, 240)); + assert_eq!( + scroll_delta_components(ScrollDirection::Left, 240), + (-240, 0) + ); + assert_eq!( + scroll_delta_components(ScrollDirection::Right, 240), + (240, 0) + ); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/smoke_tests.rs b/codex-rs/core/src/tools/handlers/gui/tests/smoke_tests.rs new file mode 100644 index 000000000000..29410e5e3e4d --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/smoke_tests.rs @@ -0,0 +1,748 @@ +use super::*; + +#[tokio::test] +async fn prepare_targeted_gui_action_is_noop_without_targeting() { + prepare_targeted_gui_action(None, None, None) + .await + .expect("no-op targeted action"); +} + +#[cfg(target_os = "macos")] +fn run_applescript(script: &str) -> String { + let output = std::process::Command::new("osascript") + .args(["-l", "AppleScript", "-e", script]) + .output() + .expect("osascript should launch"); + if !output.status.success() { + panic!( + "AppleScript failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +#[cfg(target_os = "macos")] +fn close_textedit_without_saving() { + let _ = std::process::Command::new("osascript") + .args([ + "-l", + "AppleScript", + "-e", + r#" +tell application "TextEdit" + if it is running then + repeat with docRef in documents + close docRef saving no + end repeat + activate + end if +end tell +"#, + ]) + .output(); +} + +#[cfg(target_os = "macos")] +fn read_textedit_document_text() -> String { + run_applescript( + r#" +tell application "TextEdit" + if (count of documents) is 0 then return "" + return text of document 1 +end tell +"#, + ) +} + +#[cfg(target_os = "macos")] +fn launch_textedit() { + let status = std::process::Command::new("open") + .args(["-a", "TextEdit"]) + .status() + .expect("open should launch TextEdit"); + assert!(status.success(), "open -a TextEdit should succeed"); + std::thread::sleep(std::time::Duration::from_millis(400)); + run_applescript(r#"tell application "TextEdit" to activate"#); +} + +#[cfg(target_os = "macos")] +fn wait_for_textedit_document_text(expected: &str) { + let started_at = std::time::Instant::now(); + loop { + let text = read_textedit_document_text(); + if text.contains(expected) { + return; + } + assert!( + started_at.elapsed() < std::time::Duration::from_secs(10), + "timed out waiting for TextEdit content `{expected}`, last content was `{text}`" + ); + std::thread::sleep(std::time::Duration::from_millis(150)); + } +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording and Accessibility permissions"] +async fn macos_gui_capture_smoke_test() { + let helper_path = resolve_helper_binary().expect("native GUI helper should compile"); + assert!(helper_path.exists(), "helper binary should exist"); + + let observation = observe_platform(None, false, Some("display"), None, false) + .await + .expect("display capture"); + + assert!( + !observation.image_bytes.is_empty(), + "captured image should not be empty" + ); + assert_eq!(observation.state.capture.capture_mode, CaptureMode::Display); +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Accessibility permissions"] +async fn macos_gui_move_cursor_smoke_test() { + let context = capture_context(None, false, None) + .await + .expect("capture context should be available"); + + run_gui_event( + "move_cursor", + None, + &[ + ("CODEX_GUI_X", context.cursor.x), + ("CODEX_GUI_Y", context.cursor.y), + ], + &[("CODEX_GUI_SETTLE_MS", "1".to_string())], + ) + .await + .expect("move cursor event should succeed"); +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Accessibility permissions"] +async fn macos_gui_move_tool_handler_smoke_test() { + let context = capture_context(None, false, None) + .await + .expect("capture context should be available"); + let (session, mut turn) = crate::codex::make_session_and_context().await; + let handler = GuiHandler::default(); + let payload = serde_json::json!({ + "x": context.cursor.x, + "y": context.cursor.y, + }); + + let output = handler + .handle(gui_invocation( + Arc::new(session), + Arc::new(turn), + "gui_move", + payload, + )) + .await + .expect("gui_move should succeed through the tool handler"); + + assert!(output.success); + assert_eq!(output.code_result["action_kind"], "move_cursor"); +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording and Accessibility permissions"] +async fn macos_gui_wait_smoke_test() { + std::thread::sleep(std::time::Duration::from_millis(1)); + + let context = capture_context(None, false, None) + .await + .expect("capture context should be available"); + let capture = + resolve_capture_target(&context, Some("display"), false, false).expect("display capture"); + let image_bytes = capture_region(&capture.bounds, capture.width, capture.height) + .await + .expect("screenshot"); + + assert!( + !image_bytes.is_empty(), + "refreshed image should not be empty" + ); +} + +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording and Accessibility permissions"] +async fn macos_gui_textedit_typing_smoke_test() { + close_textedit_without_saving(); + launch_textedit(); + + let handler = GuiHandler::default(); + let (session, turn) = crate::codex::make_session_and_context().await; + let session = Arc::new(session); + let turn = Arc::new(turn); + + let outcome = async { + let new_doc = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("gui_key should create a new TextEdit document"); + assert!(new_doc.success); + assert_eq!(new_doc.code_result["action_kind"], "key_press"); + + let type_first_line = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_type", + serde_json::json!({ + "app": "TextEdit", + "value": "Codex native smoke", + }), + )) + .await + .expect("gui_type should type into the new document"); + assert!(type_first_line.success); + assert_eq!(type_first_line.code_result["action_kind"], "type_text"); + wait_for_textedit_document_text("Codex native smoke"); + + let enter = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "Enter", + }), + )) + .await + .expect("gui_key should press Enter"); + assert!(enter.success); + assert_eq!(enter.code_result["action_kind"], "key_press"); + + let type_second_line = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_type", + serde_json::json!({ + "app": "TextEdit", + "value": "Second line", + "replace": false, + }), + )) + .await + .expect("gui_type should append the second line"); + assert!(type_second_line.success); + wait_for_textedit_document_text("Codex native smoke\nSecond line"); + + let observation = handler + .handle(gui_invocation( + session, + turn, + "gui_observe", + serde_json::json!({ + "app": "TextEdit", + }), + )) + .await + .expect("gui_observe should capture a refreshed TextEdit screenshot"); + assert!(observation.success); + assert_eq!(observation.code_result["capture_mode"], "window"); + assert_eq!(observation.code_result["app"], "TextEdit"); + assert!(observation.code_result["image_url"].is_string()); + + let final_text = read_textedit_document_text(); + assert!(final_text.contains("Codex native smoke\nSecond line")); + } + .await; + + close_textedit_without_saving(); + outcome +} + +/// Smoke test: execute multiple targetless actions in a single gui_batch call. +/// Validates that the batch handler correctly executes key + type steps +/// sequentially, and that the result reflects all actions. +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording and Accessibility permissions"] +async fn macos_gui_batch_targetless_smoke_test() { + close_textedit_without_saving(); + launch_textedit(); + + let handler = GuiHandler::default(); + let (session, turn) = crate::codex::make_session_and_context().await; + let session = Arc::new(session); + let turn = Arc::new(turn); + + let outcome = async { + // First create a new document + let new_doc = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("gui_key should create a new TextEdit document"); + assert!(new_doc.success); + + std::thread::sleep(std::time::Duration::from_millis(500)); + + // Now batch: type line 1, press Enter, type line 2 + let batch_start = std::time::Instant::now(); + let batch_result = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_batch", + serde_json::json!({ + "app": "TextEdit", + "steps": [ + { "action": "type", "value": "Batch line one" }, + { "action": "key", "key": "Return" }, + { "action": "type", "value": "Batch line two", "replace": false }, + ] + }), + )) + .await + .expect("gui_batch should execute all steps"); + let batch_elapsed = batch_start.elapsed(); + assert!(batch_result.success); + assert_eq!(batch_result.code_result["action_kind"], "batch"); + assert_eq!(batch_result.code_result["steps_count"], 3); + eprintln!( + "[gui_batch timing] 3 targetless steps in batch: {:?}", + batch_elapsed + ); + + wait_for_textedit_document_text("Batch line one\nBatch line two"); + let final_text = read_textedit_document_text(); + assert!( + final_text.contains("Batch line one\nBatch line two"), + "TextEdit should contain both lines, got: {final_text}" + ); + } + .await; + + close_textedit_without_saving(); + outcome +} + +/// 10-step batch with grounded click/type targets on a real app. +/// Uses live Codex credentials to call the grounding model. +/// Measures individual grounded calls vs batch grounding. +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording, Accessibility, and live Codex auth"] +async fn macos_gui_batch_10_step_grounded_vs_individual() { + use super::benchmark::live_grounding_benchmark_session; + + close_textedit_without_saving(); + launch_textedit(); + + let handler = GuiHandler::default(); + let (session, turn) = live_grounding_benchmark_session().await; + + // Ensure TextEdit has a fresh document with the formatting bar visible. + let outcome = async { + // Create new document via Cmd+N. + let new_doc = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("Cmd+N to create new document"); + assert!(new_doc.success); + std::thread::sleep(std::time::Duration::from_millis(800)); + + // ── A. 10 individual grounded calls ────────────────────────── + // Each call: screenshot → grounding → action → evidence. + let targets: Vec = vec![ + serde_json::json!({ + "action": "click", + "target": "Bold button", + "scope": "formatting toolbar", + }), + serde_json::json!({ + "action": "click", + "target": "Italic button", + "scope": "formatting toolbar", + }), + serde_json::json!({ + "action": "click", + "target": "Underline button", + "scope": "formatting toolbar", + }), + serde_json::json!({ + "action": "type", + "target": "document text area", + "value": "Hello World", + }), + serde_json::json!({ + "action": "key", + "key": "Return", + }), + serde_json::json!({ + "action": "type", + "value": "Line two", + "replace": false, + }), + serde_json::json!({ + "action": "key", + "key": "Return", + }), + serde_json::json!({ + "action": "type", + "value": "Line three", + "replace": false, + }), + serde_json::json!({ + "action": "click", + "target": "Bold button", + "scope": "formatting toolbar", + }), + serde_json::json!({ + "action": "click", + "target": "Italic button", + "scope": "formatting toolbar", + }), + ]; + + // Run the same 10 steps individually. + let individual_start = std::time::Instant::now(); + for (i, step) in targets.iter().enumerate() { + let action = step["action"].as_str().unwrap(); + let tool_name = format!("gui_{action}"); + let mut args = step.clone(); + // Remove the 'action' field — individual tools don't use it. + args.as_object_mut().unwrap().remove("action"); + args.as_object_mut() + .unwrap() + .insert("app".to_string(), serde_json::json!("TextEdit")); + let result = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + &tool_name, + args, + )) + .await; + match result { + Ok(output) => { + eprintln!( + "[individual step {i}] {action}: success={}, {:.0}ms", + output.success, + individual_start.elapsed().as_millis() + ); + } + Err(e) => { + eprintln!("[individual step {i}] {action}: error={e}"); + } + } + } + let individual_elapsed = individual_start.elapsed(); + + // ── B. Reset and run as a batch ────────────────────────────── + close_textedit_without_saving(); + std::thread::sleep(std::time::Duration::from_millis(600)); + launch_textedit(); + let new_doc2 = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("Cmd+N again"); + assert!(new_doc2.success); + std::thread::sleep(std::time::Duration::from_millis(800)); + + let batch_start = std::time::Instant::now(); + let batch_result = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_batch", + serde_json::json!({ + "app": "TextEdit", + "steps": targets, + }), + )) + .await; + + let batch_elapsed = batch_start.elapsed(); + match &batch_result { + Ok(output) => { + eprintln!( + "[batch] success={}, steps_count={}", + output.success, output.code_result["steps_count"] + ); + } + Err(e) => { + eprintln!("[batch] error={e}"); + } + } + + // ── C. Same batch with unified grounding strategy ───────────── + close_textedit_without_saving(); + std::thread::sleep(std::time::Duration::from_millis(600)); + launch_textedit(); + let new_doc3 = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("Cmd+N for unified test"); + assert!(new_doc3.success); + std::thread::sleep(std::time::Duration::from_millis(800)); + + let unified_start = std::time::Instant::now(); + let unified_result = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_batch", + serde_json::json!({ + "app": "TextEdit", + "grounding_strategy": "unified", + "steps": targets, + }), + )) + .await; + let unified_elapsed = unified_start.elapsed(); + match &unified_result { + Ok(output) => { + eprintln!( + "[unified] success={}, steps_count={}", + output.success, output.code_result["steps_count"] + ); + } + Err(e) => { + eprintln!("[unified] error={e}"); + } + } + + let parallel_speedup = individual_elapsed.as_secs_f64() / batch_elapsed.as_secs_f64(); + let unified_speedup = individual_elapsed.as_secs_f64() / unified_elapsed.as_secs_f64(); + eprintln!("====================================================="); + eprintln!( + "[TIMING] 10 individual calls: {:.1}s", + individual_elapsed.as_secs_f64() + ); + eprintln!( + "[TIMING] 10 parallel batch: {:.1}s ({:.1}x)", + batch_elapsed.as_secs_f64(), + parallel_speedup, + ); + eprintln!( + "[TIMING] 10 unified batch: {:.1}s ({:.1}x)", + unified_elapsed.as_secs_f64(), + unified_speedup, + ); + eprintln!("====================================================="); + } + .await; + + close_textedit_without_saving(); + outcome +} + +/// Timing comparison: individual gui_key/gui_type calls vs gui_batch. +/// Measures the wall-clock time difference to validate the speedup. +#[tokio::test] +#[cfg(target_os = "macos")] +#[ignore = "manual macOS GUI smoke test requiring Screen Recording and Accessibility permissions"] +async fn macos_gui_batch_vs_individual_timing() { + close_textedit_without_saving(); + launch_textedit(); + + let handler = GuiHandler::default(); + let (session, turn) = crate::codex::make_session_and_context().await; + let session = Arc::new(session); + let turn = Arc::new(turn); + + let outcome = async { + // ── Individual calls ───────────────────────────────────────── + let new_doc = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("gui_key Cmd+N"); + assert!(new_doc.success); + std::thread::sleep(std::time::Duration::from_millis(500)); + + let individual_start = std::time::Instant::now(); + + handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_type", + serde_json::json!({ + "app": "TextEdit", + "value": "Individual A", + }), + )) + .await + .expect("type A"); + handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "Return", + }), + )) + .await + .expect("Enter"); + handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_type", + serde_json::json!({ + "app": "TextEdit", + "value": "Individual B", + "replace": false, + }), + )) + .await + .expect("type B"); + handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "Return", + }), + )) + .await + .expect("Enter 2"); + handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_type", + serde_json::json!({ + "app": "TextEdit", + "value": "Individual C", + "replace": false, + }), + )) + .await + .expect("type C"); + + let individual_elapsed = individual_start.elapsed(); + wait_for_textedit_document_text("Individual A\nIndividual B\nIndividual C"); + + // ── Batched calls ──────────────────────────────────────────── + // Close and reopen TextEdit for a clean slate + close_textedit_without_saving(); + std::thread::sleep(std::time::Duration::from_millis(400)); + launch_textedit(); + + let new_doc2 = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_key", + serde_json::json!({ + "app": "TextEdit", + "key": "n", + "modifiers": ["command"], + }), + )) + .await + .expect("gui_key Cmd+N"); + assert!(new_doc2.success); + std::thread::sleep(std::time::Duration::from_millis(500)); + + let batch_start = std::time::Instant::now(); + + let batch_result = handler + .handle(gui_invocation( + session.clone(), + turn.clone(), + "gui_batch", + serde_json::json!({ + "app": "TextEdit", + "steps": [ + { "action": "type", "value": "Batched A" }, + { "action": "key", "key": "Return" }, + { "action": "type", "value": "Batched B", "replace": false }, + { "action": "key", "key": "Return" }, + { "action": "type", "value": "Batched C", "replace": false }, + ] + }), + )) + .await + .expect("gui_batch 5 steps"); + assert!(batch_result.success); + + let batch_elapsed = batch_start.elapsed(); + wait_for_textedit_document_text("Batched A\nBatched B\nBatched C"); + + let speedup = individual_elapsed.as_secs_f64() / batch_elapsed.as_secs_f64(); + eprintln!("====================================================="); + eprintln!("[TIMING] 5 individual calls: {:?}", individual_elapsed); + eprintln!("[TIMING] 5 batched steps: {:?}", batch_elapsed); + eprintln!("[TIMING] Speedup: {:.1}x", speedup); + eprintln!("====================================================="); + assert!( + speedup > 1.5, + "Batch should be significantly faster than individual calls, got {speedup:.1}x" + ); + } + .await; + + close_textedit_without_saving(); + outcome +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/type_and_input.rs b/codex-rs/core/src/tools/handlers/gui/tests/type_and_input.rs new file mode 100644 index 000000000000..c8793fb0a8ad --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/type_and_input.rs @@ -0,0 +1,176 @@ +use super::*; +use serial_test::serial; +use std::env; +use std::ffi::OsStr; + +struct EnvVarGuard { + key: &'static str, + original: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: &OsStr) -> Self { + let original = env::var_os(key); + unsafe { + env::set_var(key, value); + } + Self { key, original } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match self.original.take() { + Some(value) => unsafe { + env::set_var(self.key, value); + }, + None => unsafe { + env::remove_var(self.key); + }, + } + } +} + +#[test] +fn targeted_type_focus_point_prefers_display_box_center() { + let resolved = ResolvedTarget { + window_title: Some("Browser".to_string()), + provider: "openai:gpt-5.4".to_string(), + confidence: 0.94, + reason: Some("matched the only editable field".to_string()), + grounding_mode_requested: "complex".to_string(), + grounding_mode_effective: "complex".to_string(), + scope: Some("Workspace panel".to_string()), + point: HelperPoint { x: 114.0, y: 225.0 }, + bounds: HelperRect { + x: 80.0, + y: 200.0, + width: 240.0, + height: 40.0, + }, + local_point: None, + local_bounds: None, + raw: None, + capture_state: ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 1280, + height: 800, + image_width: 1280, + image_height: 800, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Browser".to_string()), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Browser".to_string()), + }, + }; + + let focus_point = targeted_type_focus_point(&resolved); + assert_eq!(focus_point.x, 200.0); + assert_eq!(focus_point.y, 220.0); +} + +#[test] +fn targeted_type_focus_point_falls_back_to_grounded_point_without_valid_box() { + let resolved = ResolvedTarget { + window_title: Some("Browser".to_string()), + provider: "openai:gpt-5.4".to_string(), + confidence: 0.94, + reason: Some("matched the only editable field".to_string()), + grounding_mode_requested: "complex".to_string(), + grounding_mode_effective: "complex".to_string(), + scope: Some("Workspace panel".to_string()), + point: HelperPoint { x: 114.0, y: 225.0 }, + bounds: HelperRect { + x: 80.0, + y: 200.0, + width: 0.0, + height: 40.0, + }, + local_point: None, + local_bounds: None, + raw: None, + capture_state: ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 1280, + height: 800, + image_width: 1280, + image_height: 800, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Browser".to_string()), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Browser".to_string()), + }, + }; + + let focus_point = targeted_type_focus_point(&resolved); + assert_eq!(focus_point.x, 114.0); + assert_eq!(focus_point.y, 225.0); +} + +#[test] +#[serial] +fn resolve_type_value_accepts_plain_secret_env_var_names() { + let _guard = EnvVarGuard::set("GUI_TYPE_TEST_SECRET", OsStr::new("hunter2")); + let args = TypeArgs { + value: None, + secret_env_var: Some("GUI_TYPE_TEST_SECRET".to_string()), + secret_command_env_var: None, + target: None, + location_hint: None, + scope: None, + grounding_mode: None, + type_strategy: None, + capture_mode: None, + window_title: None, + window_selector: None, + app: None, + replace: None, + submit: None, + }; + + let value = resolve_type_value(&args).expect("plain env var names should still be supported"); + assert_eq!(value, "hunter2"); +} + +#[test] +#[serial] +fn resolve_type_value_accepts_plain_secret_command_env_var_names() { + let _guard = EnvVarGuard::set( + "GUI_TYPE_TEST_SECRET_COMMAND", + OsStr::new("printf 'from-command'"), + ); + let args = TypeArgs { + value: None, + secret_env_var: None, + secret_command_env_var: Some("GUI_TYPE_TEST_SECRET_COMMAND".to_string()), + target: None, + location_hint: None, + scope: None, + grounding_mode: None, + type_strategy: None, + capture_mode: None, + window_title: None, + window_selector: None, + app: None, + replace: None, + submit: None, + }; + + let value = + resolve_type_value(&args).expect("plain command env var names should still be supported"); + assert_eq!(value, "from-command"); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/wait.rs b/codex-rs/core/src/tools/handlers/gui/tests/wait.rs new file mode 100644 index 000000000000..8812ebbc3267 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/wait.rs @@ -0,0 +1,95 @@ +use super::*; + +#[test] +fn gui_wait_reuses_previous_window_observe_target_when_not_overridden() { + let previous = ObserveState { + capture: CaptureArtifact { + origin_x: 0.0, + origin_y: 0.0, + width: 800, + height: 600, + image_width: 1600, + image_height: 1200, + + display_index: 1, + capture_mode: CaptureMode::Window, + window_title: Some("Quick Note".to_string()), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_exclusion: HostCaptureExclusionState::default(), + }, + app_name: Some("Notes".to_string()), + }; + + let mut app = None; + let mut capture_mode = None; + let mut window_selection = None; + + if app.is_none() && capture_mode.is_none() && window_selection.is_none() { + app = previous.app_name.clone(); + capture_mode = Some(previous.capture.capture_mode.to_string()); + if previous.capture.capture_mode == CaptureMode::Window { + window_selection = previous + .capture + .window_title + .as_ref() + .map(|title| WindowSelector { + title: Some(title.clone()), + title_contains: None, + index: None, + }); + } + } + + assert_eq!(app.as_deref(), Some("Notes")); + assert_eq!(capture_mode.as_deref(), Some("window")); + assert_eq!( + window_selection.and_then(|selection| selection.title), + Some("Quick Note".to_string()) + ); +} + +#[test] +fn normalize_wait_target_state_defaults_and_validates() { + assert_eq!(normalize_wait_target_state(None).unwrap(), "appear"); + assert_eq!( + normalize_wait_target_state(Some("disappear")).unwrap(), + "disappear" + ); + assert!(normalize_wait_target_state(Some("later")).is_err()); +} + +#[test] +fn fallback_probe_capture_mode_switches_to_display_after_first_app_attempt() { + assert_eq!(fallback_probe_capture_mode(None, 1, Some("Notes")), None); + assert_eq!( + fallback_probe_capture_mode(None, 2, Some("Notes")), + Some("display") + ); + assert_eq!( + fallback_probe_capture_mode(Some("window"), 2, Some("Notes")), + Some("window") + ); + assert_eq!(fallback_probe_capture_mode(None, 2, None), None); +} + +#[test] +fn fallback_probe_capture_mode_is_generic_for_waits_and_grounded_actions() { + assert_eq!( + fallback_probe_capture_mode(None, 2, Some("Finder")), + Some("display") + ); + assert_eq!( + fallback_probe_capture_mode(Some("display"), 2, Some("Finder")), + Some("display") + ); +} + +#[test] +fn remaining_wait_budget_stops_at_zero() { + let future_deadline = tokio::time::Instant::now() + Duration::from_millis(250); + assert!(remaining_wait_budget_ms(future_deadline).unwrap() > 0); + + let expired_deadline = tokio::time::Instant::now() - Duration::from_millis(1); + assert_eq!(remaining_wait_budget_ms(expired_deadline), None); +} diff --git a/codex-rs/core/src/tools/handlers/gui/tests/window_selection.rs b/codex-rs/core/src/tools/handlers/gui/tests/window_selection.rs new file mode 100644 index 000000000000..a31de34ba770 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/tests/window_selection.rs @@ -0,0 +1,146 @@ +use super::*; + +#[test] +fn normalize_window_selection_merges_top_level_and_selector_fields() { + let selector = WindowSelector { + title: None, + title_contains: Some(" Settings ".to_string()), + index: Some(2), + }; + + let selection = normalize_window_selection(Some(" Preferences "), Some(&selector)) + .expect("window selection should normalize") + .expect("window selection should exist"); + + assert_eq!(selection.title.as_deref(), Some("Preferences")); + assert_eq!(selection.title_contains.as_deref(), Some("Settings")); + assert_eq!(selection.index, Some(2)); +} + +#[test] +fn resolve_capture_target_uses_window_bounds_when_requested() { + let context = HelperCaptureContext { + app_name: Some("Notes".to_string()), + cursor: HelperPoint { x: 10.0, y: 20.0 }, + display: HelperDisplayDescriptor { + index: 1, + bounds: HelperRect { + x: 0.0, + y: 0.0, + width: 1440.0, + height: 900.0, + }, + }, + window_id: Some(42), + window_title: Some("Quick Note".to_string()), + window_bounds: Some(HelperRect { + x: 100.0, + y: 80.0, + width: 800.0, + height: 600.0, + }), + window_count: Some(3), + window_capture_strategy: Some("bounds".to_string()), + host_self_exclude_applied: Some(false), + host_frontmost_excluded: Some(false), + host_frontmost_app_name: None, + host_frontmost_bundle_id: None, + }; + + let capture = + resolve_capture_target(&context, Some("window"), true, true).expect("window capture"); + + assert_eq!(capture.mode, CaptureMode::Window); + assert_eq!(capture.width, 800); + assert_eq!(capture.height, 600); + assert_eq!(capture.window_title.as_deref(), Some("Quick Note")); + assert_eq!(capture.window_count, Some(3)); + assert_eq!(capture.window_capture_strategy.as_deref(), Some("bounds")); +} + +#[test] +#[cfg(target_os = "macos")] +fn normalize_capture_mode_env_accepts_common_variants() { + assert_eq!(normalize_capture_mode_env("window"), Some("window")); + assert_eq!(normalize_capture_mode_env(" Window "), Some("window")); + assert_eq!(normalize_capture_mode_env("display."), Some("display")); + assert_eq!(normalize_capture_mode_env("DISPLAY"), Some("display")); + assert_eq!(normalize_capture_mode_env("workspace"), None); +} + +#[test] +fn resolve_capture_target_prefers_window_for_in_app_work_when_available() { + let context = HelperCaptureContext { + app_name: Some("Notes".to_string()), + cursor: HelperPoint { x: 10.0, y: 20.0 }, + display: HelperDisplayDescriptor { + index: 1, + bounds: HelperRect { + x: 0.0, + y: 0.0, + width: 1440.0, + height: 900.0, + }, + }, + window_id: Some(42), + window_title: Some("Quick Note".to_string()), + window_bounds: Some(HelperRect { + x: 100.0, + y: 80.0, + width: 800.0, + height: 600.0, + }), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_self_exclude_applied: Some(false), + host_frontmost_excluded: Some(false), + host_frontmost_app_name: None, + host_frontmost_bundle_id: None, + }; + + let capture = resolve_capture_target(&context, None, false, true) + .expect("window should be preferred for in-app work"); + + assert_eq!(capture.mode, CaptureMode::Window); + assert_eq!(capture.width, 800); + assert_eq!(capture.height, 600); +} + +#[test] +fn resolve_capture_target_adjusts_implicit_display_for_host_self_exclude() { + let context = HelperCaptureContext { + app_name: Some("Notes".to_string()), + cursor: HelperPoint { x: 10.0, y: 20.0 }, + display: HelperDisplayDescriptor { + index: 1, + bounds: HelperRect { + x: 0.0, + y: 0.0, + width: 1440.0, + height: 900.0, + }, + }, + window_id: Some(42), + window_title: Some("Quick Note".to_string()), + window_bounds: Some(HelperRect { + x: 100.0, + y: 80.0, + width: 800.0, + height: 600.0, + }), + window_count: Some(1), + window_capture_strategy: Some("bounds".to_string()), + host_self_exclude_applied: Some(true), + host_frontmost_excluded: Some(true), + host_frontmost_app_name: Some("Codex".to_string()), + host_frontmost_bundle_id: Some("com.openai.codex".to_string()), + }; + + let capture = resolve_capture_target(&context, None, false, false) + .expect("implicit display capture should adjust to a safe window capture"); + + assert_eq!(capture.mode, CaptureMode::Window); + assert!(capture.host_self_exclude_adjusted); + assert_eq!(capture.width, 800); + assert_eq!(capture.height, 600); +} diff --git a/codex-rs/core/src/tools/handlers/gui/type_system_events.applescript b/codex-rs/core/src/tools/handlers/gui/type_system_events.applescript new file mode 100644 index 000000000000..70c5a3c4c4ea --- /dev/null +++ b/codex-rs/core/src/tools/handlers/gui/type_system_events.applescript @@ -0,0 +1,205 @@ +on absoluteDifference(lhsValue, rhsValue) + if lhsValue >= rhsValue then return lhsValue - rhsValue + return rhsValue - lhsValue +end absoluteDifference + +on textContains(haystack, needle) + if needle is "" then return true + ignoring case + return (offset of needle in haystack) is not 0 + end ignoring +end textContains + +on windowMatchesBounds(candidateWindow, boundsXText, boundsYText, boundsWidthText, boundsHeightText) + if boundsXText is "" or boundsYText is "" or boundsWidthText is "" or boundsHeightText is "" then return true + try + set {windowX, windowY} to position of candidateWindow + set {windowWidth, windowHeight} to size of candidateWindow + on error + return false + end try + set tolerance to 3 + return (my absoluteDifference(windowX as integer, boundsXText as integer) is less than or equal to tolerance) and (my absoluteDifference(windowY as integer, boundsYText as integer) is less than or equal to tolerance) and (my absoluteDifference(windowWidth as integer, boundsWidthText as integer) is less than or equal to tolerance) and (my absoluteDifference(windowHeight as integer, boundsHeightText as integer) is less than or equal to tolerance) +end windowMatchesBounds + +on matchingWindows(targetProc, exactTitle, titleContains, boundsXText, boundsYText, boundsWidthText, boundsHeightText) + set matches to {} + repeat with candidateWindow in windows of targetProc + set windowTitle to "" + try + set windowTitle to name of candidateWindow as text + end try + set exactMatch to true + if exactTitle is not "" then + ignoring case + set exactMatch to windowTitle is exactTitle + end ignoring + end if + set containsMatch to my textContains(windowTitle, titleContains) + set boundsMatch to my windowMatchesBounds(candidateWindow, boundsXText, boundsYText, boundsWidthText, boundsHeightText) + if exactMatch and containsMatch and boundsMatch then set end of matches to candidateWindow + end repeat + return matches +end matchingWindows + +on focusRequestedWindow(targetProc, exactTitle, titleContains, windowIndexText, boundsXText, boundsYText, boundsWidthText, boundsHeightText) + if exactTitle is "" and titleContains is "" and windowIndexText is "" and boundsXText is "" and boundsYText is "" and boundsWidthText is "" and boundsHeightText is "" then return + set matches to my matchingWindows(targetProc, exactTitle, titleContains, boundsXText, boundsYText, boundsWidthText, boundsHeightText) + if (count of matches) is 0 then error "Window not found for the requested selection." + set targetWindow to item 1 of matches + if windowIndexText is not "" then + set requestedIndex to windowIndexText as integer + if requestedIndex < 1 or requestedIndex > (count of matches) then error "Requested window index is out of range." + set targetWindow to item requestedIndex of matches + end if + tell application "System Events" + try + tell targetWindow to perform action "AXRaise" + end try + try + tell targetWindow to set value of attribute "AXMain" to true + end try + try + tell targetWindow to set value of attribute "AXFocused" to true + end try + end tell + delay 0.1 +end focusRequestedWindow + +on normalizedDelaySeconds(delayMsText, fallbackSeconds) + if delayMsText is "" then return fallbackSeconds + try + set candidateMs to delayMsText as integer + if candidateMs < 0 then return fallbackSeconds + return candidateMs / 1000 + on error + return fallbackSeconds + end try +end normalizedDelaySeconds + +on normalizedRepeatCount(repeatText, fallbackCount) + if repeatText is "" then return fallbackCount + try + set candidateCount to repeatText as integer + if candidateCount < 0 then return fallbackCount + return candidateCount + on error + return fallbackCount + end try +end normalizedRepeatCount + +on pasteText(rawText, preDelaySeconds, postDelaySeconds) + set previousClipboard to missing value + set hadClipboard to false + try + set previousClipboard to the clipboard + set hadClipboard to true + end try + + try + set the clipboard to rawText + delay preDelaySeconds + tell application "System Events" + keystroke "v" using command down + end tell + delay postDelaySeconds + on error errMsg number errNum + if hadClipboard then + try + set the clipboard to previousClipboard + end try + end if + error errMsg number errNum + end try + + if hadClipboard then + try + set the clipboard to previousClipboard + end try + end if +end pasteText + +on clearWithBackspace(repeatCount) + if repeatCount <= 0 then return + tell application "System Events" + repeat repeatCount times + key code 51 + delay 0.02 + end repeat + end tell +end clearWithBackspace + +on enterText(rawText, entryStrategy, preDelaySeconds, postDelaySeconds) + if entryStrategy is "keystroke" then + tell application "System Events" + keystroke rawText + end tell + delay postDelaySeconds + return + end if + if entryStrategy is "keystroke_chars" then + set keyDelayMsText to system attribute "CODEX_GUI_KEYSTROKE_CHAR_DELAY_MS" + set keyDelaySeconds to my normalizedDelaySeconds(keyDelayMsText, 0.055) + tell application "System Events" + repeat with currentCharacter in characters of rawText + set typedCharacter to contents of currentCharacter + if typedCharacter is return or typedCharacter is linefeed then + key code 36 + else + keystroke typedCharacter + end if + delay keyDelaySeconds + end repeat + end tell + delay postDelaySeconds + return + end if + my pasteText(rawText, preDelaySeconds, postDelaySeconds) +end enterText + +on run argv + set requestedApp to system attribute "CODEX_GUI_APP" + set requestedWindowTitle to system attribute "CODEX_GUI_WINDOW_TITLE" + set requestedWindowTitleContains to system attribute "CODEX_GUI_WINDOW_TITLE_CONTAINS" + set requestedWindowIndex to system attribute "CODEX_GUI_WINDOW_INDEX" + set requestedWindowBoundsX to system attribute "CODEX_GUI_WINDOW_BOUNDS_X" + set requestedWindowBoundsY to system attribute "CODEX_GUI_WINDOW_BOUNDS_Y" + set requestedWindowBoundsWidth to system attribute "CODEX_GUI_WINDOW_BOUNDS_WIDTH" + set requestedWindowBoundsHeight to system attribute "CODEX_GUI_WINDOW_BOUNDS_HEIGHT" + set replaceText to system attribute "CODEX_GUI_REPLACE" + set submitText to system attribute "CODEX_GUI_SUBMIT" + set inlineInputText to system attribute "CODEX_GUI_TEXT" + set systemEventsTypeStrategy to system attribute "CODEX_GUI_SYSTEM_EVENTS_TYPE_STRATEGY" + set clearRepeatText to system attribute "CODEX_GUI_CLEAR_REPEAT" + set pastePreDelayMsText to system attribute "CODEX_GUI_PASTE_PRE_DELAY_MS" + set pastePostDelayMsText to system attribute "CODEX_GUI_PASTE_POST_DELAY_MS" + set inputText to inlineInputText + if inputText is "" and (count of argv) > 0 then set inputText to item 1 of argv + set preDelaySeconds to my normalizedDelaySeconds(pastePreDelayMsText, 0.22) + set postDelaySeconds to my normalizedDelaySeconds(pastePostDelayMsText, 0.65) + set replaceRepeatCount to my normalizedRepeatCount(clearRepeatText, 48) + + tell application "System Events" + if requestedApp is not "" then + if not (exists application process requestedApp) then error "Application process not found: " & requestedApp + set targetProc to application process requestedApp + set frontmost of targetProc to true + delay 0.1 + else + set targetProc to first application process whose frontmost is true + end if + my focusRequestedWindow(targetProc, requestedWindowTitle, requestedWindowTitleContains, requestedWindowIndex, requestedWindowBoundsX, requestedWindowBoundsY, requestedWindowBoundsWidth, requestedWindowBoundsHeight) + + if replaceText is "1" then + if systemEventsTypeStrategy is "keystroke" or clearRepeatText is not "" then + my clearWithBackspace(replaceRepeatCount) + else + keystroke "a" using command down + end if + end if + + my enterText(inputText, systemEventsTypeStrategy, preDelaySeconds, postDelaySeconds) + if submitText is "1" then key code 36 + return "typed" + end tell +end run diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index f0a62b8c148f..d476d31a287e 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod agent_jobs; pub mod apply_patch; mod dynamic; +mod gui; mod js_repl; mod list_dir; mod mcp; @@ -36,6 +37,7 @@ pub use apply_patch::ApplyPatchHandler; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; pub use dynamic::DynamicToolHandler; +pub use gui::GuiHandler; pub use js_repl::JsReplHandler; pub use js_repl::JsReplResetHandler; pub use list_dir::ListDirHandler; diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 341db2476722..93992f68bbab 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -40,6 +40,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::CodeModeExecuteHandler; use crate::tools::handlers::CodeModeWaitHandler; use crate::tools::handlers::DynamicToolHandler; + use crate::tools::handlers::GuiHandler; use crate::tools::handlers::JsReplHandler; use crate::tools::handlers::JsReplResetHandler; use crate::tools::handlers::ListDirHandler; @@ -103,6 +104,7 @@ pub(crate) fn build_specs_with_discoverable_tools( let plan_handler = Arc::new(PlanHandler); let apply_patch_handler = Arc::new(ApplyPatchHandler); let dynamic_tool_handler = Arc::new(DynamicToolHandler); + let gui_handler = Arc::new(GuiHandler::default()); let view_image_handler = Arc::new(ViewImageHandler); let mcp_handler = Arc::new(McpHandler); let mcp_resource_handler = Arc::new(McpResourceHandler); @@ -154,6 +156,9 @@ pub(crate) fn build_specs_with_discoverable_tools( ToolHandlerKind::DynamicTool => { builder.register_handler(handler.name, dynamic_tool_handler.clone()); } + ToolHandlerKind::Gui => { + builder.register_handler(handler.name, gui_handler.clone()); + } ToolHandlerKind::JsRepl => { builder.register_handler(handler.name, js_repl_handler.clone()); } diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index 4e73be35022c..0051462ef3c1 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -240,6 +240,107 @@ fn get_memory_requires_feature_flag() { ); } +#[test] +fn gui_tools_present_when_feature_enabled() { + let config = test_config(); + let model_info = ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::GuiTools); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + let (tools, _) = build_specs( + &tools_config, + /*mcp_tools*/ None, + /*app_tools*/ None, + &[], + ) + .build(); + + assert_contains_tool_names( + &tools, + &[ + "gui_observe", + "gui_wait", + "gui_click", + "gui_drag", + "gui_scroll", + "gui_type", + "gui_key", + "gui_move", + ], + ); +} + +#[test] +fn gui_coordinate_targeting_adds_coordinate_fields_to_gui_tools() { + let config = test_config(); + let model_info = ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::GuiTools); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }) + .with_gui_coordinate_targeting(true); + let (tools, _) = build_specs( + &tools_config, + /*mcp_tools*/ None, + /*app_tools*/ None, + &[], + ) + .build(); + + let click_tool = find_tool(&tools, "gui_click"); + let drag_tool = find_tool(&tools, "gui_drag"); + + let ToolSpec::Function(click_fn) = &click_tool.spec else { + panic!("expected gui_click to be a function tool"); + }; + let JsonSchema::Object { + properties: click_props, + required: click_required, + .. + } = &click_fn.parameters + else { + panic!("expected gui_click parameters to be an object"); + }; + assert!(click_props.contains_key("x")); + assert!(click_props.contains_key("y")); + assert!(click_props.contains_key("coordinate_space")); + assert_eq!(click_required, &None); + + let ToolSpec::Function(drag_fn) = &drag_tool.spec else { + panic!("expected gui_drag to be a function tool"); + }; + let JsonSchema::Object { + properties: drag_props, + required: drag_required, + .. + } = &drag_fn.parameters + else { + panic!("expected gui_drag parameters to be an object"); + }; + assert!(drag_props.contains_key("from_x")); + assert!(drag_props.contains_key("from_y")); + assert!(drag_props.contains_key("to_x")); + assert!(drag_props.contains_key("to_y")); + assert!(drag_props.contains_key("coordinate_space")); + assert_eq!(drag_required, &None); +} fn assert_model_tools( model_slug: &str, features: &Features, diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index df1cb31a4398..42e0f4804ff7 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -82,6 +82,14 @@ impl EventProcessorWithHumanOutput { "started".style(self.dimmed) ); } + ThreadItem::BuiltinToolCall { tool, .. } => { + eprintln!( + "{} {} {}", + "tool:".style(self.bold), + tool.style(self.cyan), + "started".style(self.dimmed) + ); + } ThreadItem::WebSearch { query, .. } => { eprintln!("{} {}", "web search:".style(self.bold), query); } @@ -197,6 +205,37 @@ impl EventProcessorWithHumanOutput { eprintln!("{}", error.message.style(self.red)); } } + ThreadItem::BuiltinToolCall { + tool, + status, + output, + .. + } => { + let status_text = match status { + codex_app_server_protocol::BuiltinToolCallStatus::Completed => { + "completed".style(self.green) + } + codex_app_server_protocol::BuiltinToolCallStatus::Failed => { + "failed".style(self.red) + } + codex_app_server_protocol::BuiltinToolCallStatus::InProgress => { + "in_progress".style(self.dimmed) + } + }; + eprintln!( + "{} {} {}", + "tool:".style(self.bold), + tool.style(self.cyan), + format!("({status_text})").style(self.dimmed) + ); + if let Some(summary) = output + .as_ref() + .and_then(codex_protocol::items::builtin_tool_output_summary) + && !summary.trim().is_empty() + { + eprintln!("{}", summary.style(self.dimmed)); + } + } ThreadItem::WebSearch { query, .. } => { eprintln!("{} {}", "web search:".style(self.bold), query); } @@ -283,6 +322,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.render_item_completed(notification.item); CodexStatus::Running } + ServerNotification::RawResponseItemCompleted(_) => CodexStatus::Running, ServerNotification::ModelRerouted(notification) => { eprintln!( "{} {} -> {}", diff --git a/codex-rs/exec/src/event_processor_with_jsonl_output.rs b/codex-rs/exec/src/event_processor_with_jsonl_output.rs index 1a085d93d0f3..3e3c2ffb2c1d 100644 --- a/codex-rs/exec/src/event_processor_with_jsonl_output.rs +++ b/codex-rs/exec/src/event_processor_with_jsonl_output.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use codex_app_server_protocol::BuiltinToolCallStatus as AppServerBuiltinToolCallStatus; use codex_app_server_protocol::CollabAgentTool; use codex_app_server_protocol::CollabAgentToolCallStatus; use codex_app_server_protocol::CommandExecutionStatus; @@ -22,6 +23,8 @@ pub use crate::event_processor::CodexStatus; use crate::event_processor::EventProcessor; use crate::event_processor::handle_last_message; use crate::exec_events::AgentMessageItem; +use crate::exec_events::BuiltinToolCallItem; +use crate::exec_events::BuiltinToolCallStatus; use crate::exec_events::CollabAgentState; use crate::exec_events::CollabAgentStatus; use crate::exec_events::CollabTool; @@ -229,6 +232,35 @@ impl EventProcessorWithJsonOutput { }), }), }), + ThreadItem::BuiltinToolCall { + call_id, + tool, + namespace, + arguments, + output, + success, + status, + .. + } => Some(ExecThreadItem { + id: make_id(), + details: ThreadItemDetails::BuiltinToolCall(BuiltinToolCallItem { + call_id, + tool, + namespace, + arguments, + output, + success, + status: match status { + AppServerBuiltinToolCallStatus::InProgress => { + BuiltinToolCallStatus::InProgress + } + AppServerBuiltinToolCallStatus::Completed => { + BuiltinToolCallStatus::Completed + } + AppServerBuiltinToolCallStatus::Failed => BuiltinToolCallStatus::Failed, + }, + }), + }), ThreadItem::CollabAgentToolCall { tool, sender_thread_id, @@ -475,6 +507,7 @@ impl EventProcessorWithJsonOutput { } CodexStatus::Running } + ServerNotification::RawResponseItemCompleted(_) => CodexStatus::Running, ServerNotification::ModelRerouted(notification) => { events.push(ThreadEvent::ItemCompleted(ItemCompletedEvent { item: ExecThreadItem { @@ -676,4 +709,95 @@ mod tests { "keep existing contents" ); } + + #[test] + fn emits_builtin_function_tool_call_events_from_thread_items() { + let mut processor = EventProcessorWithJsonOutput::new(None); + + let started = processor.collect_thread_events(ServerNotification::ItemStarted( + codex_app_server_protocol::ItemStartedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item: ThreadItem::BuiltinToolCall { + id: "call-1".to_string(), + call_id: "call-1".to_string(), + tool: "gui_observe".to_string(), + namespace: Some("builtin".to_string()), + arguments: json!({ + "app": "TextEdit", + "capture_mode": "window", + }), + output: None, + success: None, + status: codex_app_server_protocol::BuiltinToolCallStatus::InProgress, + }, + }, + )); + + assert_eq!(started.status, CodexStatus::Running); + assert_eq!(started.events.len(), 1); + let started_item = match &started.events[0] { + ThreadEvent::ItemStarted(ItemStartedEvent { item }) => item, + other => panic!("expected item.started event, got {other:?}"), + }; + assert_eq!(started_item.id, "item_0"); + assert_eq!( + started_item.details, + ThreadItemDetails::BuiltinToolCall(BuiltinToolCallItem { + call_id: "call-1".to_string(), + tool: "gui_observe".to_string(), + namespace: Some("builtin".to_string()), + arguments: json!({ + "app": "TextEdit", + "capture_mode": "window", + }), + output: None, + success: None, + status: BuiltinToolCallStatus::InProgress, + }) + ); + + let completed = processor.collect_thread_events(ServerNotification::ItemCompleted( + codex_app_server_protocol::ItemCompletedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item: ThreadItem::BuiltinToolCall { + id: "call-1".to_string(), + call_id: "call-1".to_string(), + tool: "gui_observe".to_string(), + namespace: Some("builtin".to_string()), + arguments: json!({ + "app": "TextEdit", + "capture_mode": "window", + }), + output: Some(json!("{\"code\":\"observed\"}")), + success: Some(true), + status: codex_app_server_protocol::BuiltinToolCallStatus::Completed, + }, + }, + )); + + assert_eq!(completed.status, CodexStatus::Running); + assert_eq!(completed.events.len(), 1); + let completed_item = match &completed.events[0] { + ThreadEvent::ItemCompleted(ItemCompletedEvent { item }) => item, + other => panic!("expected item.completed event, got {other:?}"), + }; + assert_eq!(completed_item.id, "item_0"); + assert_eq!( + completed_item.details, + ThreadItemDetails::BuiltinToolCall(BuiltinToolCallItem { + call_id: "call-1".to_string(), + tool: "gui_observe".to_string(), + namespace: Some("builtin".to_string()), + arguments: json!({ + "app": "TextEdit", + "capture_mode": "window", + }), + output: Some(json!("{\"code\":\"observed\"}")), + success: Some(true), + status: BuiltinToolCallStatus::Completed, + }) + ); + } } diff --git a/codex-rs/exec/src/exec_events.rs b/codex-rs/exec/src/exec_events.rs index d356a6a70b7b..12adeebc2fb6 100644 --- a/codex-rs/exec/src/exec_events.rs +++ b/codex-rs/exec/src/exec_events.rs @@ -114,6 +114,10 @@ pub enum ThreadItemDetails { /// Represents a call to an MCP tool. The item starts when the invocation is /// dispatched and completes when the MCP server reports success or failure. McpToolCall(McpToolCallItem), + /// Represents a built-in function tool call such as native GUI tools. The + /// item starts when the model emits a function call and completes when the + /// corresponding function call output arrives. + BuiltinToolCall(BuiltinToolCallItem), /// Represents a call to a collab tool. The item starts when the collab tool is /// invoked and completes when the collab tool reports success or failure. CollabToolCall(CollabToolCallItem), @@ -202,6 +206,16 @@ pub enum McpToolCallStatus { Failed, } +/// The status of a built-in function tool call. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default, TS)] +#[serde(rename_all = "snake_case")] +pub enum BuiltinToolCallStatus { + #[default] + InProgress, + Completed, + Failed, +} + /// The status of a collab tool call. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default, TS)] #[serde(rename_all = "snake_case")] @@ -285,6 +299,19 @@ pub struct McpToolCallItem { pub status: McpToolCallStatus, } +/// A call to a built-in function tool. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TS)] +pub struct BuiltinToolCallItem { + pub call_id: String, + pub tool: String, + pub namespace: Option, + #[serde(default)] + pub arguments: JsonValue, + pub output: Option, + pub success: Option, + pub status: BuiltinToolCallStatus, +} + /// A web search request. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TS)] pub struct WebSearchItem { diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index c87b6dc9ff3e..b69bb9b81368 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -152,6 +152,8 @@ pub enum Feature { Plugins, /// Allow the model to invoke the built-in image generation tool. ImageGeneration, + /// Enable native macOS GUI observation and input tools. + GuiTools, /// Allow prompting and installing missing MCP dependencies. SkillMcpDependencyInstall, /// Prompt for missing skill env var dependencies. @@ -755,6 +757,16 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::GuiTools, + key: "gui_tools", + stage: Stage::Experimental { + name: "Native GUI Tools", + menu_description: "Enable native macOS GUI screenshot, click, drag, scroll, typing, and keypress tools for direct desktop interaction.", + announcement: "NEW: Native GUI Tools are now available in /experimental on macOS.", + }, + default_enabled: false, + }, FeatureSpec { id: Feature::SkillMcpDependencyInstall, key: "skill_mcp_dependency_install", diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 0777262065d0..8fe5fb55bb82 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -150,6 +150,26 @@ fn image_generation_is_under_development() { assert_eq!(Feature::ImageGeneration.default_enabled(), false); } +#[test] +fn gui_tools_is_experimental_and_user_toggleable() { + let spec = Feature::GuiTools.info(); + let stage = spec.stage; + + assert!(matches!(stage, Stage::Experimental { .. })); + assert_eq!(stage.experimental_menu_name(), Some("Native GUI Tools")); + assert_eq!( + stage.experimental_menu_description(), + Some( + "Enable native macOS GUI screenshot, click, drag, scroll, typing, and keypress tools for direct desktop interaction." + ) + ); + assert_eq!( + stage.experimental_announcement(), + Some("NEW: Native GUI Tools are now available in /experimental on macOS.") + ); + assert_eq!(Feature::GuiTools.default_enabled(), false); +} + #[test] fn tool_call_mcp_elicitation_is_stable_and_enabled_by_default() { assert_eq!(Feature::ToolCallMcpElicitation.stage(), Stage::Stable); diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 72a9551336eb..4c20c5a1b909 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -1,8 +1,10 @@ use crate::memory_citation::MemoryCitation; use crate::models::ContentItem; +use crate::models::FunctionCallOutputContentItem; use crate::models::MessagePhase; use crate::models::ResponseItem; use crate::models::WebSearchAction; +use crate::models::function_call_output_content_items_to_text; use crate::protocol::AgentMessageEvent; use crate::protocol::AgentReasoningEvent; use crate::protocol::AgentReasoningRawContentEvent; @@ -19,6 +21,7 @@ use quick_xml::se::to_string as to_xml_string; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use serde_json::Value as JsonValue; use ts_rs::TS; #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] @@ -30,6 +33,7 @@ pub enum TurnItem { AgentMessage(AgentMessageItem), Plan(PlanItem), Reasoning(ReasoningItem), + BuiltinToolCall(BuiltinToolCallItem), WebSearch(WebSearchItem), ImageGeneration(ImageGenerationItem), ContextCompaction(ContextCompactionItem), @@ -106,6 +110,63 @@ pub struct ReasoningItem { pub raw_content: Vec, } +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "items/")] +pub enum BuiltinToolCallStatus { + InProgress, + Completed, + Failed, +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct BuiltinToolCallItem { + pub id: String, + pub call_id: String, + pub tool: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub namespace: Option, + #[serde(default)] + pub arguments: JsonValue, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub success: Option, + pub status: BuiltinToolCallStatus, +} + +impl BuiltinToolCallItem { + /// Return a short human-readable summary of the tool output, if available. + pub fn output_summary(&self) -> Option { + builtin_tool_output_summary(self.output.as_ref()?) + } +} + +/// Summarize a builtin tool output value for display. +pub fn builtin_tool_output_summary(output: &JsonValue) -> Option { + match output { + JsonValue::String(text) if !text.trim().is_empty() => Some(text.clone()), + JsonValue::Array(items) => { + if items.iter().any(|item| { + item.get("type") + .and_then(JsonValue::as_str) + .is_some_and(|kind| kind == "input_image" || kind == "inputImage") + }) { + return Some("".to_string()); + } + + serde_json::from_value::>(output.clone()) + .ok() + .and_then(|items| function_call_output_content_items_to_text(&items)) + } + _ => None, + } +} + #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] pub struct WebSearchItem { pub id: String, @@ -389,6 +450,7 @@ impl TurnItem { TurnItem::AgentMessage(item) => item.id.clone(), TurnItem::Plan(item) => item.id.clone(), TurnItem::Reasoning(item) => item.id.clone(), + TurnItem::BuiltinToolCall(item) => item.id.clone(), TurnItem::WebSearch(item) => item.id.clone(), TurnItem::ImageGeneration(item) => item.id.clone(), TurnItem::ContextCompaction(item) => item.id.clone(), @@ -401,6 +463,7 @@ impl TurnItem { TurnItem::HookPrompt(_) => Vec::new(), TurnItem::AgentMessage(item) => item.as_legacy_events(), TurnItem::Plan(_) => Vec::new(), + TurnItem::BuiltinToolCall(_) => Vec::new(), TurnItem::WebSearch(item) => vec![item.as_legacy_event()], TurnItem::ImageGeneration(item) => vec![item.as_legacy_event()], TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning), @@ -445,4 +508,30 @@ mod tests { } ); } + + #[test] + fn builtin_tool_output_summary_joins_structured_text_items() { + let output = serde_json::json!([ + {"type": "input_text", "text": "first line"}, + {"type": "input_text", "text": "second line"} + ]); + + assert_eq!( + builtin_tool_output_summary(&output), + Some("first line\nsecond line".to_string()) + ); + } + + #[test] + fn builtin_tool_output_summary_prefers_image_marker_for_image_items() { + let output = serde_json::json!([ + {"type": "input_text", "text": "captured"}, + {"type": "input_image", "image_url": "data:image/png;base64,abc"} + ]); + + assert_eq!( + builtin_tool_output_summary(&output), + Some("".to_string()) + ); + } } diff --git a/codex-rs/tools/src/gui_tool.rs b/codex-rs/tools/src/gui_tool.rs new file mode 100644 index 000000000000..44f771a16d8e --- /dev/null +++ b/codex-rs/tools/src/gui_tool.rs @@ -0,0 +1,1246 @@ +use crate::JsonSchema; +use crate::ResponsesApiTool; +use crate::ToolSpec; +use std::collections::BTreeMap; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct GuiToolSchemaOptions { + pub coordinate_targeting: bool, +} + +fn string_enum_description(values: &[&str], extra: &str) -> String { + format!("Supported values: {}. {extra}", values.join(", ")) +} + +fn coordinate_space_description() -> String { + string_enum_description( + &["image_pixels", "display_points"], + "`image_pixels` means coordinates relative to the latest `gui_observe` screenshot for this conversation. `display_points` means absolute macOS display coordinates in logical points.", + ) +} + +fn window_selector_schema() -> JsonSchema { + JsonSchema::Object { + properties: BTreeMap::from([ + ( + "title".to_string(), + JsonSchema::String { + description: Some( + "Optional exact visible window title. Use this when the current app has multiple similarly named windows and you want an exact match." + .to_string(), + ), + }, + ), + ( + "title_contains".to_string(), + JsonSchema::String { + description: Some( + "Optional visible substring of the target window title. Prefer this when only part of the window title is stable or visible." + .to_string(), + ), + }, + ), + ( + "index".to_string(), + JsonSchema::Number { + description: Some( + "Optional 1-based index among matching visible windows after applying title filters. Use this as a last disambiguator when multiple windows still match." + .to_string(), + ), + }, + ), + ]), + required: None, + additional_properties: Some(false.into()), + } +} + +fn with_capture_selection_properties( + mut properties: BTreeMap, + include_window_title: bool, +) -> BTreeMap { + properties.insert( + "capture_mode".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["display", "window"], + "Use `window` for app-local work where a single window is the stable frame of reference. Use `display` for desktop-wide UI such as the Dock, menu bar, permission prompts, or cross-window drags. When omitted, GUI tools prefer `window` for in-app or window-targeted work and fall back to `display` otherwise.", + )), + }, + ); + if include_window_title { + properties.insert( + "window_title".to_string(), + JsonSchema::String { + description: Some( + "Optional exact visible window title to capture or focus. Reuse the same `window_title` across related GUI steps when you want the runtime to stay on the same surface." + .to_string(), + ), + }, + ); + } + properties.insert( + "window_selector".to_string(), + JsonSchema::Object { + properties: match window_selector_schema() { + JsonSchema::Object { properties, .. } => properties, + _ => unreachable!("window selector schema must be an object"), + }, + required: None, + additional_properties: Some(false.into()), + }, + ); + properties +} + +fn with_target_properties( + mut properties: BTreeMap, + action_description: &str, +) -> BTreeMap { + properties.insert( + "target".to_string(), + JsonSchema::String { + description: Some(format!( + "Optional semantic GUI target to resolve before {action_description}. Describe the actionable or editable control itself using visible screenshot evidence: prefer the exact on-screen text, icon, state, nearby context, and coarse location that make it unique, such as `Save button`, `Search field`, or `Muted Notifications toggle in the top-right toolbar`." + )), + }, + ); + properties.insert( + "location_hint".to_string(), + JsonSchema::String { + description: Some( + "Optional coarse position hint such as `top right`, `left sidebar`, or `near the bottom`. Use this when multiple similar controls are visible." + .to_string(), + ), + }, + ); + properties.insert( + "scope".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic region that narrows the search area, such as `left sidebar`, `toolbar`, or `composer pane`. Prefer naming the real surrounding region instead of generic container chrome." + .to_string(), + ), + }, + ); + properties.insert( + "grounding_mode".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["single", "complex"], + "Optional grounding hint. `single` suits simple isolated controls. `complex` enables the heavier validation and retry path for dense, ambiguous, or visually noisy layouts, and is a good retry mode after an initial miss.", + )), + }, + ); + properties +} + +fn with_drag_target_properties( + mut properties: BTreeMap, +) -> BTreeMap { + properties.insert( + "grounding_mode".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["single", "complex"], + "Optional grounding hint shared by drag source and destination. `single` suits simple isolated controls. `complex` enables the heavier validation and retry path for dense or ambiguous layouts and is a good retry mode after an initial miss.", + )), + }, + ); + properties.insert( + "from_target".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic GUI drag source to resolve before dragging. Describe the draggable surface itself using visible screenshot evidence, such as `Selected tab`, `Message row`, or `Resize handle near the lower-right corner`." + .to_string(), + ), + }, + ); + properties.insert( + "from_location_hint".to_string(), + JsonSchema::String { + description: Some( + "Optional coarse position hint for `from_target`, such as `left sidebar` or `near the top`." + .to_string(), + ), + }, + ); + properties.insert( + "from_scope".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic region that narrows `from_target`, such as `left sidebar` or `active tab strip`." + .to_string(), + ), + }, + ); + properties.insert( + "to_target".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic GUI drag destination to resolve before dragging. Describe the drop surface itself using visible screenshot evidence, such as `Trash`, `Calendar column`, or `right pane drop zone`." + .to_string(), + ), + }, + ); + properties.insert( + "to_location_hint".to_string(), + JsonSchema::String { + description: Some( + "Optional coarse position hint for `to_target`, such as `bottom right` or `in the center`." + .to_string(), + ), + }, + ); + properties.insert( + "to_scope".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic region that narrows `to_target`, such as `timeline`, `calendar grid`, or `drop zone`." + .to_string(), + ), + }, + ); + properties +} + +fn with_coordinate_click_properties( + mut properties: BTreeMap, +) -> BTreeMap { + properties.insert( + "x".to_string(), + JsonSchema::Number { + description: Some( + "Direct click X coordinate. When `coordinate_space` is `image_pixels`, this is relative to the latest `gui_observe` screenshot. When `coordinate_space` is `display_points`, this is an absolute macOS display X coordinate in logical points." + .to_string(), + ), + }, + ); + properties.insert( + "y".to_string(), + JsonSchema::Number { + description: Some( + "Direct click Y coordinate. When `coordinate_space` is `image_pixels`, this is relative to the latest `gui_observe` screenshot. When `coordinate_space` is `display_points`, this is an absolute macOS display Y coordinate in logical points." + .to_string(), + ), + }, + ); + properties.insert( + "coordinate_space".to_string(), + JsonSchema::String { + description: Some(coordinate_space_description()), + }, + ); + properties +} + +fn with_coordinate_drag_properties( + mut properties: BTreeMap, +) -> BTreeMap { + for (name, description) in [ + ( + "from_x", + "Direct drag source X coordinate in the selected `coordinate_space`.", + ), + ( + "from_y", + "Direct drag source Y coordinate in the selected `coordinate_space`.", + ), + ( + "to_x", + "Direct drag destination X coordinate in the selected `coordinate_space`.", + ), + ( + "to_y", + "Direct drag destination Y coordinate in the selected `coordinate_space`.", + ), + ] { + properties.insert( + name.to_string(), + JsonSchema::Number { + description: Some(description.to_string()), + }, + ); + } + properties.insert( + "coordinate_space".to_string(), + JsonSchema::String { + description: Some(coordinate_space_description()), + }, + ); + properties +} + +pub fn create_gui_observe_tool() -> ToolSpec { + create_gui_observe_tool_with_options(GuiToolSchemaOptions::default()) +} + +pub fn create_gui_observe_tool_with_options(_options: GuiToolSchemaOptions) -> ToolSpec { + let properties = with_target_properties( + with_capture_selection_properties( + BTreeMap::from([ + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before capturing. Defaults to the current frontmost app." + .to_string(), + ), + }, + ), + ( + "return_image".to_string(), + JsonSchema::Boolean { + description: Some( + "Whether to attach the captured screenshot image. Defaults to true." + .to_string(), + ), + }, + ), + ]), + true, + ), + "observing a semantic GUI target", + ); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_observe".to_string(), + description: "Capture a screenshot of the current native GUI surface for visual inspection and follow-up GUI grounding. Supports display-wide capture and focused-window capture, and can also resolve a semantic `target` within the observed GUI." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: None, + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_wait_tool() -> ToolSpec { + let properties = with_target_properties(with_capture_selection_properties( + BTreeMap::from([ + ( + "state".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["appear", "disappear"], + "Defaults to `appear`.", + )), + }, + ), + ( + "timeout_ms".to_string(), + JsonSchema::Number { + description: Some( + "Maximum time to wait for `target` to satisfy `state`. Defaults to 8000." + .to_string(), + ), + }, + ), + ( + "interval_ms".to_string(), + JsonSchema::Number { + description: Some( + "Polling interval between semantic target checks. Defaults to 350." + .to_string(), + ), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before refreshing the screenshot. Defaults to the current frontmost app." + .to_string(), + ), + }, + ), + ]), + true, + ), "waiting for a semantic GUI target"); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_wait".to_string(), + description: "Repeatedly refresh the current native GUI screenshot until a semantic target appears or disappears. Uses consecutive confirmations for stability and reuses the previous gui_observe capture selection when no explicit capture selection is provided." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["target".to_string()]), + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_click_tool() -> ToolSpec { + create_gui_click_tool_with_options(GuiToolSchemaOptions::default()) +} + +pub fn create_gui_click_tool_with_options(options: GuiToolSchemaOptions) -> ToolSpec { + let base_properties = with_target_properties( + with_capture_selection_properties( + BTreeMap::from([ + ( + "button".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["left", "right", "none"], + "Use `none` for hover-only pointer movement. Defaults to `left`.", + )), + }, + ), + ( + "clicks".to_string(), + JsonSchema::Number { + description: Some( + "Number of clicks to send. Defaults to 1. Use 2 for a double-click." + .to_string(), + ), + }, + ), + ( + "hold_ms".to_string(), + JsonSchema::Number { + description: Some( + "Optional press-and-hold duration in milliseconds before releasing. Use this for long-press interactions." + .to_string(), + ), + }, + ), + ( + "settle_ms".to_string(), + JsonSchema::Number { + description: Some( + "Optional hover settle time in milliseconds when `button` is `none`. Defaults to 200." + .to_string(), + ), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before clicking." + .to_string(), + ), + }, + ), + ]), + true, + ), + "clicking or hovering", + ); + let properties = if options.coordinate_targeting { + with_coordinate_click_properties(base_properties) + } else { + base_properties + }; + let required = if options.coordinate_targeting { + None + } else { + Some(vec!["target".to_string()]) + }; + let description = if options.coordinate_targeting { + "Click, right-click, double-click, hover, or click-and-hold in the current native GUI. Use semantic `target` fields for the supported grounding workflow. The optional `x`, `y`, and `coordinate_space` fields are kept as an experimental direct-coordinate placeholder and are disabled by default." + .to_string() + } else { + "Click, right-click, double-click, hover, or click-and-hold on a semantic target in the current native GUI." + .to_string() + }; + + ToolSpec::Function(ResponsesApiTool { + name: "gui_click".to_string(), + description, + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required, + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_drag_tool() -> ToolSpec { + create_gui_drag_tool_with_options(GuiToolSchemaOptions::default()) +} + +pub fn create_gui_drag_tool_with_options(options: GuiToolSchemaOptions) -> ToolSpec { + let base_properties = with_drag_target_properties(with_capture_selection_properties( + BTreeMap::from([ + ( + "duration_ms".to_string(), + JsonSchema::Number { + description: Some( + "Optional drag duration in milliseconds. Defaults to 450.".to_string(), + ), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before dragging.".to_string(), + ), + }, + ), + ]), + true, + )); + let properties = if options.coordinate_targeting { + with_coordinate_drag_properties(base_properties) + } else { + base_properties + }; + let required = if options.coordinate_targeting { + None + } else { + Some(vec!["from_target".to_string(), "to_target".to_string()]) + }; + let description = if options.coordinate_targeting { + "Drag in the current native GUI. Use semantic `from_target` and `to_target` for the supported grounding workflow. The optional `from_x`, `from_y`, `to_x`, `to_y`, and `coordinate_space` fields are kept as an experimental direct-coordinate placeholder and are disabled by default." + .to_string() + } else { + "Drag between semantic `from_target` and `to_target` points in the current native GUI." + .to_string() + }; + + ToolSpec::Function(ResponsesApiTool { + name: "gui_drag".to_string(), + description, + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required, + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_scroll_tool() -> ToolSpec { + let properties = with_target_properties( + with_capture_selection_properties(BTreeMap::from([ + ( + "direction".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["up", "down", "left", "right"], + "Scroll direction. Defaults to `down`.", + )), + }, + ), + ( + "distance".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["small", "medium", "page"], + "Semantic scroll distance. Defaults to `page` for targetless scrolls and `medium` for grounded scrolls.", + )), + }, + ), + ( + "amount".to_string(), + JsonSchema::Number { + description: Some( + "Optional explicit legacy line-count override. When provided, it overrides `distance`." + .to_string(), + ), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before scrolling." + .to_string(), + ), + }, + ), + ]), true), + "scrolling", + ); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_scroll".to_string(), + description: "Scroll in the current native GUI. Defaults to a targetless scroll on the current surface, or provide `target` to scroll a semantic region. Prefer `direction` with semantic `distance`; use `amount` only when you need an explicit legacy line count." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: None, + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_type_tool() -> ToolSpec { + let properties = with_target_properties( + with_capture_selection_properties(BTreeMap::from([ + ( + "value".to_string(), + JsonSchema::String { + description: Some("Literal text to type into the currently focused control." + .to_string()), + }, + ), + ( + "secret_env_var".to_string(), + JsonSchema::String { + description: Some( + "Environment variable name whose value should be typed without exposing the literal secret in the tool call." + .to_string(), + ), + }, + ), + ( + "secret_command_env_var".to_string(), + JsonSchema::String { + description: Some( + "Environment variable name containing a local shell command whose stdout should be typed without exposing the secret in the tool call." + .to_string(), + ), + }, + ), + ( + "replace".to_string(), + JsonSchema::Boolean { + description: Some( + "Whether to replace the current field contents before typing. Defaults to true." + .to_string(), + ), + }, + ), + ( + "submit".to_string(), + JsonSchema::Boolean { + description: Some( + "Whether to press Return after typing. Defaults to false.".to_string(), + ), + }, + ), + ( + "type_strategy".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &[ + "clipboard_paste", + "physical_keys", + "system_events_paste", + "system_events_keystroke", + "system_events_keystroke_chars", + ], + "When omitted, the runtime chooses the default native typing path.", + )), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before typing." + .to_string(), + ), + }, + ), + ]), true), + "typing into a semantic input target", + ); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_type".to_string(), + description: + "Type text into the currently focused native GUI control. Typically use gui_click first to focus the desired field, or provide `target` so the tool focuses the semantic input target for you. Provide exactly one of `value`, `secret_env_var`, or `secret_command_env_var`." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: None, + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_key_tool() -> ToolSpec { + let properties = with_capture_selection_properties( + BTreeMap::from([ + ( + "key".to_string(), + JsonSchema::String { + description: Some( + "Key to press, such as `Enter`, `Tab`, `Escape`, `ArrowDown`, or `s`." + .to_string(), + ), + }, + ), + ( + "modifiers".to_string(), + JsonSchema::Array { + items: Box::new(JsonSchema::String { + description: Some( + "Modifier name such as `command`, `shift`, `option`, or `control`." + .to_string(), + ), + }), + description: Some("Optional modifier list.".to_string()), + }, + ), + ( + "repeat".to_string(), + JsonSchema::Number { + description: Some( + "How many times to press the key. Defaults to 1.".to_string(), + ), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before pressing the key." + .to_string(), + ), + }, + ), + ]), + true, + ); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_key".to_string(), + description: "Press a key or hotkey in the current native GUI.".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["key".to_string()]), + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_batch_tool() -> ToolSpec { + let step_properties = BTreeMap::from([ + ( + "action".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["click", "type", "key", "scroll", "drag"], + "The GUI action to perform in this step.", + )), + }, + ), + // Semantic targeting (click, type, scroll) + ( + "target".to_string(), + JsonSchema::String { + description: Some( + "Semantic GUI target for this step. Required for `click`. Optional for `type` (to focus a field first) and `scroll` (to target a scrollable region). Describe the control using visible screenshot evidence." + .to_string(), + ), + }, + ), + ( + "location_hint".to_string(), + JsonSchema::String { + description: Some( + "Optional coarse position hint such as `top right` or `left sidebar`." + .to_string(), + ), + }, + ), + ( + "scope".to_string(), + JsonSchema::String { + description: Some( + "Optional semantic region that narrows the search area, such as `toolbar` or `dialog`." + .to_string(), + ), + }, + ), + // Click-specific + ( + "button".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["left", "right", "none"], + "For `click` action. Defaults to `left`.", + )), + }, + ), + ( + "clicks".to_string(), + JsonSchema::Number { + description: Some( + "For `click` action. Number of clicks. Defaults to 1. Use 2 for double-click." + .to_string(), + ), + }, + ), + ( + "hold_ms".to_string(), + JsonSchema::Number { + description: Some( + "For `click` action. Press-and-hold duration in milliseconds.".to_string(), + ), + }, + ), + ( + "settle_ms".to_string(), + JsonSchema::Number { + description: Some( + "For `click` action with `button: none`. Hover settle time in ms. Defaults to 200." + .to_string(), + ), + }, + ), + // Type-specific + ( + "value".to_string(), + JsonSchema::String { + description: Some( + "For `type` action. The text to type into the focused control.".to_string(), + ), + }, + ), + ( + "replace".to_string(), + JsonSchema::Boolean { + description: Some( + "For `type` action. If true (default), selects all existing text before typing." + .to_string(), + ), + }, + ), + ( + "submit".to_string(), + JsonSchema::Boolean { + description: Some( + "For `type` action. If true, presses Enter after typing. Defaults to false." + .to_string(), + ), + }, + ), + ( + "type_strategy".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &[ + "clipboard_paste", + "physical_keys", + "system_events_paste", + "system_events_keystroke", + "system_events_keystroke_chars", + ], + "For `type` action. Defaults to `clipboard_paste`.", + )), + }, + ), + // Key-specific + ( + "key".to_string(), + JsonSchema::String { + description: Some( + "For `key` action. Key name such as `Enter`, `Escape`, or a single character." + .to_string(), + ), + }, + ), + ( + "modifiers".to_string(), + JsonSchema::Array { + items: Box::new(JsonSchema::String { + description: None, + }), + description: Some( + "For `key` action. Modifier keys such as `command`, `shift`, `option`, `control`." + .to_string(), + ), + }, + ), + ( + "repeat".to_string(), + JsonSchema::Number { + description: Some( + "For `key` action. Number of times to press the key. Defaults to 1.".to_string(), + ), + }, + ), + // Scroll-specific + ( + "direction".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["up", "down", "left", "right"], + "For `scroll` action. Defaults to `down`.", + )), + }, + ), + ( + "distance".to_string(), + JsonSchema::String { + description: Some(string_enum_description( + &["small", "medium", "page"], + "For `scroll` action. Semantic scroll distance.", + )), + }, + ), + ( + "amount".to_string(), + JsonSchema::Number { + description: Some( + "For `scroll` action. Explicit line-count override for `distance`.".to_string(), + ), + }, + ), + // Drag-specific + ( + "from_target".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Semantic drag source to resolve. Describe the draggable surface using visible screenshot evidence." + .to_string(), + ), + }, + ), + ( + "from_location_hint".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Coarse position hint for the drag source." + .to_string(), + ), + }, + ), + ( + "from_scope".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Semantic region for the drag source." + .to_string(), + ), + }, + ), + ( + "to_target".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Semantic drag destination to resolve. Describe the drop surface using visible screenshot evidence." + .to_string(), + ), + }, + ), + ( + "to_location_hint".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Coarse position hint for the drag destination." + .to_string(), + ), + }, + ), + ( + "to_scope".to_string(), + JsonSchema::String { + description: Some( + "For `drag` action. Semantic region for the drag destination." + .to_string(), + ), + }, + ), + ( + "duration_ms".to_string(), + JsonSchema::Number { + description: Some( + "For `drag` action. Drag duration in milliseconds. Defaults to 450." + .to_string(), + ), + }, + ), + ]); + + let step_schema = JsonSchema::Object { + properties: step_properties, + required: Some(vec!["action".to_string()]), + additional_properties: Some(false.into()), + }; + + let mut properties = with_capture_selection_properties( + BTreeMap::from([( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before executing the batch." + .to_string(), + ), + }, + )]), + true, + ); + properties.insert( + "steps".to_string(), + JsonSchema::Array { + items: Box::new(step_schema), + description: Some( + "Array of independent GUI actions to execute in order. All steps share a single screenshot for grounding and are executed sequentially without re-observing between them. Only include steps that do NOT depend on the visual effects of earlier steps." + .to_string(), + ), + }, + ); + ToolSpec::Function(ResponsesApiTool { + name: "gui_batch".to_string(), + description: "Execute a batch of independent GUI actions in a single call for faster task completion. Takes one screenshot, resolves all semantic targets at once, and executes each step sequentially. Use this when consecutive actions do not depend on each other's visual effects. For dependent actions, use individual gui_* tools with gui_wait or gui_observe between them." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["steps".to_string()]), + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +pub fn create_gui_move_tool() -> ToolSpec { + let properties = BTreeMap::from([ + ( + "x".to_string(), + JsonSchema::Number { + description: Some("Absolute display X coordinate in logical points.".to_string()), + }, + ), + ( + "y".to_string(), + JsonSchema::Number { + description: Some("Absolute display Y coordinate in logical points.".to_string()), + }, + ), + ( + "app".to_string(), + JsonSchema::String { + description: Some( + "Optional application name to activate before moving the pointer.".to_string(), + ), + }, + ), + ]); + + ToolSpec::Function(ResponsesApiTool { + name: "gui_move".to_string(), + description: "Move the pointer to an absolute display coordinate in logical points." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["x".to_string(), "y".to_string()]), + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn function_parameters(tool: ToolSpec) -> BTreeMap { + match tool { + ToolSpec::Function(tool) => match tool.parameters { + JsonSchema::Object { properties, .. } => properties, + schema => panic!("expected object schema, got {schema:?}"), + }, + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn click_tool_exposes_semantic_target_properties() { + let tool = create_gui_click_tool(); + let properties = function_parameters(tool.clone()); + + assert!(properties.contains_key("target")); + assert!(properties.contains_key("location_hint")); + assert!(properties.contains_key("scope")); + assert!(properties.contains_key("grounding_mode")); + assert!(!properties.contains_key("x")); + assert!(!properties.contains_key("y")); + match tool { + ToolSpec::Function(tool) => { + let JsonSchema::Object { required, .. } = tool.parameters else { + panic!("expected object schema"); + }; + assert_eq!(required, Some(vec!["target".to_string()])); + } + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn click_tool_optionally_exposes_coordinate_properties() { + let tool = create_gui_click_tool_with_options(GuiToolSchemaOptions { + coordinate_targeting: true, + }); + let properties = function_parameters(tool.clone()); + + assert!(properties.contains_key("target")); + assert!(properties.contains_key("x")); + assert!(properties.contains_key("y")); + assert!(properties.contains_key("coordinate_space")); + match tool { + ToolSpec::Function(tool) => { + let JsonSchema::Object { required, .. } = tool.parameters else { + panic!("expected object schema"); + }; + assert_eq!(required, None); + assert!( + tool.description + .contains("experimental direct-coordinate placeholder") + ); + } + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn wait_tool_exposes_target_wait_controls() { + let tool = create_gui_wait_tool(); + let properties = function_parameters(tool.clone()); + + assert!(properties.contains_key("target")); + assert!(properties.contains_key("state")); + assert!(properties.contains_key("timeout_ms")); + assert!(properties.contains_key("interval_ms")); + assert!(properties.contains_key("scope")); + assert!(properties.contains_key("grounding_mode")); + assert!(!properties.contains_key("duration_ms")); + assert!(!properties.contains_key("return_image")); + match tool { + ToolSpec::Function(tool) => { + let JsonSchema::Object { required, .. } = tool.parameters else { + panic!("expected object schema"); + }; + assert_eq!(required, Some(vec!["target".to_string()])); + } + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn drag_tool_exposes_semantic_source_and_destination_properties() { + let tool = create_gui_drag_tool(); + let properties = function_parameters(tool.clone()); + + assert!(properties.contains_key("grounding_mode")); + assert!(properties.contains_key("from_target")); + assert!(properties.contains_key("from_location_hint")); + assert!(properties.contains_key("from_scope")); + assert!(properties.contains_key("to_target")); + assert!(properties.contains_key("to_location_hint")); + assert!(properties.contains_key("to_scope")); + assert!(!properties.contains_key("steps")); + assert!(!properties.contains_key("from_x")); + assert!(!properties.contains_key("from_y")); + assert!(!properties.contains_key("to_x")); + assert!(!properties.contains_key("to_y")); + match tool { + ToolSpec::Function(tool) => { + let JsonSchema::Object { required, .. } = tool.parameters else { + panic!("expected object schema"); + }; + assert_eq!( + required, + Some(vec!["from_target".to_string(), "to_target".to_string()]) + ); + } + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn drag_tool_optionally_exposes_coordinate_properties() { + let tool = create_gui_drag_tool_with_options(GuiToolSchemaOptions { + coordinate_targeting: true, + }); + let properties = function_parameters(tool.clone()); + + assert!(properties.contains_key("from_target")); + assert!(properties.contains_key("to_target")); + assert!(properties.contains_key("from_x")); + assert!(properties.contains_key("from_y")); + assert!(properties.contains_key("to_x")); + assert!(properties.contains_key("to_y")); + assert!(properties.contains_key("coordinate_space")); + match tool { + ToolSpec::Function(tool) => { + let JsonSchema::Object { required, .. } = tool.parameters else { + panic!("expected object schema"); + }; + assert_eq!(required, None); + assert!( + tool.description + .contains("experimental direct-coordinate placeholder") + ); + } + other => panic!("expected function tool, got {other:?}"), + } + } + + #[test] + fn observe_tool_exposes_semantic_grounding_properties() { + let tool = create_gui_observe_tool(); + let properties = function_parameters(tool); + + assert!(properties.contains_key("target")); + assert!(properties.contains_key("location_hint")); + assert!(properties.contains_key("scope")); + assert!(properties.contains_key("grounding_mode")); + let JsonSchema::String { description } = &properties["target"] else { + panic!("expected string schema"); + }; + let description = description.as_deref().unwrap_or_default(); + assert!(description.contains("visible screenshot evidence")); + assert!(description.contains("actionable or editable control")); + } + + #[test] + fn scroll_tool_exposes_understudy_aligned_scroll_semantics() { + let properties = function_parameters(create_gui_scroll_tool()); + + assert!(properties.contains_key("direction")); + assert!(properties.contains_key("distance")); + assert!(properties.contains_key("amount")); + assert!(!properties.contains_key("delta_x")); + assert!(!properties.contains_key("delta_y")); + assert!(!properties.contains_key("unit")); + assert!(!properties.contains_key("x")); + assert!(!properties.contains_key("y")); + } + + #[test] + fn type_tool_matches_understudy_native_contract_shape() { + let properties = function_parameters(create_gui_type_tool()); + + assert!(properties.contains_key("value")); + assert!(properties.contains_key("type_strategy")); + assert!(!properties.contains_key("text")); + assert!(!properties.contains_key("strategy")); + let JsonSchema::String { description } = &properties["type_strategy"] else { + panic!("expected string schema"); + }; + let description = description.as_deref().unwrap_or_default(); + assert!(!description.contains("unicode")); + } +} diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index 91931e89447a..bbef74a6d0a5 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -6,6 +6,7 @@ mod agent_tool; mod apply_patch_tool; mod code_mode; mod dynamic_tool; +mod gui_tool; mod image_detail; mod js_repl_tool; mod json_schema; @@ -49,6 +50,19 @@ pub use code_mode::create_code_mode_tool; pub use code_mode::create_wait_tool; pub use code_mode::tool_spec_to_code_mode_tool_definition; pub use dynamic_tool::parse_dynamic_tool; +pub use gui_tool::GuiToolSchemaOptions; +pub use gui_tool::create_gui_click_tool; +pub use gui_tool::create_gui_click_tool_with_options; +pub use gui_tool::create_gui_drag_tool; +pub use gui_tool::create_gui_drag_tool_with_options; +pub use gui_tool::create_gui_key_tool; +pub use gui_tool::create_gui_batch_tool; +pub use gui_tool::create_gui_move_tool; +pub use gui_tool::create_gui_observe_tool; +pub use gui_tool::create_gui_observe_tool_with_options; +pub use gui_tool::create_gui_scroll_tool; +pub use gui_tool::create_gui_type_tool; +pub use gui_tool::create_gui_wait_tool; pub use image_detail::can_request_original_image_detail; pub use image_detail::normalize_output_image_detail; pub use js_repl_tool::create_js_repl_reset_tool; diff --git a/codex-rs/tools/src/tool_config.rs b/codex-rs/tools/src/tool_config.rs index f48f1b8fccdf..08f59cfd578a 100644 --- a/codex-rs/tools/src/tool_config.rs +++ b/codex-rs/tools/src/tool_config.rs @@ -109,6 +109,11 @@ pub struct ToolsConfig { pub agent_jobs_tools: bool, pub agent_jobs_worker_tools: bool, pub agent_type_description: String, + pub gui_tools: bool, + pub gui_coordinate_targeting: bool, + pub gui_batch_enabled: bool, + pub gui_batch_grounding_strategy: String, + pub gui_batch_action_delay_ms: u64, } pub struct ToolsConfigParams<'a> { @@ -152,6 +157,7 @@ impl ToolsConfig { let include_original_image_detail = can_request_original_image_detail(features, model_info); let include_image_gen_tool = features.enabled(Feature::ImageGeneration) && supports_image_generation(model_info); + let include_gui_tools = features.enabled(Feature::GuiTools); let exec_permission_approvals_enabled = features.enabled(Feature::ExecPermissionApprovals); let request_permissions_tool_enabled = features.enabled(Feature::RequestPermissionsTool); let shell_command_backend = @@ -223,6 +229,11 @@ impl ToolsConfig { agent_jobs_tools: include_agent_jobs, agent_jobs_worker_tools, agent_type_description: String::new(), + gui_tools: include_gui_tools, + gui_coordinate_targeting: false, + gui_batch_enabled: true, + gui_batch_grounding_strategy: "parallel".to_string(), + gui_batch_action_delay_ms: 0, } } @@ -264,6 +275,26 @@ impl ToolsConfig { self } + pub fn with_gui_coordinate_targeting(mut self, gui_coordinate_targeting: bool) -> Self { + self.gui_coordinate_targeting = gui_coordinate_targeting; + self + } + + pub fn with_gui_batch_enabled(mut self, enabled: bool) -> Self { + self.gui_batch_enabled = enabled; + self + } + + pub fn with_gui_batch_grounding_strategy(mut self, strategy: String) -> Self { + self.gui_batch_grounding_strategy = strategy; + self + } + + pub fn with_gui_batch_action_delay_ms(mut self, delay_ms: u64) -> Self { + self.gui_batch_action_delay_ms = delay_ms; + self + } + pub fn for_code_mode_nested_tools(&self) -> Self { let mut nested = self.clone(); nested.code_mode_enabled = false; diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index 314d93f55f53..82714ee93438 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -1,4 +1,5 @@ use crate::CommandToolOptions; +use crate::GuiToolSchemaOptions; use crate::REQUEST_USER_INPUT_TOOL_NAME; use crate::ShellToolOptions; use crate::SpawnAgentToolOptions; @@ -23,6 +24,15 @@ use crate::create_close_agent_tool_v1; use crate::create_close_agent_tool_v2; use crate::create_code_mode_tool; use crate::create_exec_command_tool; +use crate::create_gui_click_tool_with_options; +use crate::create_gui_drag_tool_with_options; +use crate::create_gui_key_tool; +use crate::create_gui_batch_tool; +use crate::create_gui_move_tool; +use crate::create_gui_observe_tool_with_options; +use crate::create_gui_scroll_tool; +use crate::create_gui_type_tool; +use crate::create_gui_wait_tool; use crate::create_image_generation_tool; use crate::create_js_repl_reset_tool; use crate::create_js_repl_tool; @@ -340,6 +350,70 @@ pub fn build_tool_registry_plan( ); plan.register_handler("view_image", ToolHandlerKind::ViewImage); + if config.gui_tools { + let gui_options = GuiToolSchemaOptions { + coordinate_targeting: config.gui_coordinate_targeting, + }; + plan.push_spec( + create_gui_observe_tool_with_options(gui_options), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_wait_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_click_tool_with_options(gui_options), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_drag_tool_with_options(gui_options), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_scroll_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_type_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_key_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + plan.push_spec( + create_gui_move_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + if config.gui_batch_enabled { + plan.push_spec( + create_gui_batch_tool(), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + } + plan.register_handler("gui_observe", ToolHandlerKind::Gui); + plan.register_handler("gui_wait", ToolHandlerKind::Gui); + plan.register_handler("gui_click", ToolHandlerKind::Gui); + plan.register_handler("gui_drag", ToolHandlerKind::Gui); + plan.register_handler("gui_scroll", ToolHandlerKind::Gui); + plan.register_handler("gui_type", ToolHandlerKind::Gui); + plan.register_handler("gui_key", ToolHandlerKind::Gui); + plan.register_handler("gui_move", ToolHandlerKind::Gui); + if config.gui_batch_enabled { + plan.register_handler("gui_batch", ToolHandlerKind::Gui); + } + } + if config.collab_tools { if config.multi_agent_v2 { let agent_type_description = diff --git a/codex-rs/tools/src/tool_registry_plan_types.rs b/codex-rs/tools/src/tool_registry_plan_types.rs index bdcb75acd88f..169900eec75c 100644 --- a/codex-rs/tools/src/tool_registry_plan_types.rs +++ b/codex-rs/tools/src/tool_registry_plan_types.rs @@ -18,6 +18,7 @@ pub enum ToolHandlerKind { CodeModeExecute, CodeModeWait, DynamicTool, + Gui, JsRepl, JsReplReset, ListAgentsV2, diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index 12b1c669b666..38e5b8dc81d1 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -691,7 +691,10 @@ fn turn_snapshot_events( continue; }; match item { - TurnItem::UserMessage(_) | TurnItem::Plan(_) | TurnItem::AgentMessage(_) => { + TurnItem::UserMessage(_) + | TurnItem::Plan(_) + | TurnItem::AgentMessage(_) + | TurnItem::BuiltinToolCall(_) => { events.push(Event { id: String::new(), msg: EventMsg::ItemCompleted(ItemCompletedEvent { @@ -828,6 +831,37 @@ fn thread_item_to_core(item: &ThreadItem) -> Option { summary_text: summary.clone(), raw_content: content.clone(), })), + ThreadItem::BuiltinToolCall { + id, + call_id, + tool, + namespace, + arguments, + output, + success, + status, + } => Some(TurnItem::BuiltinToolCall( + codex_protocol::items::BuiltinToolCallItem { + id: id.clone(), + call_id: call_id.clone(), + tool: tool.clone(), + namespace: namespace.clone(), + arguments: arguments.clone(), + output: output.clone(), + success: *success, + status: match status { + codex_app_server_protocol::BuiltinToolCallStatus::InProgress => { + codex_protocol::items::BuiltinToolCallStatus::InProgress + } + codex_app_server_protocol::BuiltinToolCallStatus::Completed => { + codex_protocol::items::BuiltinToolCallStatus::Completed + } + codex_app_server_protocol::BuiltinToolCallStatus::Failed => { + codex_protocol::items::BuiltinToolCallStatus::Failed + } + }, + }, + )), ThreadItem::WebSearch { id, query, action } => Some(TurnItem::WebSearch(WebSearchItem { id: id.clone(), query: query.clone(), diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 02429e9d9b61..ee5dfe58445e 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -328,6 +328,7 @@ use crate::get_git_diff::get_git_diff; use crate::history_cell; #[cfg(test)] use crate::history_cell::AgentMessageCell; +use crate::history_cell::BuiltinToolCallCell; use crate::history_cell::HistoryCell; use crate::history_cell::McpToolCallCell; use crate::history_cell::PlainHistoryCell; @@ -3389,8 +3390,16 @@ impl ChatWidget { "codex to access {target}" )) } - GuardianAssessmentAction::Command { .. } => unreachable!(), - GuardianAssessmentAction::Execve { .. } => unreachable!(), + GuardianAssessmentAction::Command { command, .. } => { + history_cell::new_guardian_denied_action_request(format!( + "codex to run `{command}`" + )) + } + GuardianAssessmentAction::Execve { program, .. } => { + history_cell::new_guardian_denied_action_request(format!( + "codex to execute `{program}`" + )) + } } }; @@ -4537,6 +4546,71 @@ impl ChatWidget { self.had_work_activity = true; } + fn on_builtin_tool_call_begin( + &mut self, + call_id: String, + tool: String, + arguments: serde_json::Value, + ) { + self.flush_answer_stream_with_separator(); + self.flush_active_cell(); + self.active_cell = Some(Box::new(history_cell::new_active_builtin_tool_call( + call_id, + tool, + arguments, + self.config.animations, + ))); + self.bump_active_cell_revision(); + self.request_redraw(); + self.had_work_activity = true; + } + + fn try_complete_active_builtin_tool_call( + &mut self, + call_id: &str, + success: Option, + output: Option<&serde_json::Value>, + ) -> bool { + if let Some(cell) = self + .active_cell + .as_mut() + .and_then(|cell| cell.as_any_mut().downcast_mut::()) + && cell.call_id() == call_id + { + cell.complete(success, output); + self.bump_active_cell_revision(); + return true; + } + + false + } + + fn on_builtin_tool_call_end( + &mut self, + call_id: String, + tool: String, + arguments: serde_json::Value, + success: Option, + output: Option, + ) { + self.flush_answer_stream_with_separator(); + + if !self.try_complete_active_builtin_tool_call(&call_id, success, output.as_ref()) { + self.flush_active_cell(); + let mut cell = history_cell::new_active_builtin_tool_call( + call_id, + tool, + arguments, + self.config.animations, + ); + cell.complete(success, output.as_ref()); + self.active_cell = Some(Box::new(cell)); + } + + self.flush_active_cell(); + self.had_work_activity = true; + } + pub(crate) fn new_with_app_event(common: ChatWidgetInit) -> Self { Self::new_with_op_target(common, CodexOpTarget::AppEvent) } @@ -6198,6 +6272,23 @@ impl ChatWidget { saved_path, }); } + ThreadItem::BuiltinToolCall { + call_id, + tool, + arguments, + output, + success, + status, + .. + } => match status { + codex_app_server_protocol::BuiltinToolCallStatus::InProgress => { + self.on_builtin_tool_call_begin(call_id, tool, arguments); + } + codex_app_server_protocol::BuiltinToolCallStatus::Completed + | codex_app_server_protocol::BuiltinToolCallStatus::Failed => { + self.on_builtin_tool_call_end(call_id, tool, arguments, success, output); + } + }, ThreadItem::EnteredReviewMode { review, .. } => { if from_replay { self.enter_review_mode_with_hint(review, /*from_replay*/ true); @@ -6328,6 +6419,7 @@ impl ChatWidget { ServerNotification::TurnCompleted(notification) => { self.handle_turn_completed_notification(notification, replay_kind); } + ServerNotification::RawResponseItemCompleted(_) => {} ServerNotification::ItemStarted(notification) => { self.handle_item_started_notification(notification, replay_kind.is_some()); } @@ -6511,7 +6603,6 @@ impl ChatWidget { | ServerNotification::ThreadStatusChanged(_) | ServerNotification::ThreadArchived(_) | ServerNotification::ThreadUnarchived(_) - | ServerNotification::RawResponseItemCompleted(_) | ServerNotification::CommandExecOutputDelta(_) | ServerNotification::McpToolCallProgress(_) | ServerNotification::McpServerOauthLoginCompleted(_) @@ -6665,6 +6756,14 @@ impl ChatWidget { ThreadItem::ImageGeneration { id, .. } => { self.on_image_generation_begin(ImageGenerationBeginEvent { call_id: id }); } + ThreadItem::BuiltinToolCall { + call_id, + tool, + arguments, + .. + } => { + self.on_builtin_tool_call_begin(call_id, tool, arguments); + } ThreadItem::CollabAgentToolCall { id, tool, @@ -7225,6 +7324,8 @@ impl ChatWidget { exec.mark_failed(); } else if let Some(tool) = cell.as_any_mut().downcast_mut::() { tool.mark_failed(); + } else if let Some(tool) = cell.as_any_mut().downcast_mut::() { + tool.mark_failed(); } self.add_boxed_history(cell); } diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 2cabed2cc80d..74bfcd4ecf92 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -234,6 +234,97 @@ async fn live_app_server_command_execution_strips_shell_wrapper() { ); } +#[tokio::test] +async fn live_app_server_gui_builtin_tool_call_renders_history_from_thread_items() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.handle_server_notification( + ServerNotification::ItemStarted(codex_app_server_protocol::ItemStartedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item: ThreadItem::BuiltinToolCall { + id: "gui-call-1".to_string(), + call_id: "gui-call-1".to_string(), + tool: "gui_click".to_string(), + namespace: Some("builtin".to_string()), + arguments: serde_json::json!({"app":"Chrome","target":"Search"}), + output: None, + success: None, + status: codex_app_server_protocol::BuiltinToolCallStatus::InProgress, + }, + }), + /*replay_kind*/ None, + ); + + let active = lines_to_single_string( + &chat + .active_cell + .as_ref() + .expect("gui builtin tool call should create an active cell") + .display_lines(120), + ); + assert!( + active.contains("Calling gui_click"), + "expected active builtin tool call cell, got {active:?}" + ); + assert!(drain_insert_history(&mut rx).is_empty()); + + chat.handle_server_notification( + ServerNotification::ItemCompleted(codex_app_server_protocol::ItemCompletedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item: ThreadItem::BuiltinToolCall { + id: "gui-call-1".to_string(), + call_id: "gui-call-1".to_string(), + tool: "gui_click".to_string(), + namespace: Some("builtin".to_string()), + arguments: serde_json::json!({"app":"Chrome","target":"Search"}), + output: Some(serde_json::json!("{\"status\":\"clicked\"}")), + success: Some(true), + status: codex_app_server_protocol::BuiltinToolCallStatus::Completed, + }, + }), + /*replay_kind*/ None, + ); + + let cells = drain_insert_history(&mut rx); + assert_eq!(cells.len(), 1, "expected one completed builtin tool cell"); + let rendered = lines_to_single_string(&cells[0]); + assert!( + rendered.contains("Called gui_click"), + "expected completed builtin tool call cell, got {rendered:?}" + ); + assert!( + rendered.contains("\"status\": \"clicked\""), + "expected builtin tool output in rendered history, got {rendered:?}" + ); +} + +#[tokio::test] +async fn live_app_server_raw_response_builtin_tool_call_is_ignored() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.handle_server_notification( + ServerNotification::RawResponseItemCompleted( + codex_app_server_protocol::RawResponseItemCompletedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item: codex_protocol::models::ResponseItem::FunctionCall { + id: None, + name: "gui_click".to_string(), + namespace: Some("builtin".to_string()), + arguments: r#"{"app":"Chrome","target":"Search"}"#.to_string(), + call_id: "builtin-call-1".to_string(), + }, + }, + ), + /*replay_kind*/ None, + ); + + assert!(chat.active_cell.is_none()); + assert!(drain_insert_history(&mut rx).is_empty()); +} + #[test] fn app_server_patch_changes_to_core_preserves_diffs() { let changes = app_server_patch_changes_to_core(vec![FileUpdateChange { diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index cf65f918e52b..6195c4c80e1a 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1595,6 +1595,139 @@ pub(crate) fn new_active_mcp_tool_call( McpToolCallCell::new(call_id, invocation, animations_enabled) } +#[derive(Debug)] +pub(crate) struct BuiltinToolCallCell { + call_id: String, + tool: String, + arguments: serde_json::Value, + start_time: Instant, + success: Option, + output: Option, + animations_enabled: bool, +} + +impl BuiltinToolCallCell { + pub(crate) fn new( + call_id: String, + tool: String, + arguments: serde_json::Value, + animations_enabled: bool, + ) -> Self { + Self { + call_id, + tool, + arguments, + start_time: Instant::now(), + success: None, + output: None, + animations_enabled, + } + } + + pub(crate) fn call_id(&self) -> &str { + &self.call_id + } + + pub(crate) fn complete(&mut self, success: Option, output: Option<&serde_json::Value>) { + self.success = Some(!matches!(success, Some(false))); + self.output = output.and_then(codex_protocol::items::builtin_tool_output_summary); + } + + pub(crate) fn mark_failed(&mut self) { + self.success = Some(false); + if self.output.is_none() { + self.output = Some("interrupted".to_string()); + } + } +} + +impl HistoryCell for BuiltinToolCallCell { + fn display_lines(&self, width: u16) -> Vec> { + let mut lines: Vec> = Vec::new(); + let bullet = match self.success { + Some(true) => "•".green().bold(), + Some(false) => "•".red().bold(), + None => spinner(Some(self.start_time), self.animations_enabled), + }; + let header_text = if self.success.is_some() { + "Called" + } else { + "Calling" + }; + + let invocation_line = + line_to_static(&format_builtin_tool_invocation(&self.tool, &self.arguments)); + let mut compact_spans = vec![bullet.clone(), " ".into(), header_text.bold(), " ".into()]; + let mut compact_header = Line::from(compact_spans.clone()); + let reserved = compact_header.width(); + let inline_invocation = + invocation_line.width() <= (width as usize).saturating_sub(reserved); + + if inline_invocation { + compact_header.extend(invocation_line.spans.clone()); + lines.push(compact_header); + } else { + compact_spans.pop(); + lines.push(Line::from(compact_spans)); + + let opts = RtOptions::new((width as usize).saturating_sub(4)) + .initial_indent("".into()) + .subsequent_indent(" ".into()); + let wrapped = adaptive_wrap_line(&invocation_line, opts); + let body_lines: Vec> = wrapped.iter().map(line_to_static).collect(); + lines.extend(prefix_lines(body_lines, " └ ".dim(), " ".into())); + } + + if let Some(output) = &self.output { + let detail_wrap_width = (width as usize).saturating_sub(4).max(1); + let detail_text = + format_and_truncate_tool_result(output, TOOL_CALL_MAX_LINES, detail_wrap_width); + let detail_lines = detail_text + .split('\n') + .flat_map(|segment| { + let line = Line::from(segment.to_string().dim()); + adaptive_wrap_line( + &line, + RtOptions::new(detail_wrap_width) + .initial_indent("".into()) + .subsequent_indent(" ".into()), + ) + .iter() + .map(line_to_static) + .collect::>() + }) + .collect::>(); + + if !detail_lines.is_empty() { + let initial_prefix: Span<'static> = if inline_invocation { + " └ ".dim() + } else { + " ".into() + }; + lines.extend(prefix_lines(detail_lines, initial_prefix, " ".into())); + } + } + + lines + } + + fn transcript_animation_tick(&self) -> Option { + if !self.animations_enabled || self.success.is_some() { + return None; + } + Some((self.start_time.elapsed().as_millis() / 50) as u64) + } +} + +pub(crate) fn new_active_builtin_tool_call( + call_id: String, + tool: String, + arguments: serde_json::Value, + animations_enabled: bool, +) -> BuiltinToolCallCell { + BuiltinToolCallCell::new(call_id, tool, arguments, animations_enabled) +} + fn web_search_header(completed: bool) -> &'static str { if completed { "Searched" @@ -2762,6 +2895,17 @@ fn format_mcp_invocation<'a>(invocation: McpInvocation) -> Line<'a> { invocation_spans.into() } +fn format_builtin_tool_invocation<'a>(tool: &str, arguments: &serde_json::Value) -> Line<'a> { + let args_str = serde_json::to_string(arguments).unwrap_or_else(|_| arguments.to_string()); + vec![ + tool.to_string().cyan(), + "(".into(), + args_str.dim(), + ")".into(), + ] + .into() +} + #[cfg(test)] mod tests { use super::*; diff --git a/docs/gui-demo.gif b/docs/gui-demo.gif new file mode 100644 index 000000000000..bc4ad45c9f91 Binary files /dev/null and b/docs/gui-demo.gif differ