Skip to content

feat(gui): native GUI tools with builtin tool call observability #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
s-JoL wants to merge 17 commits into
base: main
Choose a base branch
from
Open

feat(gui): native GUI tools with builtin tool call observability #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
3460ac9
feat(protocol): add BuiltinToolCallItem and BuiltinToolCallStatus types
s-JoL Apr 2, 2026
86e3e61
feat(gui): add native GUI tool schema definitions
s-JoL Apr 2, 2026
79fbb6c
feat(gui): add native platform runtime and Swift helpers
s-JoL Apr 2, 2026
334584c
feat(gui): add semantic grounding engine
s-JoL Apr 2, 2026
0b8b7ab
feat(gui): add GUI tool handler with observe-act-verify loop
s-JoL Apr 2, 2026
3b6478f
feat(gui): add system prompt instructions and config options
s-JoL Apr 2, 2026
55346d2
feat(gui): promote GUI tool calls to first-class BuiltinToolCall turn…
s-JoL Apr 2, 2026
6f87ce5
feat(gui): add TUI and CLI rendering for builtin tool calls
s-JoL Apr 2, 2026
c9287d0
test(gui): add GUI tool tests, grounding benchmark, and config tests
s-JoL Apr 2, 2026
5137794
fix(gui): do not re-activate app during post-action evidence and wait…
s-JoL Apr 2, 2026
088329d
fix(gui): address code review findings for async safety, caching, and…
s-JoL Apr 2, 2026
4c63914
fix(gui): fix screenshot redaction crash, TOCTOU race, and edge groun…
s-JoL Apr 2, 2026
d8157b5
fix(gui): improve grounding coordinate accuracy and async robustness
s-JoL Apr 2, 2026
5b5a8a5
fix(gui): use modern activate API on macOS 14+ and guard mouse release
s-JoL Apr 3, 2026
acd1895
feat(gui): hide other apps during actions, fix mouse cleanup, and add…
s-JoL Apr 3, 2026
619a0e1
feat(gui): add gui_batch tool with parallel/unified grounding and dra…
s-JoL Apr 3, 2026
82f5a63
feat(gui): persistent Swift helper, JPEG screenshots, Escape safety, …
s-JoL Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions codex-rs/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,13 @@ strip = "symbols"
# See https://github.com/openai/codex/issues/1411 for details.
codegen-units = 1

# Fast local release builds: no LTO + parallel codegen.
# Use with: cargo build --profile local-release
[profile.local-release]
inherits = "release"
lto = false
codegen-units = 8

[profile.ci-test]
debug = 1 # Reduce debug symbol size
inherits = "test"
Expand Down
55 changes: 55 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ServerNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,14 @@
}
]
},
"BuiltinToolCallStatus": {
"enum": [
"inProgress",
"completed",
"failed"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
Expand Down Expand Up @@ -2871,6 +2879,53 @@
"title": "McpToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
"callId": {
"type": "string"
},
"id": {
"type": "string"
},
"namespace": {
"type": [
"string",
"null"
]
},
"output": true,
"status": {
"$ref": "#/definitions/BuiltinToolCallStatus"
},
"success": {
"type": [
"boolean",
"null"
]
},
"tool": {
"type": "string"
},
"type": {
"enum": [
"builtinToolCall"
],
"title": "BuiltinToolCallThreadItemType",
"type": "string"
}
},
"required": [
"arguments",
"callId",
"id",
"status",
"tool",
"type"
],
"title": "BuiltinToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
Expand Down
55 changes: 55 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5492,6 +5492,14 @@
}
]
},
"BuiltinToolCallStatus": {
"enum": [
"inProgress",
"completed",
"failed"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
Expand Down Expand Up @@ -12837,6 +12845,53 @@
"title": "McpToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
"callId": {
"type": "string"
},
"id": {
"type": "string"
},
"namespace": {
"type": [
"string",
"null"
]
},
"output": true,
"status": {
"$ref": "#/definitions/v2/BuiltinToolCallStatus"
},
"success": {
"type": [
"boolean",
"null"
]
},
"tool": {
"type": "string"
},
"type": {
"enum": [
"builtinToolCall"
],
"title": "BuiltinToolCallThreadItemType",
"type": "string"
}
},
"required": [
"arguments",
"callId",
"id",
"status",
"tool",
"type"
],
"title": "BuiltinToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
Expand Down
55 changes: 55 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,14 @@
}
]
},
"BuiltinToolCallStatus": {
"enum": [
"inProgress",
"completed",
"failed"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
Expand Down Expand Up @@ -10692,6 +10700,53 @@
"title": "McpToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
"callId": {
"type": "string"
},
"id": {
"type": "string"
},
"namespace": {
"type": [
"string",
"null"
]
},
"output": true,
"status": {
"$ref": "#/definitions/BuiltinToolCallStatus"
},
"success": {
"type": [
"boolean",
"null"
]
},
"tool": {
"type": "string"
},
"type": {
"enum": [
"builtinToolCall"
],
"title": "BuiltinToolCallThreadItemType",
"type": "string"
}
},
"required": [
"arguments",
"callId",
"id",
"status",
"tool",
"type"
],
"title": "BuiltinToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
Expand Down
55 changes: 55 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BuiltinToolCallStatus": {
"enum": [
"inProgress",
"completed",
"failed"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
Expand Down Expand Up @@ -816,6 +824,53 @@
"title": "McpToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
"callId": {
"type": "string"
},
"id": {
"type": "string"
},
"namespace": {
"type": [
"string",
"null"
]
},
"output": true,
"status": {
"$ref": "#/definitions/BuiltinToolCallStatus"
},
"success": {
"type": [
"boolean",
"null"
]
},
"tool": {
"type": "string"
},
"type": {
"enum": [
"builtinToolCall"
],
"title": "BuiltinToolCallThreadItemType",
"type": "string"
}
},
"required": [
"arguments",
"callId",
"id",
"status",
"tool",
"type"
],
"title": "BuiltinToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
Expand Down
55 changes: 55 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BuiltinToolCallStatus": {
"enum": [
"inProgress",
"completed",
"failed"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
Expand Down Expand Up @@ -816,6 +824,53 @@
"title": "McpToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
"callId": {
"type": "string"
},
"id": {
"type": "string"
},
"namespace": {
"type": [
"string",
"null"
]
},
"output": true,
"status": {
"$ref": "#/definitions/BuiltinToolCallStatus"
},
"success": {
"type": [
"boolean",
"null"
]
},
"tool": {
"type": "string"
},
"type": {
"enum": [
"builtinToolCall"
],
"title": "BuiltinToolCallThreadItemType",
"type": "string"
}
},
"required": [
"arguments",
"callId",
"id",
"status",
"tool",
"type"
],
"title": "BuiltinToolCallThreadItem",
"type": "object"
},
{
"properties": {
"arguments": true,
Expand Down
Loading