diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0c1c728c..443fcf03 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,9 @@ updates: interval: 'daily' time: '00:00' labels: [] + - package-ecosystem: 'npm' + directory: '/site' + schedule: + interval: 'daily' + time: '00:00' + labels: [] diff --git a/.github/rulesets/any.json b/.github/rulesets/any.json deleted file mode 100644 index 8a00e588..00000000 --- a/.github/rulesets/any.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "id": 12430557, - "name": "any", - "target": "branch", - "source_type": "Repository", - "source": "hack-ink/ELF", - "enforcement": "active", - "conditions": { - "ref_name": { - "exclude": [], - "include": [ - "~ALL" - ] - } - }, - "rules": [ - { - "type": "required_signatures" - }, - { - "type": "code_scanning", - "parameters": { - "code_scanning_tools": [ - { - "tool": "CodeQL", - "security_alerts_threshold": "high_or_higher", - "alerts_threshold": "errors" - } - ] - } - }, - { - "type": "code_quality", - "parameters": { - "severity": "errors" - } - }, - { - "type": "copilot_code_review", - "parameters": { - "review_on_push": true, - "review_draft_pull_requests": true - } - }, - { - "type": "copilot_code_review_analysis_tools", - "parameters": { - "tools": [ - { - "name": "CodeQL" - }, - { - "name": "ESLint" - }, - { - "name": "PMD" - } - ] - } - } - ], - "bypass_actors": [ - { - "actor_id": null, - "actor_type": "OrganizationAdmin", - "bypass_mode": "always" - }, - { - "actor_id": null, - "actor_type": "DeployKey", - "bypass_mode": "always" - }, - { - "actor_id": 5, - "actor_type": "RepositoryRole", - "bypass_mode": "always" - } - ] -} \ No newline at end of file diff --git a/.github/rulesets/default.json b/.github/rulesets/default.json deleted file mode 100644 index 1f715531..00000000 --- a/.github/rulesets/default.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "id": 9349786, - "name": "default", - "target": "branch", - "source_type": "Repository", - "source": "hack-ink/vibe-mono", - "enforcement": "active", - "conditions": { - "ref_name": { - "exclude": [], - "include": [ - "~DEFAULT_BRANCH" - ] - } - }, - "rules": [ - { - "type": "deletion" - }, - { - "type": "non_fast_forward" - }, - { - "type": "creation" - }, - { - "type": "required_linear_history" - }, - { - "type": "merge_queue", - "parameters": { - "merge_method": "SQUASH", - "max_entries_to_build": 5, - "min_entries_to_merge": 1, - "max_entries_to_merge": 5, - "min_entries_to_merge_wait_minutes": 5, - "grouping_strategy": "ALLGREEN", - "check_response_timeout_minutes": 60 - } - }, - { - "type": "required_signatures" - }, - { - "type": "pull_request", - "parameters": { - "required_approving_review_count": 0, - "dismiss_stale_reviews_on_push": true, - "required_reviewers": [], - "require_code_owner_review": false, - "require_last_push_approval": false, - "required_review_thread_resolution": true, - "allowed_merge_methods": [ - "squash" - ] - } - }, - { - "type": "code_scanning", - "parameters": { - "code_scanning_tools": [ - { - "tool": "CodeQL", - "security_alerts_threshold": "high_or_higher", - "alerts_threshold": "errors" - } - ] - } - }, - { - "type": "copilot_code_review", - "parameters": { - "review_on_push": true, - "review_draft_pull_requests": false - } - }, - { - "type": "code_quality", - "parameters": { - "severity": "errors" - } - } - ], - "bypass_actors": [ - { - "actor_id": null, - "actor_type": "OrganizationAdmin", - "bypass_mode": "always" - }, - { - "actor_id": null, - "actor_type": "DeployKey", - "bypass_mode": "always" - }, - { - "actor_id": 5, - "actor_type": "RepositoryRole", - "bypass_mode": "always" - } - ] -} \ No newline at end of file diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index d6115502..cb8e27ed 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -21,17 +21,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Node - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 cache: npm cache-dependency-path: site/package-lock.json - name: Configure GitHub Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 - name: Install site dependencies working-directory: site @@ -42,7 +42,7 @@ jobs: run: npm run build - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site/dist @@ -56,4 +56,4 @@ jobs: steps: - name: Deploy artifact id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.github/workflows/language.yml b/.github/workflows/language.yml index 2de57715..d14c58f8 100644 --- a/.github/workflows/language.yml +++ b/.github/workflows/language.yml @@ -20,25 +20,21 @@ on: - '.gitignore' - 'docs/**' merge_group: - paths-ignore: - - '**/*.md' - - '.gitignore' - - 'docs/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - rust: - name: Rust checks + rust-check: + name: Rust check runs-on: ubuntu-latest steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: cache: true rustflags: '' @@ -48,61 +44,80 @@ jobs: run: rustup toolchain install nightly --component rustfmt - name: Install cargo-make - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@3fa6878dc4ae603f73960271565a082bf196ab96 # v2.77.2 with: tool: cargo-make - - name: Install vibe-style (latest release) - run: | - set -euo pipefail - VERSION="$(curl -fsSL https://api.github.com/repos/hack-ink/vibe-style/releases/latest | grep -oE '"tag_name": "v[^"]+"' | cut -d'"' -f4)" - TARGET="x86_64-unknown-linux-gnu" - ASSET="vibe-style-${TARGET}-${VERSION}.tgz" - - curl -fsSLO "https://github.com/hack-ink/vibe-style/releases/download/${VERSION}/${ASSET}" - tar -xzf "${ASSET}" - - mkdir -p "$HOME/.cargo/bin" - install -m 0755 "vibe-style-${TARGET}-${VERSION}/vstyle" "$HOME/.cargo/bin/vstyle" - install -m 0755 "vibe-style-${TARGET}-${VERSION}/cargo-vstyle" "$HOME/.cargo/bin/cargo-vstyle" - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Install nextest - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@3fa6878dc4ae603f73960271565a082bf196ab96 # v2.77.2 with: tool: nextest - - name: Run lint - run: cargo make lint - - - name: Run Rust format checks + - name: Run Rust format check run: cargo make fmt-rust-check - - name: Run tests + - name: Run Rust style check + uses: hack-ink/vibe-style@bfb4d2d2f5e4b5e5ce8de4ed1d708b3a2f0e61fe # v0.2.1 + with: + language: rust + workspace: true + args: --all-features + version: v0.2.1 + + - name: Run Rust clippy + run: cargo make check-rust + + - name: Run Rust tests run: cargo make test-rust - toml: - name: TOML checks + site-check: + name: Site check + runs-on: ubuntu-latest + steps: + - name: Fetch latest code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: site/package-lock.json + + - name: Install cargo-make + uses: taiki-e/install-action@3fa6878dc4ae603f73960271565a082bf196ab96 # v2.77.2 + with: + tool: cargo-make + + - name: Install site dependencies + working-directory: site + run: npm ci + + - name: Run site and content checks + run: cargo make check-site + + toml-check: + name: TOML check runs-on: ubuntu-latest steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: cache: true rustflags: '' - name: Install cargo-make - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@3fa6878dc4ae603f73960271565a082bf196ab96 # v2.77.2 with: tool: cargo-make - name: Install taplo - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@3fa6878dc4ae603f73960271565a082bf196ab96 # v2.77.2 with: tool: taplo - - name: Run TOML format checks + - name: Run TOML format check run: cargo make fmt-toml-check diff --git a/.github/workflows/refresh-github-signals.yml b/.github/workflows/refresh-github-signals.yml index ae2fc202..ed73416a 100644 --- a/.github/workflows/refresh-github-signals.yml +++ b/.github/workflows/refresh-github-signals.yml @@ -21,13 +21,13 @@ jobs: GH_API_TOKEN: ${{ secrets.GITHUB_PAT_Y }} steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: main fetch-depth: 0 - name: Set up Node - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 cache: npm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d4cbe5a..2edab7a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ permissions: contents: write env: + BINARY_NAME: decodex CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse CARGO_TERM_COLOR: always @@ -28,14 +29,13 @@ jobs: [ { name: aarch64-apple-darwin, os: macos-latest }, { name: x86_64-unknown-linux-gnu, os: ubuntu-latest }, - { name: x86_64-pc-windows-msvc, os: windows-latest }, ] steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: cache: true components: rustfmt, clippy @@ -44,63 +44,42 @@ jobs: run: rustup target add ${{ matrix.target.name }} - name: Build - run: cargo build --profile final-release --locked --target ${{ matrix.target.name }} + run: cargo build --profile final-release --locked --package "${BINARY_NAME}" --target ${{ matrix.target.name }} - name: Pack (macOS) if: matrix.target.os == 'macos-latest' run: | - mv target/${{ matrix.target.name }}/final-release/name_placeholder . - zip name_placeholder-${{ matrix.target.name }}.zip name_placeholder - - - name: Pack (Windows) - if: matrix.target.os == 'windows-latest' - run: | - mv target/${{ matrix.target.name }}/final-release/name_placeholder.exe . - Compress-Archive -Path name_placeholder.exe -DestinationPath name_placeholder-${{ matrix.target.name }}.zip + cp "target/${{ matrix.target.name }}/final-release/${BINARY_NAME}" . + zip "${BINARY_NAME}-${{ matrix.target.name }}.zip" "${BINARY_NAME}" - name: Pack (Linux) if: matrix.target.os == 'ubuntu-latest' run: | - mv target/${{ matrix.target.name }}/final-release/name_placeholder . - tar -czvf name_placeholder-${{ matrix.target.name }}.tar.gz name_placeholder + cp "target/${{ matrix.target.name }}/final-release/${BINARY_NAME}" . + tar -czvf "${BINARY_NAME}-${{ matrix.target.name }}.tar.gz" "${BINARY_NAME}" - name: Upload artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: name_placeholder-${{ matrix.target.name }} - path: name_placeholder-${{ matrix.target.name }}.* + name: ${{ env.BINARY_NAME }}-${{ matrix.target.name }} + path: ${{ env.BINARY_NAME }}-${{ matrix.target.name }}.* retention-days: 1 - # release: - # name: Release - # runs-on: ubuntu-latest - # steps: - # - name: Publish - # uses: softprops/action-gh-release@v2 - # with: - # discussion_category_name: Announcements - # generate_release_notes: true - release: name: Release runs-on: ubuntu-latest needs: [build] steps: - name: Download artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - - name: Hash + - name: Collect artifacts run: | mkdir -p artifacts - mv name_placeholder-*/* artifacts/ - cd artifacts - sha256sum * | tee ../SHA256 - md5sum * | tee ../MD5 - mv ../SHA256 . - mv ../MD5 . + mv "${BINARY_NAME}-"*/* artifacts/ - name: Publish - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0 with: discussion_category_name: Announcements generate_release_notes: true @@ -111,10 +90,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Fetch latest code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: cache: true components: rustfmt, clippy @@ -123,4 +102,4 @@ jobs: run: cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} - name: Publish - run: cargo publish --locked + run: cargo publish --locked --package "${BINARY_NAME}" diff --git a/Cargo.lock b/Cargo.lock index 8297c4cc..7d9d8427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,7 +62,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -73,7 +73,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -82,6 +82,34 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -97,6 +125,49 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "camino" version = "1.2.2" @@ -108,9 +179,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a0c0e6148f11f01f32650a2ea02d532b2ad4e81d8bd41e6e565b5adc5e6082" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" dependencies = [ "serde", "serde_core", @@ -130,17 +201,35 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -160,9 +249,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -176,6 +265,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "color-eyre" version = "0.6.5" @@ -209,6 +307,47 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -224,6 +363,15 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "darling" version = "0.20.11" @@ -259,6 +407,29 @@ dependencies = [ "syn", ] +[[package]] +name = "decodex" +version = "0.1.0" +dependencies = [ + "base64", + "clap", + "color-eyre", + "globset", + "libc", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha1", + "tempfile", + "time", + "toml", + "tracing", + "tracing-appender", + "tracing-subscriber", + "vergen-gitcl", +] + [[package]] name = "deranged" version = "0.5.8" @@ -300,598 +471,2295 @@ dependencies = [ ] [[package]] -name = "directories" -version = "6.0.0" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "dirs-sys", + "block-buffer", + "const-oid", + "crypto-common", ] [[package]] -name = "dirs-sys" -version = "0.5.0" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "eyre" -version = "0.6.12" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" -dependencies = [ - "indenter", - "once_cell", -] +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "fnv" -version = "1.0.7" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "getrandom" -version = "0.2.17" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "cfg-if", "libc", - "wasi", + "windows-sys 0.61.2", ] [[package]] -name = "gimli" -version = "0.32.3" +name = "eyre" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] [[package]] -name = "heck" -version = "0.5.0" +name = "fallible-iterator" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] -name = "ident_case" -version = "1.0.1" +name = "fallible-streaming-iterator" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] -name = "indenter" -version = "0.3.4" +name = "fastrand" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "itoa" -version = "1.0.17" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "lazy_static" -version = "1.5.0" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "libc" -version = "0.2.183" +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "libredox" -version = "0.1.14" +name = "form_urlencoded" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "libc", + "percent-encoding", ] [[package]] -name = "log" -version = "0.4.29" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] -name = "matchers" -version = "0.2.0" +name = "futures-channel" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ - "regex-automata", + "futures-core", + "futures-sink", ] [[package]] -name = "memchr" -version = "2.8.0" +name = "futures-core" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] -name = "miniz_oxide" -version = "0.8.9" +name = "futures-io" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] -name = "name_placeholder" -version = "0.1.0" -dependencies = [ - "clap", - "color-eyre", - "directories", - "tracing", - "tracing-appender", - "tracing-subscriber", - "vergen-gitcl", -] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] -name = "nu-ansi-term" -version = "0.50.3" +name = "futures-task" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys", -] +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] -name = "num-conv" -version = "0.2.0" +name = "futures-util" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] [[package]] -name = "num_threads" -version = "0.1.7" +name = "getrandom" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ + "cfg-if", + "js-sys", "libc", + "wasi", + "wasm-bindgen", ] [[package]] -name = "object" -version = "0.37.3" +name = "getrandom" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "memchr", + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "getrandom" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "gimli" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] -name = "option-ext" -version = "0.2.0" +name = "globset" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] [[package]] -name = "owo-colors" -version = "4.3.0" +name = "hashbrown" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] [[package]] -name = "pin-project-lite" -version = "0.2.17" +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] [[package]] -name = "powerfmt" -version = "0.2.0" +name = "hashbrown" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] -name = "proc-macro2" -version = "1.0.106" +name = "hashlink" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" dependencies = [ - "unicode-ident", + "hashbrown 0.16.1", ] [[package]] -name = "quote" -version = "1.0.45" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "redox_users" -version = "0.5.2" +name = "http" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ - "getrandom", - "libredox", - "thiserror", + "bytes", + "itoa", ] [[package]] -name = "regex" -version = "1.12.3" +name = "http-body" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "bytes", + "http", ] [[package]] -name = "regex-automata" -version = "0.4.14" +name = "http-body-util" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", ] [[package]] -name = "regex-syntax" -version = "0.8.10" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "rustc-demangle" -version = "0.1.27" +name = "hybrid-array" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +dependencies = [ + "typenum", +] [[package]] -name = "rustversion" -version = "1.0.22" +name = "hyper" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] [[package]] -name = "semver" -version = "1.0.27" +name = "hyper-rustls" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "serde", - "serde_core", + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", ] [[package]] -name = "serde" -version = "1.0.228" +name = "hyper-util" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "serde_core", - "serde_derive", + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", ] [[package]] -name = "serde_core" -version = "1.0.228" +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "serde_derive", + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "serde_derive" -version = "1.0.228" +name = "icu_locale_core" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ - "proc-macro2", - "quote", - "syn", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "serde_json" -version = "1.0.149" +name = "icu_normalizer" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", ] [[package]] -name = "sharded-slab" -version = "0.1.7" +name = "icu_normalizer_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] -name = "smallvec" -version = "1.15.1" +name = "icu_properties" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] [[package]] -name = "strsim" -version = "0.11.1" +name = "icu_properties_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] -name = "syn" -version = "2.0.117" +name = "icu_provider" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", ] [[package]] -name = "thiserror" -version = "2.0.18" +name = "id-arena" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl", -] +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] -name = "thiserror-impl" -version = "2.0.18" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "proc-macro2", - "quote", - "syn", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "thread_local" -version = "1.1.9" +name = "idna_adapter" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ - "cfg-if", + "icu_normalizer", + "icu_properties", ] [[package]] -name = "time" -version = "0.3.47" +name = "indenter" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", + "equivalent", + "hashbrown 0.17.1", + "serde", "serde_core", - "time-core", - "time-macros", ] [[package]] -name = "time-core" -version = "0.1.8" +name = "ipnet" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] -name = "time-macros" -version = "0.2.27" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] -name = "tracing" -version = "0.1.44" +name = "itoa" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "tracing-appender" -version = "0.2.4" +name = "jni" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "crossbeam-channel", + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", "thiserror", - "time", - "tracing-subscriber", + "walkdir", + "windows-link", ] [[package]] -name = "tracing-attributes" -version = "0.1.31" +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", "quote", + "rustc_version", + "simd_cesu8", "syn", ] [[package]] -name = "tracing-core" -version = "0.1.36" +name = "jni-sys" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" dependencies = [ - "once_cell", - "valuable", + "jni-sys-macros", ] [[package]] -name = "tracing-error" -version = "0.2.1" +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ - "tracing", - "tracing-subscriber", + "quote", + "syn", ] [[package]] -name = "tracing-log" -version = "0.2.0" +name = "jobserver" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "log", - "once_cell", - "tracing-core", + "getrandom 0.3.4", + "libc", ] [[package]] -name = "tracing-subscriber" -version = "0.3.23" +name = "js-sys" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ - "matchers", - "nu-ansi-term", + "cfg-if", + "futures-util", "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", + "wasm-bindgen", ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "utf8parse" -version = "0.2.2" +name = "leb128fmt" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] -name = "valuable" -version = "0.1.1" +name = "libc" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "vergen" -version = "9.1.0" +name = "libsqlite3-sys" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ - "anyhow", - "cargo_metadata", - "derive_builder", - "regex", - "rustversion", - "vergen-lib", + "cc", + "pkg-config", + "vcpkg", ] [[package]] -name = "vergen-gitcl" -version = "9.1.0" +name = "linux-raw-sys" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ff3b5300a085d6bcd8fc96a507f706a28ae3814693236c9b409db71a1d15b9" -dependencies = [ - "anyhow", - "derive_builder", - "rustversion", - "time", - "vergen", - "vergen-lib", -] +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] -name = "vergen-lib" -version = "9.1.0" +name = "litemap" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" -dependencies = [ - "anyhow", - "derive_builder", - "rustversion", -] +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "log" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "windows-link" -version = "0.2.1" +name = "lru-slab" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "windows-sys" -version = "0.61.2" +name = "matchers" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "windows-link", + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2c760607300407ddeaee518acf28c795661b7108c75421303dbefb237d3a36" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vergen" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +dependencies = [ + "anyhow", + "cargo_metadata", + "derive_builder", + "regex", + "rustversion", + "vergen-lib", +] + +[[package]] +name = "vergen-gitcl" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ff3b5300a085d6bcd8fc96a507f706a28ae3814693236c9b409db71a1d15b9" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f3a13934..4396f458 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,34 +1,35 @@ -[package] +[workspace] +members = ["apps/*"] +resolver = "3" + +[workspace.package] authors = ["Xavier Lau "] -build = "build.rs" -categories = [] -description = "description_placeholder" +description = "Decodex runtime, static signal site, and repository-native automation tooling." edition = "2024" -homepage = "https://hack.ink/name_placeholder" -keywords = [] +homepage = "https://decodex.space" license = "GPL-3.0" -name = "name_placeholder" -readme = "README.md" -repository = "https://github.com/hack-ink/name_placeholder" -resolver = "3" +repository = "https://github.com/hack-ink/decodex" version = "0.1.0" -[package.metadata.docs.rs] -all-features = true - -[profile.final-release] -inherits = "release" -lto = true - -[build-dependencies] -# crates.io -vergen-gitcl = { version = "9.1", features = ["cargo"] } - -[dependencies] -# crates.io +[workspace.dependencies] +base64 = { version = "0.22" } clap = { version = "4.6", features = ["derive"] } color-eyre = { version = "0.6" } -directories = { version = "6.0" } +globset = { version = "0.4" } +libc = { version = "0.2" } +reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "rustls"] } +rusqlite = { version = "0.39", features = ["bundled"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +sha1 = { version = "0.11" } +tempfile = { version = "3.27" } +time = { version = "0.3", features = ["formatting"] } +toml = { version = "1.1" } tracing = { version = "0.1" } tracing-appender = { version = "0.2" } tracing-subscriber = { version = "0.3", features = ["env-filter"] } +vergen-gitcl = { version = "9.1", features = ["cargo"] } + +[profile.final-release] +inherits = "release" +lto = true diff --git a/Makefile.toml b/Makefile.toml index fdb46000..01063fd6 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,30 +1,28 @@ -# Rust workspace tasks. +# Decodex workspace tasks. -# Lint -# | task | type | cwd | -# | --------------- | --------- | --- | -# | lint | composite | | -# | lint-fix | composite | | -# | lint-rust | command | | -# | lint-fix-rust | extend | | -# | lint-vstyle | command | | -# | lint-fix-vstyle | command | | - -[tasks.lint] -workspace = false -dependencies = [ - "lint-rust", - "lint-vstyle", -] +# Check +# | task | type | cwd | +# | ------------------- | --------- | ---- | +# | check | composite | | +# | check-rust | command | | +# | check-site | composite | | +# | check-site-content | script | | +# | check-site-types | command | site | +# | check-vstyle | composite | | +# | check-vstyle-rust | command | | -[tasks.lint-fix] +[tasks.check] +clear = true workspace = false dependencies = [ - "lint-fix-rust", - "lint-fix-vstyle", + "fmt-check", + "check-rust", + "check-site", + "check-vstyle", + "test", ] -[tasks.lint-rust] +[tasks.check-rust] workspace = false command = "cargo" args = [ @@ -51,80 +49,48 @@ args = [ "warnings", ] -[tasks.lint-fix-rust] -extend = "lint-rust" -args = [ - "clippy", - "--fix", - "--allow-dirty", - "--all-features", - "--all-targets", - "--workspace", - "--", - "-D", - "clippy::all", - "-D", - "clippy::too_many_lines", - "-D", - "clippy::unwrap_used", - "-D", - "clippy::use_self", - "-D", - "clippy::wildcard_imports", - "-D", - "missing-docs", - "-D", - "unused-crate-dependencies", - "-D", - "warnings", +[tasks.check-site] +workspace = false +dependencies = [ + "check-site-content", + "build-site", + "check-site-types", ] -[tasks.lint-vstyle] +[tasks.check-site-content] workspace = false -command = "cargo" -args = [ - "vstyle", - "curate", - "--workspace", - "--all-features" +script = [ + "python3 tools/github/validate_change_bundle.py tools/github/bundles", + "python3 tools/github/validate_signal_entry.py site/src/content/signals", ] -[tasks.lint-fix-vstyle] +[tasks.check-site-types] workspace = false -command = "cargo" +cwd = "site" +command = "npm" args = [ - "vstyle", - "tune", - "--workspace", - "--all-features", - "--strict", + "run", + "check", ] - -# Test -# | task | type | cwd | -# | --------- | --------- | --- | -# | test | composite | | -# | test-rust | command | | - -[tasks.test] +[tasks.check-vstyle] workspace = false dependencies = [ - "test-rust", + "check-vstyle-rust", ] -[tasks.test-rust] +[tasks.check-vstyle-rust] workspace = false command = "cargo" args = [ - "nextest", - "run", + "vstyle", + "curate", + "--language", + "rust", "--workspace", - "--all-targets", "--all-features", ] - # Format # | task | type | cwd | # | -------------- | --------- | --- | @@ -171,54 +137,122 @@ args = [ "--check", ] +# Lint +# | task | type | cwd | +# | ---------------- | --------- | --- | +# | lint | composite | | +# | lint-rust | command | | +# | lint-vstyle | composite | | +# | lint-vstyle-rust | command | | -# Meta -# | task | type | cwd | -# | ------ | --------- | --- | -# | checks | composite | | -# | decodex-checks | composite | | -# | github-content-check | script | | -# | site-build | command | site | -# | site-check | command | site | +[tasks.lint] +workspace = false +dependencies = [ + "lint-rust", + "lint-vstyle", +] -[tasks.checks] +[tasks.lint-rust] +workspace = false +command = "cargo" +args = [ + "clippy", + "--fix", + "--allow-dirty", + "--all-features", + "--all-targets", + "--workspace", + "--", + "-D", + "clippy::all", + "-D", + "clippy::too_many_lines", + "-D", + "clippy::unwrap_used", + "-D", + "clippy::use_self", + "-D", + "clippy::wildcard_imports", + "-D", + "missing-docs", + "-D", + "unused-crate-dependencies", + "-D", + "warnings", +] + +[tasks.lint-vstyle] workspace = false dependencies = [ - "lint", - "test", - "fmt-check", + "lint-vstyle-rust", ] -[tasks.site-build] +[tasks.lint-vstyle-rust] workspace = false -cwd = "site" -command = "npm" +command = "cargo" args = [ + "vstyle", + "tune", + "--language", + "rust", + "--workspace", + "--all-features", + "--strict", +] + +# Test +# | task | type | cwd | +# | --------- | --------- | --- | +# | test | composite | | +# | test-rust | command | | + +[tasks.test] +clear = true +workspace = false +dependencies = [ + "test-rust", +] + +[tasks.test-rust] +workspace = false +command = "cargo" +args = [ + "nextest", "run", - "build", + "--workspace", + "--all-targets", + "--all-features", ] -[tasks.site-check] +# Build +# | task | type | cwd | +# | ---------- | ------- | ---- | +# | build-site | command | site | + +[tasks.build-site] workspace = false cwd = "site" command = "npm" args = [ "run", - "check", + "build", ] -[tasks.github-content-check] +# Meta +# | task | type | cwd | +# | -------------------- | --------- | --- | +# | checks | composite | | +# | decodex-checks | composite | | + +[tasks.checks] workspace = false -script = [ - "python3 tools/github/validate_change_bundle.py tools/github/bundles", - "python3 tools/github/validate_signal_entry.py site/src/content/signals", +dependencies = [ + "check", ] [tasks.decodex-checks] workspace = false dependencies = [ "fmt-check", - "github-content-check", - "site-build", - "site-check", + "check-site", ] diff --git a/README.md b/README.md index e5178f44..2965ee5a 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,179 @@ # Decodex -Decodex is a GitHub-first signal layer for Codex changes. +Decodex is a mono repo for repo-native agent orchestration and public Codex signal +publishing. + +The repository has two deliberately separate product surfaces: + +- a local Rust runtime and operator control plane for retained coding-agent lanes +- a static public site that publishes GitHub-backed Codex change signals -The immediate MVP goal is to turn GitHub PR and commit activity into compact, -actionable signal entries that answer: +The static site remains the public surface by default. Runtime and operator behavior +does not become a public site backend just because both surfaces now live in one +workspace. + +## Status + +This repository now integrates the former runtime repository and the static signal site +into one Decodex workspace: + +- `apps/decodex/` owns the Rust package that builds the `decodex` CLI and runtime. +- `site/` owns the Astro static site and checked-in public content. +- `tools/github/` owns deterministic GitHub bundle, release-delta, render, and + validation scripts. +- `plugins/decodex/` owns the installable Decodex plugin and reusable agent-facing + skills. +- `docs/` remains the authoritative documentation surface. + +Supported runtime host targets are macOS and Linux. Windows remains unsupported for the +runtime. + +## Repository Layout + +- `apps/decodex/src/` holds the Decodex runtime, orchestration logic, tracker + integrations, app-server integration, operator HTTP server, and authoritative + implementation behavior. +- `site/` holds the static Astro application and site-owned content collections. +- `tools/github/` holds deterministic public-signal collection and validation tooling. +- `plugins/decodex/` holds the canonical Decodex plugin source. +- `docs/spec/` holds normative runtime, workflow, site, and content contracts. +- `docs/runbook/` holds operator procedures, validation sequences, deployment steps, and + content workflows. +- `docs/reference/` holds current repository and artifact surface maps. +- `docs/decisions/` holds durable design rationale and tradeoffs. +- `docs/research/` holds machine-authored research run artifacts used by shipped + research tooling. +- `docs/plans/` holds historical saved plan artifacts from the static-site bootstrap. +- `scripts/` and `dev/` hold repository-level helpers that are not part of the shipped + runtime binary. +- `decodex.example.toml` is the redacted project-config template; live project + contracts live under `~/.codex/decodex/projects//`. -- what changed -- why it matters -- how to try it +## Runtime CLI -## Current status +From the workspace root: -This repository started as an organization Rust template. It is being reshaped into a -static-site plus tooling layout for Decodex. +```sh +cargo run -p decodex -- --help +cargo run -p decodex -- probe stdio:// +cargo run -p decodex -- project list +cargo run -p decodex -- status +cargo run -p decodex -- run --dry-run +cargo run -p decodex -- serve --interval 60s --listen-address 127.0.0.1:8912 +``` -During the transition: +Install the local runtime binary from the package directory: -- `site/` owns the Astro-based static site -- `tools/` owns the deterministic GitHub collection, render, and validation scripts -- `skills/` owns repo-local AI workflow entrypoints such as the GitHub signal drafting skill -- `docs/` remains the authoritative documentation surface -- the root Rust scaffold remains a legacy template surface until it is explicitly - removed or repurposed +```sh +cargo install --path apps/decodex --force +decodex --version +``` -## Documentation entry points +Project contracts are managed outside checkouts under +`~/.codex/decodex/projects//` with fixed filenames: -- `docs/index.md` routes documentation reads. -- `docs/spec/` defines normative contracts. -- `docs/guide/` defines repeatable workflows. -- `docs/plans/` stores saved `plan/1` artifacts. +- `project.toml` for service paths and credential environment-variable names +- `WORKFLOW.md` for execution policy -## MVP direction +Starting `decodex serve` without `--config` loads enabled projects from the explicit +registry only. It does not scan Codex history, repo-local config files, or currently +open worktrees to infer projects. -The first delivery focus is the GitHub lane: +## Static Site -- PR-first analysis -- commit and diff evidence -- local or trusted-runner Codex analysis -- static-site build and deployment through CI +The public site is an Astro static site under `site/`. It renders checked-in content and +generated JSON artifacts, then deploys through GitHub Pages. -The current seed path is live in-repo: +The public site owns: -- `tools/github/build_change_bundle.py` builds a normalized GitHub bundle -- `skills/decodex-github-signal/SKILL.md` defines the Codex editorial step -- `tools/github/sync_latest_signals.py` discovers recent merged PRs and refreshes the content artifacts -- `tools/github/render_signal_entry.py` renders a reviewed analysis draft into site content -- `tools/github/validate_signal_entry.py` validates the published collection -- `.github/workflows/refresh-github-signals.yml` refreshes GitHub-backed signals every hour from a trusted runner -- `.github/workflows/deploy-pages.yml` publishes the Astro site to GitHub Pages on pushes to `main` -- `site/src/content/signals/openai-codex-pr-15222.json` is the first real bundle-backed signal -- `cargo make decodex-checks` runs the current repo-native validation surface for the MVP +- Codex signal cards +- release-delta presentation +- recommended config artifacts +- static assets and public page rendering -## Production domain +The public site does not own: -The intended public domain is: +- retained-lane scheduling +- tracker writes +- local operator state +- app-server orchestration +- the operator dashboard served by `decodex serve` -- `https://decodex.space` +The static-site boundary is recorded in +[`docs/decisions/static-public-site.md`](docs/decisions/static-public-site.md). GitHub +Pages setup for `https://decodex.space` lives in +[`docs/runbook/github-pages-deploy.md`](docs/runbook/github-pages-deploy.md). -Repository code now includes the Pages deployment workflow and site URL. The -remaining GitHub Pages source selection, custom-domain, and DNS provider changes -are documented in: +## GitHub Signal Pipeline + +The GitHub-first public signal path stays deterministic and reviewable: -- `docs/guide/github_pages_deploy.md` +- `tools/github/build_change_bundle.py` builds normalized GitHub bundles. +- `plugins/decodex/skills/github-signal/SKILL.md` defines the Codex editorial step. +- `tools/github/sync_latest_signals.py` discovers recent merged PRs and refreshes + content artifacts. +- `tools/github/render_signal_entry.py` renders reviewed analysis drafts into site + content. +- `tools/github/validate_signal_entry.py` validates the published signal collection. +- `.github/workflows/refresh-github-signals.yml` refreshes GitHub-backed signals every + hour from a trusted runner. +- `.github/workflows/deploy-pages.yml` publishes the Astro site to GitHub Pages on + pushes to `main`. + +The governing workflow lives in +[`docs/runbook/local-github-signal-workflow.md`](docs/runbook/local-github-signal-workflow.md). + +## Operator Dashboard + +`decodex serve` owns the local operator listener. It serves one read-only operator +console from `GET /` and `GET /dashboard`, plus the same JSON status snapshot model from +`GET /state`. + +For dashboard UI development, use the mock operator state server: + +```sh +node dev/operator-dashboard-mock.mjs --listen-address 127.0.0.1:57399 +node dev/operator-dashboard-mock.mjs --listen-address 127.0.0.1:57399 --use-codex-auth +``` + +The dashboard semantics and local-vs-external state boundary live in +[`docs/reference/operator-control-plane.md`](docs/reference/operator-control-plane.md). + +## Development + +Repo-native validation is owned by `Makefile.toml`. + +Runtime checks follow the Decodex task structure: + +```sh +cargo make check +cargo make fmt +cargo make lint +cargo make test +``` + +Whole-workspace checks include runtime, static-site, and content validation: + +```sh +cargo make checks +``` + +Static-site/content checks are available separately: + +```sh +cargo make decodex-checks +``` + +## Documentation + +- Repository router: [`docs/index.md`](docs/index.md) +- Documentation policy: [`docs/policy.md`](docs/policy.md) +- Specifications: [`docs/spec/index.md`](docs/spec/index.md) +- Operational runbooks: [`docs/runbook/index.md`](docs/runbook/index.md) +- Reference docs: [`docs/reference/index.md`](docs/reference/index.md) +- Design decisions: [`docs/decisions/index.md`](docs/decisions/index.md) +- Workspace layout: [`docs/reference/workspace-layout.md`](docs/reference/workspace-layout.md) +- Static-site decision: [`docs/decisions/static-public-site.md`](docs/decisions/static-public-site.md) ## License diff --git a/apps/decodex/Cargo.toml b/apps/decodex/Cargo.toml new file mode 100644 index 00000000..38946d7a --- /dev/null +++ b/apps/decodex/Cargo.toml @@ -0,0 +1,42 @@ +[package] +authors.workspace = true +build = "build.rs" +categories = [] +description = "Repo-native orchestration for autonomous coding agents." +edition.workspace = true +homepage.workspace = true +keywords = ["agents", "automation", "codex", "orchestration"] +license.workspace = true +name = "decodex" +readme = "README.md" +repository.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[build-dependencies] +# crates.io +vergen-gitcl = { workspace = true } + +[dependencies] +# crates.io +base64 = { workspace = true } +clap = { workspace = true } +color-eyre = { workspace = true } +globset = { workspace = true } +libc = { workspace = true } +reqwest = { workspace = true } +rusqlite = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha1 = { workspace = true } +time = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +# crates.io +tempfile = { workspace = true } diff --git a/apps/decodex/README.md b/apps/decodex/README.md new file mode 100644 index 00000000..ca77b50d --- /dev/null +++ b/apps/decodex/README.md @@ -0,0 +1,6 @@ +# Decodex Runtime + +This package builds the `decodex` CLI and runtime. + +The repository-level README and docs remain the canonical entry points for workspace +layout, operator workflow, and the static signal site. diff --git a/apps/decodex/build.rs b/apps/decodex/build.rs new file mode 100644 index 00000000..80bdfa63 --- /dev/null +++ b/apps/decodex/build.rs @@ -0,0 +1,50 @@ +#![allow(missing_docs)] + +use std::{error::Error, process::Command}; + +use vergen_gitcl::{CargoBuilder, Emitter, GitclBuilder}; + +fn main() -> Result<(), Box> { + let mut emitter = Emitter::default(); + + emit_git_rerun_hints(); + + emitter.add_instructions(&CargoBuilder::default().target_triple(true).build()?)?; + + // Disable the git version if installed from . + if emitter.add_instructions(&GitclBuilder::default().sha(true).build()?).is_err() { + println!("cargo:rustc-env=VERGEN_GIT_SHA=crates.io"); + } + + emitter.emit()?; + + Ok(()) +} + +fn emit_git_rerun_hints() { + println!("cargo:rerun-if-changed=build.rs"); + + let Ok(git_dir) = git_output(&["rev-parse", "--path-format=absolute", "--git-dir"]) else { + return; + }; + + println!("cargo:rerun-if-changed={git_dir}/HEAD"); + + let Ok(head_ref) = git_output(&["symbolic-ref", "-q", "HEAD"]) else { + return; + }; + let git_common_dir = git_output(&["rev-parse", "--path-format=absolute", "--git-common-dir"]) + .unwrap_or_else(|_| git_dir.clone()); + + println!("cargo:rerun-if-changed={git_common_dir}/{head_ref}"); +} + +fn git_output(args: &[&str]) -> Result> { + let output = Command::new("git").args(args).output()?; + + if !output.status.success() { + return Err(format!("git {} failed", args.join(" ")).into()); + } + + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} diff --git a/apps/decodex/src/agent.rs b/apps/decodex/src/agent.rs new file mode 100644 index 00000000..e2f4873c --- /dev/null +++ b/apps/decodex/src/agent.rs @@ -0,0 +1,26 @@ +mod app_server; +mod codex_accounts; +mod decodex_tool_bridge; +mod json_rpc; +mod tracker_tool_bridge; + +#[cfg(test)] pub(crate) use self::tracker_tool_bridge::DynamicToolHandler; +pub(crate) use self::{ + app_server::{ + ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, + AppServerRunRequest, AppServerRunResult, AppServerTurnFailure, TurnContinuationGuard, + execute_app_server_run, probe_app_server, + }, + codex_accounts::{CodexAccountPool, CodexAccountProvider}, + decodex_tool_bridge::{DecodexRunContext, DecodexToolBridge}, + json_rpc::{AppServerHomePreflightFailure, AppServerProcessEnv, AppServerTransportFailure}, + tracker_tool_bridge::{ + ISSUE_COMMENT_TOOL_NAME, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + ISSUE_TRANSITION_TOOL_NAME, ReviewExecutionMode, ReviewHandoffContext, + ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, + RunCompletionDisposition, TrackerToolBridge, + }, +}; diff --git a/apps/decodex/src/agent/app_server.rs b/apps/decodex/src/agent/app_server.rs new file mode 100644 index 00000000..a432bdf0 --- /dev/null +++ b/apps/decodex/src/agent/app_server.rs @@ -0,0 +1,3533 @@ +mod protocol; +mod turn_failure; + +pub(crate) use turn_failure::AppServerTurnFailure; + +use std::{ + collections::{BTreeMap, HashMap}, + env, + error::Error, + fmt::{self, Display, Formatter}, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; + +use color_eyre::Report; +use serde::Serialize; +use serde_json::{self, Value}; +use time::OffsetDateTime; + +use self::protocol::{ + AgentMessageDeltaNotification, AppServerClient, ChatgptAuthTokensRefreshParams, + ChatgptAuthTokensRefreshResponse, CommandExecParams, CommandExecResponse, + CommandExecutionApprovalDecision, CommandExecutionRequestApprovalResponse, ConfigReadParams, + DynamicToolCallParams, EffectiveThreadConfig, ErrorNotification, FileChangeApprovalDecision, + FileChangeRequestApprovalResponse, InitializeResponse, ItemCompletedNotification, + ListMcpServerStatusParams, ListMcpServerStatusResponse, LoginAccountParams, + McpServerElicitationAction, McpServerElicitationRequestResponse, McpServerStatusSummary, + ModelListParams, ModelListResponse, ModelProviderCapabilitiesReadResponse, ModelSummary, + PermissionGrantScope, PermissionsRequestApprovalResponse, PluginListParams, PluginListResponse, + ProbeDynamicToolHandler, RunOutcome, RuntimeConfigSummary, SkillsListParams, + SkillsListResponse, ThreadResumeRequest, ThreadSessionResponse, ThreadStartRequest, + ThreadStatusChangedNotification, ToolRequestUserInputResponse, TurnCompletedNotification, + TurnError, TurnStartRequest, UserInput, +}; +use crate::{ + agent::{ + app_server::protocol::LoginAccountResponse, + codex_accounts::{CodexAccountLogin, CodexAccountProvider}, + json_rpc::{ + AppServerHomePreflightFailure, AppServerOutputTimeout, AppServerProcessEnv, + JsonRpcConnection, JsonRpcMessage, JsonRpcNotification, JsonRpcRequest, + ResolvedAppServerCodexHomeEnv, WireMessage, + }, + tracker_tool_bridge::{ + self, DynamicToolCallResponse, DynamicToolContentItem, DynamicToolHandler, + DynamicToolSpec, TurnCompletionStatus, + }, + }, + prelude::eyre, + state::{ + self, CodexAccountActivitySummary, CodexAccountMarker, EffectiveRuntimeMarker, + RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, StateStore, + }, +}; + +pub(crate) const ACTIVE_RUN_IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +const PROBE_TIMEOUT: Duration = Duration::from_secs(30); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const PROBE_RUN_ID: &str = "protocol-probe-run"; +const PROBE_ISSUE_ID: &str = "protocol-probe"; +const PROBE_EXPECTED_OUTPUT: &str = "PROBE_OK"; +const PROBE_COMMAND_EXEC_EXPECTED_OUTPUT: &str = "COMMAND_EXEC_OK"; +const PROBE_COMMAND_EXEC_TIMEOUT_MS: u64 = 5_000; +const PROBE_COMMAND_EXEC_OUTPUT_BYTES_CAP: u64 = 1_024; +const PROBE_DEVELOPER_INSTRUCTIONS: &str = "You are a protocol probe. You must call the dynamic tool `echo_probe` exactly once with the JSON argument `{\"text\":\"PROBE_OK\"}`. Do not use shell. Do not inspect files. After the tool response is returned, reply with the exact text PROBE_OK and nothing else."; +const PROBE_USER_INPUT: &str = "Call `echo_probe` with `{\\\"text\\\":\\\"PROBE_OK\\\"}`. After the tool succeeds, reply with the exact text PROBE_OK."; +const PREFLIGHT_EVENT_TYPE: &str = "app-server/preflight"; +const PREFLIGHT_MODEL_PAGE_LIMIT: u32 = 200; +const PREFLIGHT_MCP_PAGE_LIMIT: u32 = 200; +const PREFLIGHT_MCP_DETAIL: &str = "toolsAndAuthOnly"; +const PREFLIGHT_CHECK_CONFIG: &str = "config"; +const PREFLIGHT_CHECK_MODEL: &str = "model"; +const PREFLIGHT_CHECK_MODEL_PROVIDER: &str = "model_provider"; +const PREFLIGHT_CHECK_SKILLS: &str = "skills"; +const PREFLIGHT_CHECK_PLUGINS: &str = "plugins"; +const PREFLIGHT_CHECK_MCP: &str = "mcp"; +const JSONRPC_METHOD_NOT_FOUND: i64 = -32_601; +const CHILD_BUCKET_MODEL: &str = "Model"; +const CHILD_BUCKET_PROTOCOL: &str = "Protocol"; +const CHILD_BUCKET_TOOL: &str = "Tool"; +const CHILD_BUCKET_SHELL: &str = "Shell"; +const CHILD_BUCKET_TRACKER: &str = "Tracker"; +const CHILD_BUCKET_BROWSER_IMAGE: &str = "Browser/Image"; +const CHILD_BUCKET_PR_LAND: &str = "PR/Land"; +const LARGE_CHILD_OUTPUT_BYTES: i64 = 100_000; +const RECENT_PROTOCOL_ACTIVITY_LIMIT: usize = 8; +const INPUT_TOKEN_KEYS: &[&str] = &[ + "input_tokens", + "inputTokens", + "prompt_tokens", + "promptTokens", + "total_input_tokens", + "totalInputTokens", +]; +const OUTPUT_TOKEN_KEYS: &[&str] = &[ + "output_tokens", + "outputTokens", + "completion_tokens", + "completionTokens", + "total_output_tokens", + "totalOutputTokens", +]; + +pub(crate) trait TurnContinuationGuard { + fn should_continue_turn(&self, turn_count: u32) -> crate::prelude::Result; + fn validate_continuation_boundary(&self, _turn_count: u32) -> crate::prelude::Result<()> { + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct AppServerCapabilityPreflightReport { + checks: Vec, +} +impl AppServerCapabilityPreflightReport { + fn new() -> Self { + Self { checks: Vec::new() } + } + + #[cfg(test)] + fn checks(&self) -> &[AppServerCapabilityPreflightCheck] { + &self.checks + } + + fn push_ok( + &mut self, + name: &'static str, + summary: impl Into, + details: BTreeMap, + ) { + self.checks.push(AppServerCapabilityPreflightCheck { + name, + status: AppServerCapabilityPreflightStatus::Ok, + summary: summary.into(), + details, + }); + } + + fn push_blocked( + &mut self, + name: &'static str, + summary: impl Into, + details: BTreeMap, + ) { + self.checks.push(AppServerCapabilityPreflightCheck { + name, + status: AppServerCapabilityPreflightStatus::Blocked, + summary: summary.into(), + details, + }); + } + + fn has_blockers(&self) -> bool { + self.checks.iter().any(|check| check.status == AppServerCapabilityPreflightStatus::Blocked) + } + + fn blocker_summary(&self) -> String { + let blockers = self + .checks + .iter() + .filter(|check| check.status == AppServerCapabilityPreflightStatus::Blocked) + .map(|check| format!("{}: {}", check.name, check.summary)) + .collect::>(); + + if blockers.is_empty() { String::from("no blockers recorded") } else { blockers.join("; ") } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AppServerCapabilityPreflightFailure { + kind: AppServerCapabilityPreflightFailureKind, + report: AppServerCapabilityPreflightReport, +} +impl AppServerCapabilityPreflightFailure { + fn blocked(report: AppServerCapabilityPreflightReport) -> Self { + Self { kind: AppServerCapabilityPreflightFailureKind::BlockedState, report } + } + + fn method_failed( + method: &'static str, + error: String, + report: AppServerCapabilityPreflightReport, + ) -> Self { + Self { + kind: AppServerCapabilityPreflightFailureKind::MethodFailed { method, error }, + report, + } + } + + #[cfg(test)] + pub(crate) fn blocked_for_test(check: &'static str, summary: &str) -> Self { + let mut report = AppServerCapabilityPreflightReport::new(); + + report.push_blocked(check, summary, BTreeMap::new()); + + Self::blocked(report) + } + + pub(crate) fn error_class(&self) -> &'static str { + match self.kind { + AppServerCapabilityPreflightFailureKind::MethodFailed { .. } => + "app_server_introspection_method_failed", + AppServerCapabilityPreflightFailureKind::BlockedState => + "app_server_runtime_preflight_failed", + } + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + format!( + "inspect Codex app-server preflight blocker `{}`, repair the local Codex config/model/provider/skills/plugin/MCP state, restart `decodex serve`, {recovery_gate}", + self.blocker_summary() + ) + } + + fn blocker_summary(&self) -> String { + match &self.kind { + AppServerCapabilityPreflightFailureKind::MethodFailed { method, error } => { + format!("{}: `{method}` returned {error}", check_name_for_method(method)) + }, + AppServerCapabilityPreflightFailureKind::BlockedState => self.report.blocker_summary(), + } + } + + #[cfg(test)] + fn report(&self) -> &AppServerCapabilityPreflightReport { + &self.report + } +} + +impl Display for AppServerCapabilityPreflightFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!(formatter, "app_server_preflight_failed: {}", self.blocker_summary()) + } +} + +impl Error for AppServerCapabilityPreflightFailure {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AppServerDynamicToolFailure { + kind: AppServerDynamicToolFailureKind, + tool: Option, + message: String, +} +impl AppServerDynamicToolFailure { + fn protocol(tool: Option, message: impl Into) -> Self { + Self { kind: AppServerDynamicToolFailureKind::Protocol, tool, message: message.into() } + } + + fn tool(tool: Option, message: impl Into) -> Self { + Self { kind: AppServerDynamicToolFailureKind::Tool, tool, message: message.into() } + } + + pub(crate) fn error_class(&self) -> &'static str { + match self.kind { + AppServerDynamicToolFailureKind::Protocol => "app_server_dynamic_tool_protocol_failure", + AppServerDynamicToolFailureKind::Tool => "app_server_dynamic_tool_failed", + } + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + match self.kind { + AppServerDynamicToolFailureKind::Protocol => format!( + "inspect the app-server dynamic tool declaration and `item/tool/call` payload, repair the protocol mismatch manually, {recovery_gate}" + ), + AppServerDynamicToolFailureKind::Tool => format!( + "inspect the dynamic tool response and lane state, correct the tool call or underlying service state manually, {recovery_gate}" + ), + } + } + + fn diagnostic_next_action(&self) -> &'static str { + match self.kind { + AppServerDynamicToolFailureKind::Protocol => + "inspect the declared dynamic tool surface and item/tool/call payload before retrying the lane", + AppServerDynamicToolFailureKind::Tool => + "inspect the tool response, correct the call arguments or backing state, and retry the tool call", + } + } +} + +impl Display for AppServerDynamicToolFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!(formatter, "app_server_dynamic_tool_failure: {}", self.message)?; + + if let Some(tool) = self.tool.as_deref() { + write!(formatter, " (tool `{tool}`)")?; + } + + Ok(()) + } +} + +impl Error for AppServerDynamicToolFailure {} + +#[derive(Clone)] +pub(crate) struct AppServerRunRequest<'a> { + pub(crate) run_id: String, + pub(crate) issue_id: String, + pub(crate) attempt_number: i64, + pub(crate) listen: String, + pub(crate) cwd: String, + pub(crate) developer_instructions: String, + pub(crate) user_input: String, + pub(crate) max_turns: u32, + pub(crate) timeout: Duration, + pub(crate) process_env: AppServerProcessEnv, + pub(crate) continuation_user_input: Option, + pub(crate) activity_marker_path: Option, + pub(crate) resume_thread_id: Option, + pub(crate) command_exec_health_check: Option, + pub(crate) dynamic_tool_handler: Option<&'a dyn DynamicToolHandler>, + pub(crate) continuation_guard: Option<&'a dyn TurnContinuationGuard>, + pub(crate) codex_account_provider: Option<&'a dyn CodexAccountProvider>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CommandExecHealthCheck { + pub(crate) command: Vec, + pub(crate) expected_stdout: String, + pub(crate) timeout_ms: u64, + pub(crate) output_bytes_cap: u64, +} +impl CommandExecHealthCheck { + fn probe() -> Self { + Self { + command: vec![ + String::from("/bin/sh"), + String::from("-c"), + format!("printf {PROBE_COMMAND_EXEC_EXPECTED_OUTPUT}"), + ], + expected_stdout: String::from(PROBE_COMMAND_EXEC_EXPECTED_OUTPUT), + timeout_ms: PROBE_COMMAND_EXEC_TIMEOUT_MS, + output_bytes_cap: PROBE_COMMAND_EXEC_OUTPUT_BYTES_CAP, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AppServerRunResult { + pub(crate) user_agent: String, + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) turn_count: u32, + pub(crate) event_count: i64, + pub(crate) final_output: String, + pub(crate) continuation_pending: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct AppServerCapabilityPreflightCheck { + name: &'static str, + status: AppServerCapabilityPreflightStatus, + summary: String, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + details: BTreeMap, +} + +#[derive(Clone, Debug)] +struct ChildActivityEvent { + event_bucket: String, + event_detail: Option, + transition_bucket: Option, + transition_detail: Option, + tool_name: Option, + tool_call: bool, + tool_output_bytes: Option, + input_tokens: Option, + output_tokens: Option, + completed: bool, +} + +#[derive(Clone, Copy, Debug, Default)] +struct LargeOutputStats { + count: i64, + max_bytes: i64, +} + +struct ChildActivityAccumulator { + started_at: Instant, + last_observed_at: Instant, + current_bucket: Option, + current_detail: Option, + active_tool_name: Option, + large_output_stats: HashMap, + summary: state::ChildAgentActivitySummary, +} +impl ChildActivityAccumulator { + fn new() -> Self { + let now = Instant::now(); + + Self { + started_at: now, + last_observed_at: now, + current_bucket: None, + current_detail: None, + active_tool_name: None, + large_output_stats: HashMap::new(), + summary: state::ChildAgentActivitySummary::default(), + } + } + + fn record(&mut self, event_type: &str, payload: &str) -> state::ChildAgentActivitySummary { + let now = Instant::now(); + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + + self.add_elapsed_time(now); + + let event = + classify_child_activity_event(event_type, payload, self.active_tool_name.as_deref()); + + self.summary.event_count += 1; + self.summary.wall_seconds = + duration_seconds_i64(now.saturating_duration_since(self.started_at)); + + self.record_event_bucket(&event); + + if let Some(tool_name) = event.tool_name.as_ref().filter(|_tool_name| event.tool_call) { + self.active_tool_name = Some(tool_name.clone()); + } + + if event_type == "item/tool/call/response" { + self.active_tool_name = None; + } + if event.completed { + self.set_current(None, None, None); + } else if let Some(next_bucket) = event.transition_bucket { + self.set_current(Some(next_bucket), event.transition_detail, Some(now_unix_epoch)); + } + + self.summary.current_elapsed_seconds = + self.summary.current_started_unix_epoch.and_then(|started_at| { + now_unix_epoch.checked_sub(started_at).filter(|elapsed| *elapsed >= 0) + }); + self.last_observed_at = now; + + self.summary.clone() + } + + fn add_elapsed_time(&mut self, now: Instant) { + let Some(bucket_name) = self.current_bucket.clone() else { + return; + }; + let seconds = duration_seconds_i64(now.saturating_duration_since(self.last_observed_at)); + let bucket = child_activity_bucket_mut(&mut self.summary, &bucket_name); + + bucket.wall_seconds = bucket.wall_seconds.saturating_add(seconds); + } + + fn record_event_bucket(&mut self, event: &ChildActivityEvent) { + { + let bucket = child_activity_bucket_mut(&mut self.summary, &event.event_bucket); + + bucket.event_count += 1; + + if event.tool_call { + bucket.tool_call_count += 1; + } + + if let Some(input_tokens) = event.input_tokens { + bucket.input_tokens = bucket.input_tokens.saturating_add(input_tokens); + } + if let Some(output_tokens) = event.output_tokens { + bucket.output_tokens = bucket.output_tokens.saturating_add(output_tokens); + } + if let Some(output_bytes) = event.tool_output_bytes { + bucket.output_bytes = bucket.output_bytes.saturating_add(output_bytes); + } + } + + if event.tool_call { + self.summary.tool_call_count += 1; + } + + if let Some(input_tokens) = event.input_tokens { + self.summary.input_tokens_current = Some(input_tokens); + self.summary.input_tokens_max = Some( + self.summary + .input_tokens_max + .map_or(input_tokens, |max_tokens| max_tokens.max(input_tokens)), + ); + self.summary.input_tokens_cumulative = + self.summary.input_tokens_cumulative.saturating_add(input_tokens); + } + if let Some(output_tokens) = event.output_tokens { + self.summary.output_tokens_cumulative = + self.summary.output_tokens_cumulative.saturating_add(output_tokens); + } + if let Some(output_bytes) = event.tool_output_bytes { + self.record_tool_output(event, output_bytes); + } + } + + fn record_tool_output(&mut self, event: &ChildActivityEvent, output_bytes: i64) { + let tool_name = + event.tool_name.as_deref().or(event.event_detail.as_deref()).unwrap_or("tool"); + + if self.summary.largest_tool_output_bytes.is_none_or(|largest| output_bytes > largest) { + self.summary.largest_tool_output_bytes = Some(output_bytes); + self.summary.largest_tool_output_tool = Some(tool_name.to_owned()); + } + if output_bytes < LARGE_CHILD_OUTPUT_BYTES { + return; + } + + let stats = self.large_output_stats.entry(tool_name.to_owned()).or_default(); + + stats.count += 1; + stats.max_bytes = stats.max_bytes.max(output_bytes); + + self.refresh_large_output_warnings(); + } + + fn refresh_large_output_warnings(&mut self) { + let mut entries = self + .large_output_stats + .iter() + .map(|(tool_name, stats)| (tool_name.clone(), *stats)) + .collect::>(); + + entries.sort_by(|left, right| { + right.1.max_bytes.cmp(&left.1.max_bytes).then_with(|| left.0.cmp(&right.0)) + }); + + self.summary.large_output_warnings = entries + .into_iter() + .take(4) + .map(|(tool_name, stats)| { + if stats.count > 1 { + format!( + "{tool_name} repeated {} large outputs; largest {} bytes", + stats.count, stats.max_bytes + ) + } else { + format!("{tool_name} produced a large output: {} bytes", stats.max_bytes) + } + }) + .collect(); + } + + fn set_current( + &mut self, + bucket: Option, + detail: Option, + started_unix_epoch: Option, + ) { + if self.current_bucket == bucket && self.current_detail == detail { + return; + } + + self.current_bucket = bucket.clone(); + self.current_detail = detail.clone(); + self.summary.current_bucket = bucket; + self.summary.current_detail = detail; + self.summary.current_started_unix_epoch = started_unix_epoch; + self.summary.current_elapsed_seconds = None; + } +} + +struct ProtocolActivityAccumulator { + summary: state::ProtocolActivitySummary, +} +impl ProtocolActivityAccumulator { + fn new() -> Self { + Self { summary: state::ProtocolActivitySummary::default() } + } + + fn record( + &mut self, + event_type: &str, + payload: &str, + child_activity: &state::ChildAgentActivitySummary, + ) -> state::ProtocolActivitySummary { + self.summary.recent_events.push(protocol_activity_event(event_type, payload)); + + if self.summary.recent_events.len() > RECENT_PROTOCOL_ACTIVITY_LIMIT { + let remove_count = + self.summary.recent_events.len().saturating_sub(RECENT_PROTOCOL_ACTIVITY_LIMIT); + + self.summary.recent_events.drain(0..remove_count); + } + + if let Some(turn_status) = protocol_turn_status_from_payload(event_type, payload) { + self.summary.turn_status = Some(turn_status); + } + if let Some(waiting_reason) = protocol_waiting_reason(event_type, payload, child_activity) { + self.summary.waiting_reason = Some(waiting_reason); + } + if let Some(rate_limit_status) = protocol_rate_limit_status(event_type, payload) { + self.summary.rate_limit_status = Some(rate_limit_status); + } + + self.summary.clone() + } +} + +struct RunRecorder<'a> { + state_store: &'a StateStore, + run_id: &'a str, + attempt_number: i64, + activity_marker_path: Option<&'a PathBuf>, + thread_id: Option, + turn_id: Option, + next_sequence: i64, + child_activity: ChildActivityAccumulator, + protocol_activity: ProtocolActivityAccumulator, +} +impl<'a> RunRecorder<'a> { + fn new( + state_store: &'a StateStore, + run_id: &'a str, + attempt_number: i64, + activity_marker_path: Option<&'a PathBuf>, + ) -> Self { + Self { + state_store, + run_id, + attempt_number, + activity_marker_path, + thread_id: None, + turn_id: None, + next_sequence: 1, + child_activity: ChildActivityAccumulator::new(), + protocol_activity: ProtocolActivityAccumulator::new(), + } + } + + fn mark_activity(&self) -> crate::prelude::Result<()> { + if let Some(marker_path) = self.activity_marker_path { + write_activity_marker_best_effort(marker_path, self.run_id, self.attempt_number); + }; + + Ok(()) + } + + fn set_thread_id(&mut self, thread_id: &str) -> crate::prelude::Result<()> { + self.thread_id = Some(thread_id.to_owned()); + + if let Some(marker_path) = self.activity_marker_path { + write_thread_marker_best_effort( + marker_path, + self.run_id, + self.attempt_number, + thread_id, + ); + } + + Ok(()) + } + + fn set_turn_id(&mut self, turn_id: &str) -> crate::prelude::Result<()> { + self.turn_id = Some(turn_id.to_owned()); + + if let Some(marker_path) = self.activity_marker_path { + write_turn_marker_best_effort(marker_path, self.run_id, self.attempt_number, turn_id); + } + + Ok(()) + } + + fn set_thread_status( + &mut self, + status: &str, + active_flags: &[String], + ) -> crate::prelude::Result<()> { + if let Some(marker_path) = self.activity_marker_path { + write_thread_status_marker_best_effort( + marker_path, + self.run_id, + self.attempt_number, + self.thread_id.as_deref(), + self.turn_id.as_deref(), + status, + active_flags, + ); + } + + Ok(()) + } + + fn set_effective_runtime( + &mut self, + runtime: &EffectiveThreadConfig, + ) -> crate::prelude::Result<()> { + if let Some(marker_path) = self.activity_marker_path { + write_effective_runtime_marker_best_effort( + marker_path, + self.run_id, + self.attempt_number, + self.thread_id.as_deref(), + self.turn_id.as_deref(), + runtime, + ); + } + + Ok(()) + } + + fn set_codex_account( + &mut self, + summary: &CodexAccountActivitySummary, + account_summaries: &[CodexAccountActivitySummary], + ) -> crate::prelude::Result<()> { + if let Some(marker_path) = self.activity_marker_path { + write_codex_account_marker_best_effort( + marker_path, + self.run_id, + self.attempt_number, + summary, + account_summaries, + ); + } + + Ok(()) + } + + fn record(&mut self, event_type: &str, payload: &str) -> crate::prelude::Result<()> { + self.state_store.append_event(self.run_id, self.next_sequence, event_type, payload)?; + + let child_activity = self.child_activity.record(event_type, payload); + let protocol_activity = self.protocol_activity.record(event_type, payload, &child_activity); + + if let Some(marker_path) = self.activity_marker_path { + let activity = state::ProtocolActivityMarker { + run_id: self.run_id, + attempt_number: self.attempt_number, + thread_id: self.thread_id.as_deref(), + turn_id: self.turn_id.as_deref(), + event_count: self.next_sequence, + last_event_type: event_type, + child_agent_activity: Some(&child_activity), + protocol_activity: Some(&protocol_activity), + }; + + write_protocol_activity_marker_best_effort(marker_path, &activity); + } + + self.next_sequence += 1; + + Ok(()) + } +} + +struct TurnLoopResult { + turn_id: String, + turn_count: u32, + final_output: String, + continuation_pending: bool, +} + +#[derive(Clone, Copy)] +struct RequestDispatchContext<'a> { + phase: RequestWaitPhase, + dynamic_tool_handler: Option<&'a dyn DynamicToolHandler>, + codex_account_provider: Option<&'a dyn CodexAccountProvider>, + target_thread_id: Option<&'a str>, + target_turn_id: Option<&'a str>, +} +impl<'a> RequestDispatchContext<'a> { + fn new( + phase: RequestWaitPhase, + dynamic_tool_handler: Option<&'a dyn DynamicToolHandler>, + codex_account_provider: Option<&'a dyn CodexAccountProvider>, + target_thread_id: Option<&'a str>, + target_turn_id: Option<&'a str>, + ) -> Self { + Self { + phase, + dynamic_tool_handler, + codex_account_provider, + target_thread_id, + target_turn_id, + } + } +} + +#[derive(Debug)] +struct DynamicToolCallDispatch { + response: DynamicToolCallResponse, + diagnostic: Option, + terminal_failure: Option, +} +impl DynamicToolCallDispatch { + fn success(response: DynamicToolCallResponse) -> Self { + Self { response, diagnostic: None, terminal_failure: None } + } + + fn tool_failure( + response: DynamicToolCallResponse, + tool: Option, + namespace: Option, + ) -> Self { + let message = dynamic_tool_response_text(&response); + let failure = AppServerDynamicToolFailure::tool(tool.clone(), message.clone()); + + Self { + response, + diagnostic: Some(DynamicToolFailureDiagnostic::from_failure(&failure, namespace)), + terminal_failure: None, + } + } + + fn protocol_failure(tool: Option, namespace: Option, message: String) -> Self { + let failure = AppServerDynamicToolFailure::protocol(tool, message.clone()); + + Self { + response: DynamicToolCallResponse::failure(message), + diagnostic: Some(DynamicToolFailureDiagnostic::from_failure(&failure, namespace)), + terminal_failure: Some(failure), + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DynamicToolFailureDiagnostic { + failure_class: &'static str, + tool: Option, + namespace: Option, + message: String, + next_action: &'static str, +} +impl DynamicToolFailureDiagnostic { + fn from_failure(failure: &AppServerDynamicToolFailure, namespace: Option) -> Self { + Self { + failure_class: failure.error_class(), + tool: failure.tool.clone(), + namespace, + message: failure.message.clone(), + next_action: failure.diagnostic_next_action(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AppServerDynamicToolFailureKind { + Protocol, + Tool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum AppServerCapabilityPreflightStatus { + Ok, + Blocked, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum AppServerCapabilityPreflightFailureKind { + MethodFailed { method: &'static str, error: String }, + BlockedState, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RequestWaitPhase { + Initialize, + AccountLogin, + ThreadStart, + ThreadResume, + TurnStart, + TurnExecution, +} +impl RequestWaitPhase { + fn label(self) -> &'static str { + match self { + Self::Initialize => "initialize", + Self::AccountLogin => "account/login/start", + Self::ThreadStart => "thread/start", + Self::ThreadResume => "thread/resume", + Self::TurnStart => "turn/start", + Self::TurnExecution => "turn execution", + } + } +} + +pub(crate) fn execute_app_server_run( + request: &AppServerRunRequest<'_>, + state_store: &StateStore, +) -> crate::prelude::Result { + state_store.record_run_attempt( + &request.run_id, + &request.issue_id, + request.attempt_number, + "starting", + )?; + + if let Some(marker_path) = request.activity_marker_path.as_ref() { + write_activity_marker_best_effort(marker_path, &request.run_id, request.attempt_number); + } + + let result = execute_app_server_run_inner(request, state_store); + + if result.is_err() { + state_store.record_run_attempt( + &request.run_id, + &request.issue_id, + request.attempt_number, + "failed", + )?; + + if let Some(marker_path) = request.activity_marker_path.as_ref() { + write_activity_marker_best_effort(marker_path, &request.run_id, request.attempt_number); + } + } + + result +} + +pub(crate) fn probe_app_server(listen: &str) -> crate::prelude::Result { + let state_store = StateStore::open_in_memory()?; + let probe_tool_handler = ProbeDynamicToolHandler; + let result = execute_app_server_run( + &AppServerRunRequest { + run_id: PROBE_RUN_ID.to_owned(), + issue_id: PROBE_ISSUE_ID.to_owned(), + attempt_number: 1, + listen: listen.to_owned(), + cwd: env::current_dir()?.display().to_string(), + developer_instructions: PROBE_DEVELOPER_INSTRUCTIONS.to_owned(), + user_input: PROBE_USER_INPUT.to_owned(), + max_turns: 1, + timeout: PROBE_TIMEOUT, + process_env: AppServerProcessEnv::default(), + continuation_user_input: None, + activity_marker_path: None, + resume_thread_id: None, + command_exec_health_check: Some(CommandExecHealthCheck::probe()), + dynamic_tool_handler: Some(&probe_tool_handler), + continuation_guard: None, + codex_account_provider: None, + }, + &state_store, + )?; + + if result.final_output.trim() != PROBE_EXPECTED_OUTPUT { + eyre::bail!( + "Protocol probe completed, but the final output was `{}` instead of `{PROBE_EXPECTED_OUTPUT}`.", + result.final_output.trim() + ); + } + + Ok(result) +} + +fn classify_child_activity_event( + event_type: &str, + payload: &str, + active_tool_name: Option<&str>, +) -> ChildActivityEvent { + let payload_value = serde_json::from_str::(payload).ok(); + let input_tokens = + payload_value.as_ref().and_then(|value| find_numeric_field(value, INPUT_TOKEN_KEYS)); + let output_tokens = + payload_value.as_ref().and_then(|value| find_numeric_field(value, OUTPUT_TOKEN_KEYS)); + + match event_type { + "item/tool/call" => + child_tool_call_event(payload_value.as_ref(), input_tokens, output_tokens), + "item/tool/call/response" => + child_tool_response_event(payload_value.as_ref(), active_tool_name, payload), + "item/completed" => child_item_completed_event(payload_value.as_ref(), payload), + "item/agentMessage/delta" => ChildActivityEvent { + event_bucket: CHILD_BUCKET_MODEL.to_owned(), + event_detail: Some(String::from("agent_message_delta")), + transition_bucket: Some(CHILD_BUCKET_MODEL.to_owned()), + transition_detail: Some(String::from("streaming response")), + tool_name: None, + tool_call: false, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: false, + }, + "turn/completed" => ChildActivityEvent { + event_bucket: CHILD_BUCKET_MODEL.to_owned(), + event_detail: Some(String::from("turn_completed")), + transition_bucket: None, + transition_detail: None, + tool_name: None, + tool_call: false, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: true, + }, + "thread/status/changed" => ChildActivityEvent { + event_bucket: CHILD_BUCKET_MODEL.to_owned(), + event_detail: Some(String::from("thread_status")), + transition_bucket: Some(CHILD_BUCKET_MODEL.to_owned()), + transition_detail: Some(String::from("child thread active")), + tool_name: None, + tool_call: false, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: false, + }, + other => ChildActivityEvent { + event_bucket: CHILD_BUCKET_PROTOCOL.to_owned(), + event_detail: Some(other.to_owned()), + transition_bucket: None, + transition_detail: None, + tool_name: None, + tool_call: false, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: false, + }, + } +} + +fn protocol_activity_event(event_type: &str, payload: &str) -> state::ProtocolActivityEventSummary { + let payload_value = serde_json::from_str::(payload).ok(); + + state::ProtocolActivityEventSummary { + event_type: event_type.to_owned(), + category: protocol_activity_category(event_type).to_owned(), + detail: protocol_activity_detail(event_type, payload_value.as_ref()), + } +} + +fn protocol_activity_category(event_type: &str) -> &'static str { + let normalized = event_type.to_ascii_lowercase(); + + if normalized.starts_with("turn/") { + return "turn"; + } + if normalized.contains("plan") { + return "plan"; + } + if normalized.contains("diff") || normalized.contains("filechange") { + return "diff"; + } + if normalized.contains("command") + && (normalized.contains("output") || normalized.contains("delta")) + { + return "command_output"; + } + if normalized.contains("ratelimit") || normalized.contains("rate_limit") { + return "rate_limit"; + } + if normalized.starts_with("account/") { + return "account"; + } + if normalized == "item/tool/call/failure" { + return "protocol_error"; + } + if normalized.ends_with("/response") || normalized == "json-rpc/error/response" { + return "server_request_resolution"; + } + if normalized.starts_with("item/") { + return "item"; + } + if normalized == "thread/status/changed" { + return "thread"; + } + if normalized == "error" || normalized.contains("error") { + return "protocol_error"; + } + + "protocol" +} + +fn protocol_activity_detail(event_type: &str, payload_value: Option<&Value>) -> Option { + if event_type == "thread/status/changed" { + return payload_value.and_then(|value| { + string_at_paths(value, &[&["params", "status", "type"], &["status", "type"]]) + }); + } + if event_type.starts_with("turn/") { + return protocol_turn_status_from_value(event_type, payload_value) + .or_else(|| Some(String::from("running"))); + } + if event_type == "item/tool/call" { + return payload_value.and_then(extract_tool_name); + } + if event_type == "item/tool/call/failure" { + return payload_value.and_then(|value| { + string_at_paths(value, &[&["failureClass"], &["failure_class"]]) + .or_else(|| string_at_paths(value, &[&["tool"]])) + }); + } + if event_type == "item/completed" { + return payload_value.and_then(|value| { + string_at_paths(value, &[&["params", "item", "type"], &["item", "type"]]) + }); + } + if event_type.starts_with("account/") { + return protocol_account_detail(payload_value); + } + if event_type == "error" { + return payload_value + .and_then(|value| { + string_at_paths( + value, + &[&["params", "error", "codexErrorInfo"], &["error", "codexErrorInfo"]], + ) + }) + .or_else(|| Some(String::from("error"))); + } + + None +} + +fn protocol_account_detail(payload_value: Option<&Value>) -> Option { + let value = payload_value?; + let plan = string_at_paths( + value, + &[ + &["params", "planType"], + &["params", "chatgptPlanType"], + &["planType"], + &["chatgptPlanType"], + ], + ); + let status = string_at_paths( + value, + &[&["params", "status"], &["params", "refreshStatus"], &["status"], &["refreshStatus"]], + ); + + match (plan, status) { + (Some(plan), Some(status)) => Some(format!("{plan}/{status}")), + (Some(plan), None) => Some(plan), + (None, Some(status)) => Some(status), + (None, None) => None, + } +} + +fn protocol_turn_status_from_payload(event_type: &str, payload: &str) -> Option { + let payload_value = serde_json::from_str::(payload).ok(); + + protocol_turn_status_from_value(event_type, payload_value.as_ref()) +} + +fn protocol_turn_status_from_value( + event_type: &str, + payload_value: Option<&Value>, +) -> Option { + match event_type { + "turn/started" => Some(String::from("running")), + "turn/completed" => payload_value + .and_then(|value| { + string_at_paths(value, &[&["params", "turn", "status"], &["turn", "status"]]) + }) + .or_else(|| Some(String::from("completed"))), + _ => None, + } +} + +fn protocol_waiting_reason( + event_type: &str, + payload: &str, + child_activity: &state::ChildAgentActivitySummary, +) -> Option { + let payload_value = serde_json::from_str::(payload).ok(); + + if event_type == "thread/status/changed" + && let Some(reason) = thread_status_waiting_reason(payload_value.as_ref()) + { + return Some(reason); + } + if interactive_flag_for_request(event_type).is_some() { + return Some(String::from("approval_or_user_input")); + } + if event_type == "item/tool/call" { + return Some(String::from("tool_execution")); + } + if protocol_activity_category(event_type) == "command_output" { + return Some(String::from("tool_execution")); + } + if matches!(event_type, "item/tool/call/response" | "item/completed" | "turn/started") + || event_type.ends_with("/delta") + || event_type.ends_with("/response") + { + return Some(String::from("model_execution")); + } + if event_type == "turn/completed" { + return Some(String::from("turn_completed")); + } + + if let Some(current_bucket) = child_activity.current_bucket.as_deref() { + return Some(match current_bucket { + CHILD_BUCKET_MODEL => String::from("model_execution"), + CHILD_BUCKET_PROTOCOL => String::from("protocol_activity"), + _ => String::from("tool_execution"), + }); + } + + None +} + +fn thread_status_waiting_reason(payload_value: Option<&Value>) -> Option { + let value = payload_value?; + let flags = + value_at_paths(value, &[&["params", "status", "activeFlags"], &["status", "activeFlags"]])?; + let flags = flags.as_array()?; + + if flags + .iter() + .filter_map(Value::as_str) + .any(|flag| matches!(flag, "waitingOnApproval" | "waitingOnUserInput")) + { + return Some(String::from("approval_or_user_input")); + } + + None +} + +fn protocol_rate_limit_status(event_type: &str, payload: &str) -> Option { + let payload_value = serde_json::from_str::(payload).ok()?; + + find_string_field(&payload_value, &["rateLimitReachedType", "rate_limit_reached_type"]) + .or_else(|| { + find_string_field(&payload_value, &["codexErrorInfo", "codex_error_info"]) + .filter(|value| value.to_ascii_lowercase().contains("limit")) + }) + .or_else(|| { + event_type.to_ascii_lowercase().contains("ratelimit").then(|| event_type.to_owned()) + }) +} + +fn child_tool_call_event( + payload_value: Option<&Value>, + input_tokens: Option, + output_tokens: Option, +) -> ChildActivityEvent { + let tool_name = + payload_value.and_then(extract_tool_name).unwrap_or_else(|| String::from("tool")); + let arguments = payload_value.and_then(extract_tool_arguments); + let (bucket, detail) = child_tool_bucket(&tool_name, arguments.as_ref()); + + ChildActivityEvent { + event_bucket: bucket.clone(), + event_detail: Some(detail.clone()), + transition_bucket: Some(bucket), + transition_detail: Some(detail), + tool_name: Some(tool_name), + tool_call: true, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: false, + } +} + +fn child_tool_response_event( + payload_value: Option<&Value>, + active_tool_name: Option<&str>, + payload: &str, +) -> ChildActivityEvent { + let tool_name = active_tool_name.unwrap_or("tool").to_owned(); + let (bucket, detail) = child_tool_bucket(&tool_name, None); + let output_bytes = tool_output_size(payload_value, payload); + + ChildActivityEvent { + event_bucket: bucket, + event_detail: Some(detail), + transition_bucket: Some(CHILD_BUCKET_MODEL.to_owned()), + transition_detail: Some(String::from("waiting after tool output")), + tool_name: Some(tool_name), + tool_call: false, + tool_output_bytes: Some(output_bytes), + input_tokens: payload_value.and_then(|value| find_numeric_field(value, INPUT_TOKEN_KEYS)), + output_tokens: payload_value.and_then(|value| find_numeric_field(value, OUTPUT_TOKEN_KEYS)), + completed: false, + } +} + +fn child_item_completed_event(payload_value: Option<&Value>, payload: &str) -> ChildActivityEvent { + let item_kind = payload_value + .and_then(|value| string_at_paths(value, &[&["params", "item", "type"], &["item", "type"]])) + .unwrap_or_else(|| String::from("item")); + let tool_name = payload_value.and_then(extract_tool_name); + let input_tokens = payload_value.and_then(|value| find_numeric_field(value, INPUT_TOKEN_KEYS)); + let output_tokens = + payload_value.and_then(|value| find_numeric_field(value, OUTPUT_TOKEN_KEYS)); + + if let Some(tool_name) = tool_name + && item_kind != "agentMessage" + { + let (bucket, detail) = child_tool_bucket(&tool_name, None); + + return ChildActivityEvent { + event_bucket: bucket, + event_detail: Some(detail), + transition_bucket: Some(CHILD_BUCKET_MODEL.to_owned()), + transition_detail: Some(String::from("waiting after completed item")), + tool_name: Some(tool_name), + tool_call: false, + tool_output_bytes: Some(tool_output_size(payload_value, payload)), + input_tokens, + output_tokens, + completed: false, + }; + } + + ChildActivityEvent { + event_bucket: CHILD_BUCKET_MODEL.to_owned(), + event_detail: Some(item_kind), + transition_bucket: Some(CHILD_BUCKET_MODEL.to_owned()), + transition_detail: Some(String::from("model output")), + tool_name: None, + tool_call: false, + tool_output_bytes: None, + input_tokens, + output_tokens, + completed: false, + } +} + +fn child_tool_bucket(tool_name: &str, arguments: Option<&Value>) -> (String, String) { + let normalized_tool = tool_name.to_ascii_lowercase(); + + if is_tracker_tool_name(&normalized_tool) { + return (CHILD_BUCKET_TRACKER.to_owned(), tool_name.to_owned()); + } + if normalized_tool.contains("view_image") + || normalized_tool.contains("screenshot") + || normalized_tool.contains("image_query") + || normalized_tool.contains("browser") + { + return (CHILD_BUCKET_BROWSER_IMAGE.to_owned(), tool_name.to_owned()); + } + if normalized_tool.contains("exec_command") { + let command_category = arguments + .and_then(extract_command_text) + .map(|command| shell_command_category(&command)) + .unwrap_or_else(|| String::from("shell")); + + if command_category == "pr_land" { + return (CHILD_BUCKET_PR_LAND.to_owned(), String::from("exec_command: pr_land")); + } + + return (CHILD_BUCKET_SHELL.to_owned(), format!("exec_command: {command_category}")); + } + + (CHILD_BUCKET_TOOL.to_owned(), tool_name.to_owned()) +} + +fn is_tracker_tool_name(normalized_tool: &str) -> bool { + matches!( + normalized_tool, + "issue_transition" + | "issue_comment" + | "issue_progress_checkpoint" + | "issue_review_checkpoint" + | "issue_review_handoff" + | "issue_review_repair_complete" + | "issue_delivery_closeout_complete" + | "issue_terminal_finalize" + | "issue_label_add" + ) || normalized_tool.ends_with(".issue_transition") + || normalized_tool.ends_with(".issue_comment") + || normalized_tool.ends_with(".issue_progress_checkpoint") + || normalized_tool.ends_with(".issue_review_checkpoint") + || normalized_tool.ends_with(".issue_review_handoff") + || normalized_tool.ends_with(".issue_review_repair_complete") + || normalized_tool.ends_with(".issue_delivery_closeout_complete") + || normalized_tool.ends_with(".issue_terminal_finalize") + || normalized_tool.ends_with(".issue_label_add") +} + +fn shell_command_category(command: &str) -> String { + let trimmed = command.trim(); + let lowered = trimmed.to_ascii_lowercase(); + + if lowered.starts_with("git push") + || lowered.starts_with("gh pr") + || lowered.contains(" gh pr ") + || lowered.contains("decodex land") + || lowered.contains("issue_terminal_finalize") + { + return String::from("pr_land"); + } + if lowered.starts_with("cargo make") + || lowered.starts_with("cargo test") + || lowered.starts_with("npm run check") + || lowered.contains(" nextest ") + { + return String::from("checks"); + } + if lowered.starts_with("git ") { + return String::from("git"); + } + if lowered.starts_with("gh ") { + return String::from("gh"); + } + if lowered.contains("vite") || lowered.contains("dev server") || lowered.contains("localhost") { + return String::from("dev_server"); + } + if lowered.contains("playwright") || lowered.contains("browser") { + return String::from("browser_smoke"); + } + + String::from("shell") +} + +fn child_activity_bucket_mut<'a>( + summary: &'a mut state::ChildAgentActivitySummary, + name: &str, +) -> &'a mut state::ChildAgentActivityBucket { + if let Some(index) = summary.buckets.iter().position(|bucket| bucket.name == name) { + return &mut summary.buckets[index]; + } + + summary.buckets.push(state::ChildAgentActivityBucket { + name: name.to_owned(), + ..state::ChildAgentActivityBucket::default() + }); + + let last_index = summary.buckets.len().saturating_sub(1); + + &mut summary.buckets[last_index] +} + +fn extract_tool_name(value: &Value) -> Option { + let tool = string_at_paths( + value, + &[ + &["params", "tool"], + &["params", "name"], + &["params", "item", "tool"], + &["params", "item", "name"], + &["tool"], + &["name"], + &["item", "tool"], + &["item", "name"], + ], + )?; + let namespace = string_at_paths(value, &[&["params", "namespace"], &["namespace"]]); + + Some(match namespace { + Some(namespace) if !namespace.is_empty() => format!("{namespace}.{tool}"), + _ => tool, + }) +} + +fn extract_tool_arguments(value: &Value) -> Option { + let arguments = value_at_paths( + value, + &[ + &["params", "arguments"], + &["params", "item", "arguments"], + &["arguments"], + &["item", "arguments"], + ], + )?; + + if let Some(arguments_text) = arguments.as_str() + && let Ok(parsed_arguments) = serde_json::from_str::(arguments_text) + { + return Some(parsed_arguments); + } + + Some(arguments.clone()) +} + +fn extract_command_text(arguments: &Value) -> Option { + string_at_paths(arguments, &[&["cmd"], &["command"], &["argv", "0"]]) +} + +fn string_at_paths(value: &Value, paths: &[&[&str]]) -> Option { + paths + .iter() + .find_map(|path| value_at_path(value, path).and_then(Value::as_str).map(str::to_owned)) +} + +fn value_at_paths<'a>(value: &'a Value, paths: &[&[&str]]) -> Option<&'a Value> { + paths.iter().find_map(|path| value_at_path(value, path)) +} + +fn value_at_path<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { + let mut current = value; + + for part in path { + current = current.get(*part)?; + } + + Some(current) +} + +fn tool_output_size(value: Option<&Value>, payload: &str) -> i64 { + let largest_string = value.map(largest_string_len).unwrap_or(0); + let payload_len = i64::try_from(payload.len()).unwrap_or(i64::MAX); + + largest_string.max(payload_len) +} + +fn largest_string_len(value: &Value) -> i64 { + match value { + Value::String(text) => i64::try_from(text.len()).unwrap_or(i64::MAX), + Value::Array(items) => items.iter().map(largest_string_len).max().unwrap_or(0), + Value::Object(entries) => entries.values().map(largest_string_len).max().unwrap_or(0), + _ => 0, + } +} + +fn find_numeric_field(value: &Value, keys: &[&str]) -> Option { + match value { + Value::Object(entries) => { + for (key, nested) in entries { + if keys.iter().any(|candidate| *candidate == key) + && let Some(number) = json_number_to_i64(nested) + { + return Some(number); + } + } + + entries.values().find_map(|nested| find_numeric_field(nested, keys)) + }, + Value::Array(items) => items.iter().find_map(|nested| find_numeric_field(nested, keys)), + _ => None, + } +} + +fn find_string_field(value: &Value, keys: &[&str]) -> Option { + match value { + Value::Object(entries) => { + for (key, nested) in entries { + if keys.iter().any(|candidate| *candidate == key) + && let Some(text) = nested.as_str() + { + return Some(text.to_owned()); + } + } + + entries.values().find_map(|nested| find_string_field(nested, keys)) + }, + Value::Array(items) => items.iter().find_map(|nested| find_string_field(nested, keys)), + _ => None, + } +} + +fn json_number_to_i64(value: &Value) -> Option { + value.as_i64().or_else(|| value.as_u64().and_then(|number| i64::try_from(number).ok())) +} + +fn redact_identifier(identifier: &str) -> String { + let tail = + identifier.chars().rev().take(6).collect::>().into_iter().rev().collect::(); + + if tail.is_empty() { String::from("unknown") } else { format!("...{tail}") } +} + +fn duration_seconds_i64(duration: Duration) -> i64 { + i64::try_from(duration.as_secs()).unwrap_or(i64::MAX) +} + +fn write_activity_marker_best_effort(marker_path: &Path, run_id: &str, attempt_number: i64) { + if let Err(error) = state::write_run_operation_marker( + marker_path, + run_id, + attempt_number, + RUN_OPERATION_AGENT_RUN, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree activity marker." + ); + } +} + +fn write_activity_marker_best_effort_for_request(request: &AppServerRunRequest<'_>) { + if let Some(marker_path) = request.activity_marker_path.as_ref() { + write_activity_marker_best_effort(marker_path, &request.run_id, request.attempt_number); + } +} + +fn write_capability_preflight_marker_best_effort(request: &AppServerRunRequest<'_>) { + if let Some(marker_path) = request.activity_marker_path.as_ref() + && let Err(error) = state::write_run_operation_marker( + marker_path, + &request.run_id, + request.attempt_number, + RUN_OPERATION_APP_SERVER_PREFLIGHT, + ) { + tracing::warn!( + ?error, + run_id = request.run_id, + attempt_number = request.attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree app-server preflight marker." + ); + } +} + +fn write_protocol_activity_marker_best_effort( + marker_path: &Path, + activity: &state::ProtocolActivityMarker<'_>, +) { + if let Err(error) = state::write_run_protocol_activity_marker(marker_path, activity) { + tracing::warn!( + ?error, + run_id = activity.run_id, + attempt_number = activity.attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree protocol-activity marker." + ); + } +} + +fn write_turn_marker_best_effort( + marker_path: &Path, + run_id: &str, + attempt_number: i64, + turn_id: &str, +) { + if let Err(error) = state::write_run_turn_marker(marker_path, run_id, attempt_number, turn_id) { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree turn marker." + ); + } +} + +fn write_thread_status_marker_best_effort( + marker_path: &Path, + run_id: &str, + attempt_number: i64, + thread_id: Option<&str>, + turn_id: Option<&str>, + thread_status: &str, + thread_active_flags: &[String], +) { + if let Err(error) = state::write_run_thread_status_marker( + marker_path, + run_id, + attempt_number, + thread_id, + turn_id, + thread_status, + thread_active_flags, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree thread-status marker." + ); + } +} + +fn write_effective_runtime_marker_best_effort( + marker_path: &Path, + run_id: &str, + attempt_number: i64, + thread_id: Option<&str>, + turn_id: Option<&str>, + runtime: &EffectiveThreadConfig, +) { + if let Err(error) = state::write_run_effective_runtime_marker( + marker_path, + run_id, + attempt_number, + &EffectiveRuntimeMarker { + thread_id, + turn_id, + effective_model: &runtime.model, + effective_model_provider: &runtime.model_provider, + effective_cwd: &runtime.cwd, + effective_approval_policy: &runtime.approval_policy, + effective_approvals_reviewer: &runtime.approvals_reviewer, + effective_sandbox_mode: &runtime.sandbox_mode, + }, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree effective-runtime marker." + ); + } +} + +fn write_codex_account_marker_best_effort( + marker_path: &Path, + run_id: &str, + attempt_number: i64, + summary: &CodexAccountActivitySummary, + account_summaries: &[CodexAccountActivitySummary], +) { + if let Err(error) = state::write_run_account_marker( + marker_path, + &CodexAccountMarker { + run_id, + attempt_number, + account: summary, + accounts: account_summaries, + }, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree Codex account marker." + ); + } +} + +fn write_thread_marker_best_effort( + marker_path: &Path, + run_id: &str, + attempt_number: i64, + thread_id: &str, +) { + if let Err(error) = + state::write_run_thread_marker(marker_path, run_id, attempt_number, thread_id) + { + tracing::warn!( + ?error, + run_id, + attempt_number, + marker_path = %marker_path.display(), + "Failed to update worktree thread marker." + ); + } +} + +fn execute_app_server_run_inner( + request: &AppServerRunRequest<'_>, + state_store: &StateStore, +) -> crate::prelude::Result { + let mut recorder = RunRecorder::new( + state_store, + &request.run_id, + request.attempt_number, + request.activity_marker_path.as_ref(), + ); + let expected_codex_home = request.process_env.resolve_codex_home_env()?; + let mut client = AppServerClient::spawn(&request.listen, &request.process_env)?; + let initialize_response = initialize_client_for_run( + &mut client, + &mut recorder, + request.dynamic_tool_handler, + &expected_codex_home, + )?; + + client.mark_initialized()?; + + write_capability_preflight_marker_best_effort(request); + run_app_server_capability_preflight(&mut client, &mut recorder, &request.cwd)?; + write_activity_marker_best_effort_for_request(request); + + if let Some(health_check) = request.command_exec_health_check.as_ref() { + run_command_exec_health_check(&mut client, &mut recorder, request, health_check)?; + } + + flush_pending_messages(&mut client, &mut recorder, None)?; + login_codex_account_for_run(&mut client, &mut recorder, request)?; + flush_pending_messages(&mut client, &mut recorder, None)?; + + let thread_response = start_or_resume_thread_session(&mut client, &mut recorder, request)?; + let thread_id = thread_response.thread.id.clone(); + let effective_thread_config = thread_response.effective_config(); + + record_thread_session_start( + state_store, + request, + &mut recorder, + &thread_id, + &effective_thread_config, + )?; + flush_pending_messages(&mut client, &mut recorder, Some(&thread_id))?; + + state_store.record_run_attempt( + &request.run_id, + &request.issue_id, + request.attempt_number, + "running", + )?; + recorder.mark_activity()?; + + let turn_result = + execute_turn_loop(&mut client, &mut recorder, request, state_store, &thread_id)?; + + state_store.record_run_attempt( + &request.run_id, + &request.issue_id, + request.attempt_number, + "succeeded", + )?; + recorder.mark_activity()?; + + Ok(AppServerRunResult { + user_agent: initialize_response.user_agent, + thread_id, + turn_id: turn_result.turn_id, + turn_count: turn_result.turn_count, + event_count: state_store.event_count(&request.run_id)?, + final_output: turn_result.final_output, + continuation_pending: turn_result.continuation_pending, + }) +} + +fn initialize_client_for_run( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + expected_codex_home: &ResolvedAppServerCodexHomeEnv, +) -> crate::prelude::Result { + let response = client.initialize_with_handler( + dynamic_tool_handler.is_some(), + |connection, wire_message, server_request| { + handle_server_request_while_waiting( + connection, + recorder, + wire_message, + server_request, + RequestDispatchContext::new( + RequestWaitPhase::Initialize, + dynamic_tool_handler, + None, + None, + None, + ), + ) + }, + )?; + + validate_initialize_codex_home(expected_codex_home, &response)?; + + Ok(response) +} + +fn run_app_server_capability_preflight( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + cwd: &str, +) -> crate::prelude::Result { + let mut report = AppServerCapabilityPreflightReport::new(); + let config = preflight_request(recorder, &report, "config/read", || { + client.read_config(&ConfigReadParams { cwd: Some(cwd.to_owned()), include_layers: false }) + })?; + + record_config_preflight(&mut report, &config.config); + + let models = list_all_models_for_preflight(client, recorder, &report)?; + + record_model_preflight(&mut report, &config.config, &models); + + let provider_capabilities = + preflight_request(recorder, &report, "modelProvider/capabilities/read", || { + client.read_model_provider_capabilities() + })?; + + record_model_provider_preflight(&mut report, &provider_capabilities); + + let skills = preflight_request(recorder, &report, "skills/list", || { + client.list_skills(&SkillsListParams { + cwds: vec![cwd.to_owned()], + force_reload: false, + per_cwd_extra_user_roots: None, + }) + })?; + + record_skills_preflight(&mut report, cwd, &skills); + + let plugins = preflight_request(recorder, &report, "plugin/list", || { + client.list_plugins(&PluginListParams { cwds: Some(vec![cwd.to_owned()]) }) + })?; + + record_plugin_preflight(&mut report, &plugins); + + let mcp_servers = list_all_mcp_servers_for_preflight(client, recorder, &report)?; + + record_mcp_preflight(&mut report, &mcp_servers); + record_app_server_preflight_report(recorder, &report)?; + + if report.has_blockers() { + return Err(Report::new(AppServerCapabilityPreflightFailure::blocked(report))); + } + + Ok(report) +} + +fn preflight_request( + recorder: &mut RunRecorder<'_>, + report: &AppServerCapabilityPreflightReport, + method: &'static str, + request: F, +) -> crate::prelude::Result +where + F: FnOnce() -> crate::prelude::Result, +{ + match request() { + Ok(response) => Ok(response), + Err(error) => { + let mut failed_report = report.clone(); + let mut details = BTreeMap::new(); + + details.insert(String::from("method"), method.to_owned()); + details.insert(String::from("error"), error.to_string()); + failed_report.push_blocked( + check_name_for_method(method), + format!("`{method}` failed before thread/start."), + details, + ); + + record_app_server_preflight_report(recorder, &failed_report)?; + + Err(Report::new(AppServerCapabilityPreflightFailure::method_failed( + method, + error.to_string(), + failed_report, + ))) + }, + } +} + +fn list_all_models_for_preflight( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + report: &AppServerCapabilityPreflightReport, +) -> crate::prelude::Result> { + let mut cursor = None; + let mut models = Vec::new(); + + loop { + let response: ModelListResponse = + preflight_request(recorder, report, "model/list", || { + client.list_models(&ModelListParams { + cursor: cursor.clone(), + include_hidden: Some(true), + limit: Some(PREFLIGHT_MODEL_PAGE_LIMIT), + }) + })?; + + models.extend(response.data); + + let Some(next_cursor) = response.next_cursor else { + return Ok(models); + }; + + cursor = Some(next_cursor); + } +} + +fn list_all_mcp_servers_for_preflight( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + report: &AppServerCapabilityPreflightReport, +) -> crate::prelude::Result> { + let mut cursor = None; + let mut servers = Vec::new(); + + loop { + let response: ListMcpServerStatusResponse = + preflight_request(recorder, report, "mcpServerStatus/list", || { + client.list_mcp_server_status(&ListMcpServerStatusParams { + cursor: cursor.clone(), + detail: Some(PREFLIGHT_MCP_DETAIL.to_owned()), + limit: Some(PREFLIGHT_MCP_PAGE_LIMIT), + }) + })?; + + servers.extend(response.data); + + let Some(next_cursor) = response.next_cursor else { + return Ok(servers); + }; + + cursor = Some(next_cursor); + } +} + +fn record_config_preflight( + report: &mut AppServerCapabilityPreflightReport, + config: &RuntimeConfigSummary, +) { + let mut details = BTreeMap::new(); + + insert_optional_detail(&mut details, "model", config.model.as_deref()); + insert_optional_detail(&mut details, "model_provider", config.model_provider.as_deref()); + + if let Some(approval_policy) = config.approval_policy.as_ref().and_then(config_value_name) { + details.insert(String::from("approval_policy"), approval_policy); + } + if let Some(sandbox_mode) = config.sandbox_mode.as_ref().and_then(config_value_name) { + details.insert(String::from("sandbox_mode"), sandbox_mode); + } + + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + details, + ); +} + +fn record_model_preflight( + report: &mut AppServerCapabilityPreflightReport, + config: &RuntimeConfigSummary, + models: &[ModelSummary], +) { + let configured_model = config.model.as_deref().filter(|model| !model.trim().is_empty()); + let default_model = models.iter().find(|model| model.is_default); + let matching_config_model = configured_model + .and_then(|configured| models.iter().find(|model| model_matches_config(model, configured))); + let mut details = BTreeMap::new(); + + details.insert(String::from("model_count"), models.len().to_string()); + + if let Some(configured_model) = configured_model { + details.insert(String::from("configured_model"), configured_model.to_owned()); + } + if let Some(model) = default_model { + details.insert(String::from("default_model"), model.model.clone()); + } + if let Some(model) = matching_config_model { + details.insert(String::from("matched_model_id"), model.id.clone()); + } + + if models.is_empty() { + report.push_blocked( + PREFLIGHT_CHECK_MODEL, + "model/list returned no available models.", + details, + ); + } else if configured_model.is_some() && matching_config_model.is_none() { + report.push_blocked( + PREFLIGHT_CHECK_MODEL, + "configured model was not present in model/list.", + details, + ); + } else if configured_model.is_none() && default_model.is_none() { + report.push_blocked( + PREFLIGHT_CHECK_MODEL, + "no configured model or default model was present.", + details, + ); + } else { + report.push_ok( + PREFLIGHT_CHECK_MODEL, + "model/list returned an executable model selection.", + details, + ); + } +} + +fn record_model_provider_preflight( + report: &mut AppServerCapabilityPreflightReport, + capabilities: &ModelProviderCapabilitiesReadResponse, +) { + let mut details = BTreeMap::new(); + + details.insert(String::from("web_search"), capabilities.web_search.to_string()); + details.insert(String::from("image_generation"), capabilities.image_generation.to_string()); + details.insert(String::from("namespace_tools"), capabilities.namespace_tools.to_string()); + report.push_ok( + PREFLIGHT_CHECK_MODEL_PROVIDER, + "modelProvider/capabilities/read returned provider capabilities.", + details, + ); +} + +fn record_skills_preflight( + report: &mut AppServerCapabilityPreflightReport, + cwd: &str, + skills: &SkillsListResponse, +) { + let cwd_entry = skills.data.iter().find(|entry| entry.cwd == cwd); + let all_skill_count: usize = skills.data.iter().map(|entry| entry.skills.len()).sum(); + let enabled_skill_count: usize = skills + .data + .iter() + .flat_map(|entry| entry.skills.iter()) + .filter(|skill| skill.enabled) + .count(); + let errors = skills.data.iter().flat_map(|entry| entry.errors.iter()).collect::>(); + let mut details = BTreeMap::new(); + + details.insert(String::from("cwd"), cwd.to_owned()); + details.insert(String::from("entry_count"), skills.data.len().to_string()); + details.insert(String::from("skill_count"), all_skill_count.to_string()); + details.insert(String::from("enabled_skill_count"), enabled_skill_count.to_string()); + + if let Some(first_error) = errors.first() { + details.insert(String::from("first_error_path"), first_error.path.clone()); + details.insert(String::from("first_error"), first_error.message.clone()); + } + + if cwd_entry.is_none() { + report.push_blocked( + PREFLIGHT_CHECK_SKILLS, + "skills/list did not return an entry for the run cwd.", + details, + ); + } else if !errors.is_empty() { + report.push_blocked( + PREFLIGHT_CHECK_SKILLS, + "skills/list returned skill scan errors.", + details, + ); + } else if enabled_skill_count == 0 { + report.push_blocked( + PREFLIGHT_CHECK_SKILLS, + "skills/list returned no enabled skills.", + details, + ); + } else { + report.push_ok(PREFLIGHT_CHECK_SKILLS, "skills/list returned enabled skills.", details); + } +} + +fn record_plugin_preflight( + report: &mut AppServerCapabilityPreflightReport, + plugins: &PluginListResponse, +) { + let plugin_count: usize = + plugins.marketplaces.iter().map(|marketplace| marketplace.plugins.len()).sum(); + let installed_count = plugins + .marketplaces + .iter() + .flat_map(|marketplace| marketplace.plugins.iter()) + .filter(|plugin| plugin.installed) + .count(); + let enabled_count = plugins + .marketplaces + .iter() + .flat_map(|marketplace| marketplace.plugins.iter()) + .filter(|plugin| plugin.enabled) + .count(); + let mut details = BTreeMap::new(); + + details.insert(String::from("marketplace_count"), plugins.marketplaces.len().to_string()); + details.insert(String::from("plugin_count"), plugin_count.to_string()); + details.insert(String::from("installed_plugin_count"), installed_count.to_string()); + details.insert(String::from("enabled_plugin_count"), enabled_count.to_string()); + + if let Some(first_error) = plugins.marketplace_load_errors.first() { + details.insert(String::from("first_error_path"), first_error.marketplace_path.clone()); + details.insert(String::from("first_error"), first_error.message.clone()); + } + + if !plugins.marketplace_load_errors.is_empty() { + report.push_blocked( + PREFLIGHT_CHECK_PLUGINS, + "plugin/list returned marketplace load errors.", + details, + ); + } else if plugins.marketplaces.is_empty() { + report.push_blocked( + PREFLIGHT_CHECK_PLUGINS, + "plugin/list returned no marketplaces.", + details, + ); + } else { + report.push_ok(PREFLIGHT_CHECK_PLUGINS, "plugin/list returned plugin inventory.", details); + } +} + +fn record_mcp_preflight( + report: &mut AppServerCapabilityPreflightReport, + servers: &[McpServerStatusSummary], +) { + let not_logged_in = servers + .iter() + .filter(|server| server.auth_status == "notLoggedIn") + .map(|server| server.name.clone()) + .collect::>(); + let tool_count: usize = servers.iter().map(|server| server.tools.len()).sum(); + let mut details = BTreeMap::new(); + + details.insert(String::from("server_count"), servers.len().to_string()); + details.insert(String::from("tool_count"), tool_count.to_string()); + + if !not_logged_in.is_empty() { + details.insert(String::from("not_logged_in_servers"), not_logged_in.join(", ")); + } + if !not_logged_in.is_empty() { + report.push_blocked( + PREFLIGHT_CHECK_MCP, + "mcpServerStatus/list returned MCP servers that are not logged in.", + details, + ); + } else { + report.push_ok( + PREFLIGHT_CHECK_MCP, + "mcpServerStatus/list returned MCP server state.", + details, + ); + } +} + +fn record_app_server_preflight_report( + recorder: &mut RunRecorder<'_>, + report: &AppServerCapabilityPreflightReport, +) -> crate::prelude::Result<()> { + recorder.record(PREFLIGHT_EVENT_TYPE, &serde_json::to_string(report)?) +} + +fn model_matches_config(model: &ModelSummary, configured_model: &str) -> bool { + model.model == configured_model || model.id == configured_model +} + +fn insert_optional_detail(details: &mut BTreeMap, name: &str, value: Option<&str>) { + if let Some(value) = value.filter(|value| !value.is_empty()) { + details.insert(name.to_owned(), value.to_owned()); + } +} + +fn config_value_name(value: &Value) -> Option { + match value { + Value::String(value) if !value.is_empty() => Some(value.clone()), + Value::Object(object) => object + .get("type") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| (object.len() == 1).then(|| object.keys().next().cloned()).flatten()), + _ => None, + } +} + +fn check_name_for_method(method: &str) -> &'static str { + match method { + "config/read" => PREFLIGHT_CHECK_CONFIG, + "model/list" => PREFLIGHT_CHECK_MODEL, + "modelProvider/capabilities/read" => PREFLIGHT_CHECK_MODEL_PROVIDER, + "skills/list" => PREFLIGHT_CHECK_SKILLS, + "plugin/list" => PREFLIGHT_CHECK_PLUGINS, + "mcpServerStatus/list" => PREFLIGHT_CHECK_MCP, + _ => "introspection", + } +} + +fn run_command_exec_health_check( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, + health_check: &CommandExecHealthCheck, +) -> crate::prelude::Result<()> { + let params = build_command_exec_health_check_params(health_check, &request.cwd); + let response = client.command_exec(¶ms)?; + + flush_pending_messages(client, recorder, None)?; + + validate_command_exec_health_check_result(health_check, &response) +} + +fn build_command_exec_health_check_params( + health_check: &CommandExecHealthCheck, + cwd: &str, +) -> CommandExecParams { + CommandExecParams { + command: health_check.command.clone(), + cwd: Some(cwd.to_owned()), + timeout_ms: Some(health_check.timeout_ms), + output_bytes_cap: Some(health_check.output_bytes_cap), + } +} + +fn validate_command_exec_health_check_result( + health_check: &CommandExecHealthCheck, + response: &CommandExecResponse, +) -> crate::prelude::Result<()> { + if response.exit_code != 0 { + eyre::bail!( + "`command/exec` health check failed with exit code {}. stdout: {:?}; stderr: {:?}", + response.exit_code, + response.stdout, + response.stderr + ); + } + if response.stdout != health_check.expected_stdout { + eyre::bail!( + "`command/exec` health check returned stdout {:?}, expected {:?}. stderr: {:?}", + response.stdout, + health_check.expected_stdout, + response.stderr + ); + } + if !response.stderr.is_empty() { + eyre::bail!("`command/exec` health check wrote unexpected stderr: {:?}", response.stderr); + } + + Ok(()) +} + +fn login_codex_account_for_run( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, +) -> crate::prelude::Result<()> { + let Some(account_provider) = request.codex_account_provider else { + return Ok(()); + }; + let account = account_provider.select_account()?; + + recorder.set_codex_account(account.summary(), account.account_summaries())?; + + record_codex_account_login(recorder, account.summary())?; + + let response = client.login_account_with_handler( + login_account_params(&account), + |connection, wire_message, server_request| { + handle_server_request_while_waiting( + connection, + recorder, + wire_message, + server_request, + RequestDispatchContext::new( + RequestWaitPhase::AccountLogin, + request.dynamic_tool_handler, + request.codex_account_provider, + None, + None, + ), + ) + }, + )?; + + match response { + LoginAccountResponse::ChatgptAuthTokens {} => { + recorder.record( + "account/login/start/response", + &serde_json::json!({ + "type": "chatgptAuthTokens", + "accountFingerprint": account.summary().account_fingerprint.as_str(), + "planType": account.summary().plan_type.as_deref(), + }) + .to_string(), + )?; + }, + } + + Ok(()) +} + +fn start_or_resume_thread_session( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, +) -> crate::prelude::Result { + if let Some(resume_thread_id) = request.resume_thread_id.as_deref() { + return resume_existing_thread_session(client, recorder, request, resume_thread_id); + } + + start_fresh_thread_session(client, recorder, request) +} + +fn start_fresh_thread_session( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, +) -> crate::prelude::Result { + let thread_start_request = build_thread_start_request(request)?; + + client.start_thread_with_handler( + thread_start_request, + |connection, wire_message, server_request| { + handle_server_request_while_waiting( + connection, + recorder, + wire_message, + server_request, + RequestDispatchContext::new( + RequestWaitPhase::ThreadStart, + request.dynamic_tool_handler, + request.codex_account_provider, + None, + None, + ), + ) + }, + ) +} + +fn resume_existing_thread_session( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, + resume_thread_id: &str, +) -> crate::prelude::Result { + match client.resume_thread_with_handler( + build_thread_resume_request(resume_thread_id, request), + |connection, wire_message, server_request| { + handle_server_request_while_waiting( + connection, + recorder, + wire_message, + server_request, + RequestDispatchContext::new( + RequestWaitPhase::ThreadResume, + request.dynamic_tool_handler, + request.codex_account_provider, + Some(resume_thread_id), + None, + ), + ) + }, + ) { + Ok(response) => Ok(response), + Err(error) if thread_resume_error_allows_fallback(&error) => { + recorder.record( + "thread/resume/miss", + &serde_json::json!({ + "requestedThreadId": resume_thread_id, + "error": error.to_string(), + }) + .to_string(), + )?; + + start_fresh_thread_session(client, recorder, request) + }, + Err(error) => Err(error), + } +} + +fn record_thread_session_start( + state_store: &StateStore, + request: &AppServerRunRequest<'_>, + recorder: &mut RunRecorder<'_>, + thread_id: &str, + effective_thread_config: &EffectiveThreadConfig, +) -> crate::prelude::Result<()> { + state_store.update_run_thread(&request.run_id, thread_id)?; + recorder.set_thread_id(thread_id)?; + recorder.set_effective_runtime(effective_thread_config)?; + + validate_effective_thread_config(&request.cwd, effective_thread_config)?; + + recorder.mark_activity() +} + +fn execute_turn_loop( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &AppServerRunRequest<'_>, + state_store: &StateStore, + thread_id: &str, +) -> crate::prelude::Result { + let mut next_input = request.user_input.clone(); + let mut turn_count = 0_u32; + + loop { + let turn_id = start_turn_for_run( + client, + recorder, + request.dynamic_tool_handler, + request.codex_account_provider, + thread_id, + &next_input, + )?; + + turn_count = turn_count.saturating_add(1); + + state_store.update_run_turn(&request.run_id, &turn_id)?; + recorder.set_turn_id(&turn_id)?; + + flush_pending_messages(client, recorder, Some(thread_id))?; + + let final_output = wait_for_turn_completion( + client, + recorder, + thread_id, + &turn_id, + request.timeout, + request.dynamic_tool_handler, + request.codex_account_provider, + )? + .final_output; + + if let Some(continuation_pending) = + resolve_turn_completion(request, turn_count, &final_output)? + { + return Ok(TurnLoopResult { turn_id, turn_count, final_output, continuation_pending }); + } + + next_input = + request.continuation_user_input.clone().unwrap_or_else(|| request.user_input.clone()); + } +} + +fn start_turn_for_run( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + codex_account_provider: Option<&dyn CodexAccountProvider>, + thread_id: &str, + next_input: &str, +) -> crate::prelude::Result { + let turn_response = client.start_turn_with_handler( + build_turn_start_request(thread_id, next_input), + |connection, wire_message, server_request| { + handle_server_request_while_waiting( + connection, + recorder, + wire_message, + server_request, + RequestDispatchContext::new( + RequestWaitPhase::TurnStart, + dynamic_tool_handler, + codex_account_provider, + Some(thread_id), + None, + ), + ) + }, + )?; + + Ok(turn_response.turn.id) +} + +fn resolve_turn_completion( + request: &AppServerRunRequest<'_>, + turn_count: u32, + final_output: &str, +) -> crate::prelude::Result> { + match classify_turn_completion(request.dynamic_tool_handler, final_output)? { + TurnCompletionStatus::Complete => Ok(Some(false)), + TurnCompletionStatus::Continue => { + if request.max_turns <= 1 { + reject_nonterminal_single_turn_completion( + request.dynamic_tool_handler, + final_output, + )?; + } + if turn_count >= request.max_turns { + return Ok(Some(true)); + } + if continuation_boundary_reached(request.continuation_guard, turn_count)? { + return Ok(Some(true)); + } + + Ok(None) + }, + } +} + +fn build_thread_start_request( + request: &AppServerRunRequest<'_>, +) -> crate::prelude::Result { + let dynamic_tools = + request.dynamic_tool_handler.map(validated_dynamic_tool_specs).transpose()?; + + Ok(ThreadStartRequest { + cwd: Some(request.cwd.clone()), + dynamic_tools, + developer_instructions: Some(request.developer_instructions.clone()), + ..ThreadStartRequest::default() + }) +} + +fn validated_dynamic_tool_specs( + handler: &dyn DynamicToolHandler, +) -> crate::prelude::Result> { + let tool_specs = handler.tool_specs(); + + for spec in &tool_specs { + if !tracker_tool_bridge::dynamic_tool_identifier_is_valid(&spec.name) { + return Err(Report::new(AppServerDynamicToolFailure::protocol( + Some(spec.name.clone()), + format!( + "Dynamic tool name `{}` does not match the Codex app-server identifier pattern `^[a-zA-Z0-9_-]+$`.", + spec.name + ), + ))); + } + + if let Some(namespace) = spec.namespace.as_deref() + && !tracker_tool_bridge::dynamic_tool_identifier_is_valid(namespace) + { + return Err(Report::new(AppServerDynamicToolFailure::protocol( + Some(format!("{namespace}.{}", spec.name)), + format!( + "Dynamic tool namespace `{namespace}` does not match the Codex app-server identifier pattern `^[a-zA-Z0-9_-]+$`." + ), + ))); + } + } + + Ok(tool_specs) +} + +fn build_thread_resume_request( + resume_thread_id: &str, + request: &AppServerRunRequest<'_>, +) -> ThreadResumeRequest { + ThreadResumeRequest { + thread_id: resume_thread_id.to_owned(), + cwd: Some(request.cwd.clone()), + developer_instructions: Some(request.developer_instructions.clone()), + ..ThreadResumeRequest::default() + } +} + +fn classify_turn_completion( + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + final_output: &str, +) -> crate::prelude::Result { + if let Some(dynamic_tool_handler) = dynamic_tool_handler { + return dynamic_tool_handler.classify_turn_completion(final_output); + } + + Ok(TurnCompletionStatus::Complete) +} + +fn reject_nonterminal_single_turn_completion( + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + final_output: &str, +) -> crate::prelude::Result<()> { + if let Some(dynamic_tool_handler) = dynamic_tool_handler { + dynamic_tool_handler.validate_turn_completion(final_output)?; + } + + eyre::bail!( + "Turn completed without a terminal completion path while same-thread continuation is disabled." + ); +} + +fn continuation_boundary_reached( + continuation_guard: Option<&dyn TurnContinuationGuard>, + turn_count: u32, +) -> crate::prelude::Result { + let Some(continuation_guard) = continuation_guard else { + return Ok(false); + }; + + if continuation_guard.should_continue_turn(turn_count)? { + return Ok(false); + } + + continuation_guard.validate_continuation_boundary(turn_count)?; + + Ok(true) +} + +fn build_turn_start_request(thread_id: &str, user_input: &str) -> TurnStartRequest { + TurnStartRequest { + thread_id: thread_id.to_owned(), + input: vec![UserInput::Text { text: user_input.to_owned() }], + ..TurnStartRequest::default() + } +} + +fn login_account_params(account: &CodexAccountLogin) -> LoginAccountParams { + LoginAccountParams::ChatgptAuthTokens { + access_token: account.access_token().to_owned(), + chatgpt_account_id: account.account_id().to_owned(), + chatgpt_plan_type: account.plan_type().map(str::to_owned), + } +} + +fn record_codex_account_login( + recorder: &mut RunRecorder<'_>, + summary: &CodexAccountActivitySummary, +) -> crate::prelude::Result<()> { + recorder.record( + "account/login/start", + &serde_json::json!({ + "type": "chatgptAuthTokens", + "accountFingerprint": summary.account_fingerprint.as_str(), + "planType": summary.plan_type.as_deref(), + "status": summary.status.as_str(), + "refreshStatus": summary.refresh_status.as_str(), + "primaryRemainingPercent": summary.primary_remaining_percent, + "secondaryRemainingPercent": summary.secondary_remaining_percent, + "rateLimitReachedType": summary.rate_limit_reached_type.as_deref(), + }) + .to_string(), + ) +} + +fn flush_pending_messages( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + target_thread_id: Option<&str>, +) -> crate::prelude::Result<()> { + for message in client.drain_pending() { + if targets_thread(&message, target_thread_id) { + recorder.record(message_type(&message), &message.raw)?; + + apply_protocol_message_side_effects(recorder, &message)?; + } + } + + Ok(()) +} + +fn wait_for_turn_completion( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + target_thread_id: &str, + target_turn_id: &str, + timeout: Duration, + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + codex_account_provider: Option<&dyn CodexAccountProvider>, +) -> crate::prelude::Result { + let mut last_activity_at = Instant::now(); + let mut final_output = String::new(); + let mut latest_turn_failure: Option = None; + + loop { + let now = Instant::now(); + let Some(wait_timeout) = remaining_idle_budget(last_activity_at, now, timeout) else { + return Err(turn_wait_timeout_error( + target_thread_id, + target_turn_id, + latest_turn_failure, + )); + }; + let wire_message = + recv_turn_wire_message(client, wait_timeout, latest_turn_failure.as_ref())?; + + if !targets_thread(&wire_message, Some(target_thread_id)) { + tracing::debug!(raw = %wire_message.raw, "Ignoring app-server message for another thread."); + + continue; + } + + last_activity_at = Instant::now(); + + recorder.record(message_type(&wire_message), &wire_message.raw)?; + + apply_protocol_message_side_effects(recorder, &wire_message)?; + + match &wire_message.message { + JsonRpcMessage::Notification(notification) => match notification.method.as_str() { + "thread/status/changed" => { + let payload: ThreadStatusChangedNotification = + serde_json::from_value(notification.params.clone())?; + + if payload.status.kind == "systemError" && latest_turn_failure.is_none() { + latest_turn_failure = + Some(AppServerTurnFailure::from_system_error(&payload.thread_id)); + } + }, + "error" => { + if let Some((failure, will_retry)) = failure_from_error_notification( + notification, + target_thread_id, + target_turn_id, + )? { + if failure.requires_operator_attention() && will_retry != Some(true) { + return Err(Report::new(failure)); + } + + latest_turn_failure = Some(failure); + } + }, + "item/agentMessage/delta" => { + let payload: AgentMessageDeltaNotification = + serde_json::from_value(notification.params.clone())?; + + final_output.push_str(&payload.delta); + }, + "item/completed" => { + let payload: ItemCompletedNotification = + serde_json::from_value(notification.params.clone())?; + + if payload.item.kind == "agentMessage" + && let Some(text) = payload.item.text + { + final_output = text; + } + }, + "turn/completed" => { + let payload: TurnCompletedNotification = + serde_json::from_value(notification.params.clone())?; + + if payload.turn.id != target_turn_id { + continue; + } + if payload.turn.status == "completed" { + return Ok(RunOutcome { final_output }); + } + + if let Some(error) = payload.turn.error.as_ref() { + return Err(Report::new(turn_failure_from_turn_error( + target_thread_id, + Some(&payload.turn.id), + &payload.turn.status, + error, + ))); + } + if let Some(failure) = latest_turn_failure { + return Err(Report::new(failure)); + } + + eyre::bail!( + "Turn `{}` ended with status `{}` without an explicit error payload.", + payload.turn.id, + payload.turn.status + ); + }, + _ => {}, + }, + JsonRpcMessage::Request(request) => handle_server_request_during_turn_execution( + client, + recorder, + request, + RequestDispatchContext::new( + RequestWaitPhase::TurnExecution, + dynamic_tool_handler, + codex_account_provider, + Some(target_thread_id), + Some(target_turn_id), + ), + )?, + JsonRpcMessage::Response(_) | JsonRpcMessage::Error(_) => { + eyre::bail!( + "Received an unexpected JSON-RPC response while waiting for turn completion." + ); + }, + } + } +} + +fn turn_wait_timeout_error( + target_thread_id: &str, + target_turn_id: &str, + latest_turn_failure: Option, +) -> Report { + let message = format!( + "Timed out while waiting for turn `{target_turn_id}` on thread `{target_thread_id}`." + ); + + if let Some(failure) = latest_turn_failure { + return Report::new(failure).wrap_err(message); + } + + eyre::eyre!(message) +} + +fn recv_turn_wire_message( + client: &mut AppServerClient, + wait_timeout: Duration, + latest_turn_failure: Option<&AppServerTurnFailure>, +) -> crate::prelude::Result { + match client.recv(Some(wait_timeout)) { + Ok(wire_message) => Ok(wire_message), + Err(error) => { + if error.downcast_ref::().is_some() + && let Some(failure) = latest_turn_failure + { + return Err(Report::new(failure.clone()) + .wrap_err("Timed out while waiting for additional app-server output.")); + } + + Err(error) + }, + } +} + +fn failure_from_error_notification( + notification: &JsonRpcNotification, + target_thread_id: &str, + target_turn_id: &str, +) -> crate::prelude::Result)>> { + let payload: ErrorNotification = serde_json::from_value(notification.params.clone())?; + let payload_turn_matches = + payload.turn_id.as_deref().is_none_or(|turn_id| turn_id == target_turn_id); + let payload_thread_matches = + payload.thread_id.as_deref().is_none_or(|thread_id| thread_id == target_thread_id); + + if !payload_thread_matches || !payload_turn_matches { + return Ok(None); + } + + let failure = turn_failure_from_turn_error( + target_thread_id, + payload.turn_id.as_deref(), + "failed", + &payload.error, + ); + + Ok(Some((failure, payload.will_retry))) +} + +fn turn_failure_from_turn_error( + thread_id: &str, + turn_id: Option<&str>, + status: &str, + error: &TurnError, +) -> AppServerTurnFailure { + AppServerTurnFailure::new( + thread_id, + turn_id.map(str::to_owned), + status, + error.message.clone(), + error.codex_error_info.clone(), + ) +} + +fn remaining_idle_budget( + last_activity_at: Instant, + now: Instant, + timeout: Duration, +) -> Option { + timeout.checked_sub(now.saturating_duration_since(last_activity_at)) +} + +fn handle_server_request_while_waiting( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + wire_message: &WireMessage, + request: &JsonRpcRequest, + context: RequestDispatchContext<'_>, +) -> crate::prelude::Result<()> { + if targets_thread(wire_message, context.target_thread_id) { + record_wire_message_safely(recorder, wire_message)?; + record_interactive_request_state(recorder, request)?; + } else if request.method == "account/chatgptAuthTokens/refresh" { + record_codex_account_refresh_request(recorder, request)?; + } + + dispatch_server_request(connection, recorder, request, context) +} + +fn handle_server_request_during_turn_execution( + client: &mut AppServerClient, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + context: RequestDispatchContext<'_>, +) -> crate::prelude::Result<()> { + record_server_request_safely(recorder, request)?; + record_interactive_request_state(recorder, request)?; + + dispatch_server_request(&mut client.connection, recorder, request, context) +} + +fn dispatch_server_request( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + context: RequestDispatchContext<'_>, +) -> crate::prelude::Result<()> { + match request.method.as_str() { + "item/tool/call" if context.phase == RequestWaitPhase::TurnExecution => + dispatch_dynamic_tool_call(connection, recorder, request, context), + "account/chatgptAuthTokens/refresh" => + dispatch_codex_account_refresh(connection, recorder, request, context), + "item/tool/call" => respond_to_dynamic_tool_call_dispatch( + connection, + recorder, + request, + dynamic_tool_call_unavailable_for_phase(context.phase), + ), + "item/commandExecution/requestApproval" => reject_interactive_server_request( + connection, + recorder, + request, + context.phase, + "item/commandExecution/requestApproval/response", + &CommandExecutionRequestApprovalResponse { + decision: CommandExecutionApprovalDecision::Decline, + }, + ), + "item/fileChange/requestApproval" => reject_interactive_server_request( + connection, + recorder, + request, + context.phase, + "item/fileChange/requestApproval/response", + &FileChangeRequestApprovalResponse { decision: FileChangeApprovalDecision::Decline }, + ), + "item/tool/requestUserInput" => reject_interactive_server_request( + connection, + recorder, + request, + context.phase, + "item/tool/requestUserInput/response", + &ToolRequestUserInputResponse::default(), + ), + "item/permissions/requestApproval" => reject_interactive_server_request( + connection, + recorder, + request, + context.phase, + "item/permissions/requestApproval/response", + &PermissionsRequestApprovalResponse { + permissions: Default::default(), + scope: PermissionGrantScope::Turn, + }, + ), + "mcpServer/elicitation/request" => reject_interactive_server_request( + connection, + recorder, + request, + context.phase, + "mcpServer/elicitation/request/response", + &McpServerElicitationRequestResponse { + action: McpServerElicitationAction::Decline, + content: None, + meta: None, + }, + ), + other => + reject_unsupported_server_request(connection, recorder, request, context.phase, other), + } +} + +fn record_server_request( + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, +) -> crate::prelude::Result<()> { + recorder.record( + request.method.as_str(), + &serde_json::json!({ + "id": request.id.clone(), + "method": request.method.clone(), + "params": request.params.clone(), + }) + .to_string(), + ) +} + +fn record_server_request_safely( + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, +) -> crate::prelude::Result<()> { + if request.method == "account/chatgptAuthTokens/refresh" { + return record_codex_account_refresh_request(recorder, request); + } + + record_server_request(recorder, request) +} + +fn record_wire_message_safely( + recorder: &mut RunRecorder<'_>, + wire_message: &WireMessage, +) -> crate::prelude::Result<()> { + match &wire_message.message { + JsonRpcMessage::Request(request) + if request.method == "account/chatgptAuthTokens/refresh" => + record_codex_account_refresh_request(recorder, request), + _ => recorder.record(message_type(wire_message), &wire_message.raw), + } +} + +fn record_codex_account_refresh_request( + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, +) -> crate::prelude::Result<()> { + let params = serde_json::from_value::(request.params.clone()) + .unwrap_or(ChatgptAuthTokensRefreshParams { reason: None, previous_account_id: None }); + + recorder.record( + "account/chatgptAuthTokens/refresh", + &serde_json::json!({ + "id": request.id.clone(), + "method": request.method.as_str(), + "reason": params.reason.as_deref(), + "previousAccountFingerprint": params.previous_account_id.as_deref().map(redact_identifier), + }) + .to_string(), + ) +} + +fn dispatch_codex_account_refresh( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + context: RequestDispatchContext<'_>, +) -> crate::prelude::Result<()> { + let account_provider = context.codex_account_provider.ok_or_else(|| { + eyre::eyre!( + "app_server_protocol_failure: received `account/chatgptAuthTokens/refresh` without a configured Codex account provider." + ) + })?; + let params = serde_json::from_value::(request.params.clone())?; + let account = account_provider.refresh_account(params.previous_account_id.as_deref())?; + let response = ChatgptAuthTokensRefreshResponse { + access_token: account.access_token().to_owned(), + chatgpt_account_id: account.account_id().to_owned(), + chatgpt_plan_type: account.plan_type().map(str::to_owned), + }; + + recorder.set_codex_account(account.summary(), account.account_summaries())?; + connection.respond(&request.id, &response)?; + + recorder.record( + "account/chatgptAuthTokens/refresh/response", + &serde_json::json!({ + "type": "chatgptAuthTokens", + "accountFingerprint": account.summary().account_fingerprint.as_str(), + "planType": account.summary().plan_type.as_deref(), + "refreshStatus": account.summary().refresh_status.as_str(), + "primaryRemainingPercent": account.summary().primary_remaining_percent, + "secondaryRemainingPercent": account.summary().secondary_remaining_percent, + }) + .to_string(), + ) +} + +fn dispatch_dynamic_tool_call( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + context: RequestDispatchContext<'_>, +) -> crate::prelude::Result<()> { + let target_turn_id = context.target_turn_id.ok_or_else(|| { + eyre::eyre!("app_server_protocol_failure: turn execution request missing turn context") + })?; + let target_thread_id = context.target_thread_id.ok_or_else(|| { + eyre::eyre!("app_server_protocol_failure: turn execution request missing thread context") + })?; + let dispatch = handle_dynamic_tool_call( + context.dynamic_tool_handler, + request, + target_thread_id, + target_turn_id, + ); + + respond_to_dynamic_tool_call_dispatch(connection, recorder, request, dispatch) +} + +fn dynamic_tool_call_unavailable_for_phase(phase: RequestWaitPhase) -> DynamicToolCallDispatch { + DynamicToolCallDispatch::protocol_failure( + None, + None, + format!("Dynamic tool calls are unavailable while waiting for {}.", phase.label()), + ) +} + +fn respond_to_dynamic_tool_call_dispatch( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + dispatch: DynamicToolCallDispatch, +) -> crate::prelude::Result<()> { + record_server_request_response( + connection, + recorder, + request, + "item/tool/call/response", + &dispatch.response, + )?; + + if let Some(diagnostic) = dispatch.diagnostic.as_ref() { + tracing::warn!( + failure_class = diagnostic.failure_class, + tool = diagnostic.tool.as_deref().unwrap_or("unknown"), + next_action = diagnostic.next_action, + message = diagnostic.message, + "Dynamic tool call failed." + ); + + recorder.record("item/tool/call/failure", &serde_json::to_string(diagnostic)?)?; + } + if let Some(terminal_failure) = dispatch.terminal_failure { + return Err(Report::new(terminal_failure)); + } + + Ok(()) +} + +fn reject_unsupported_server_request( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + phase: RequestWaitPhase, + method: &str, +) -> crate::prelude::Result<()> { + let message = format!("unsupported non-interactive server request `{method}`"); + + connection.respond_error(&request.id, JSONRPC_METHOD_NOT_FOUND, &message)?; + recorder.record( + "json-rpc/error/response", + &serde_json::json!({ + "code": JSONRPC_METHOD_NOT_FOUND, + "message": message, + }) + .to_string(), + )?; + + eyre::bail!( + "app_server_protocol_failure: unsupported server request `{method}` while waiting for {}.", + phase.label() + ); +} + +fn record_server_request_response( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + event_type: &str, + response: &T, +) -> crate::prelude::Result<()> +where + T: Serialize, +{ + connection.respond(&request.id, response)?; + + recorder.record(event_type, &serde_json::to_string(response)?) +} + +fn reject_interactive_server_request( + connection: &mut JsonRpcConnection, + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, + phase: RequestWaitPhase, + event_type: &str, + response: &T, +) -> crate::prelude::Result<()> +where + T: Serialize, +{ + record_server_request_response(connection, recorder, request, event_type, response)?; + + Err(noninteractive_interaction_required(request.method.as_str(), phase)) +} + +fn noninteractive_interaction_required(method: &str, phase: RequestWaitPhase) -> Report { + eyre::eyre!( + "noninteractive_interaction_required: server request `{method}` requires interactive handling during {}.", + phase.label() + ) +} + +fn record_interactive_request_state( + recorder: &mut RunRecorder<'_>, + request: &JsonRpcRequest, +) -> crate::prelude::Result<()> { + let Some(flag) = interactive_flag_for_request(request.method.as_str()) else { + return Ok(()); + }; + + if let Some(thread_id) = thread_id_from_value(&request.params) { + recorder.set_thread_id(thread_id)?; + } + if let Some(turn_id) = turn_id_from_value(&request.params) { + recorder.set_turn_id(turn_id)?; + } + + recorder.set_thread_status("active", &[flag.to_owned()]) +} + +fn interactive_flag_for_request(method: &str) -> Option<&'static str> { + match method { + "item/tool/requestUserInput" => Some("waitingOnUserInput"), + "item/commandExecution/requestApproval" + | "item/fileChange/requestApproval" + | "item/permissions/requestApproval" + | "mcpServer/elicitation/request" => Some("waitingOnApproval"), + _ => None, + } +} + +fn apply_protocol_message_side_effects( + recorder: &mut RunRecorder<'_>, + message: &WireMessage, +) -> crate::prelude::Result<()> { + match &message.message { + JsonRpcMessage::Notification(notification) + if notification.method == "thread/status/changed" => + { + let payload: ThreadStatusChangedNotification = + serde_json::from_value(notification.params.clone())?; + + if recorder.thread_id.is_none() { + recorder.set_thread_id(&payload.thread_id)?; + } + + recorder.set_thread_status(&payload.status.kind, &payload.status.active_flags)?; + }, + _ => {}, + } + + Ok(()) +} + +fn validate_effective_thread_config( + cwd: &str, + runtime: &EffectiveThreadConfig, +) -> crate::prelude::Result<()> { + if runtime.cwd != cwd { + eyre::bail!( + "app_server_protocol_failure: effective cwd `{}` did not match requested worktree `{cwd}`.", + runtime.cwd + ); + } + if runtime.approval_policy != "never" { + eyre::bail!( + "app_server_protocol_failure: effective approval policy `{}` is interactive; Decodex requires `never`.", + runtime.approval_policy + ); + } + if runtime.sandbox_mode == "readOnly" { + eyre::bail!( + "app_server_protocol_failure: effective sandbox mode `readOnly` does not allow Decodex execution." + ); + } + + Ok(()) +} + +fn validate_initialize_codex_home( + expected: &ResolvedAppServerCodexHomeEnv, + response: &InitializeResponse, +) -> crate::prelude::Result<()> { + let expected_home = normalized_home_path(expected.codex_home()); + let resolved_home = normalized_home_path(Path::new(&response.codex_home)); + + if resolved_home != expected_home { + tracing::warn!( + expected_codex_home = %expected.codex_home().display(), + resolved_codex_home = %response.codex_home, + "Codex app-server resolved an unexpected Codex home." + ); + + return Err(Report::new(AppServerHomePreflightFailure::initialize_mismatch( + response.codex_home.clone(), + expected.codex_home().display().to_string(), + ))); + } + + Ok(()) +} + +fn normalized_home_path(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn thread_resume_error_allows_fallback(error: &Report) -> bool { + let message = error.to_string().to_lowercase(); + + message.contains("no rollout found for thread id") + || message.contains("thread not found") + || message.contains("failed to load rollout") +} + +fn handle_dynamic_tool_call( + dynamic_tool_handler: Option<&dyn DynamicToolHandler>, + request: &JsonRpcRequest, + target_thread_id: &str, + target_turn_id: &str, +) -> DynamicToolCallDispatch { + let payload = + match validated_dynamic_tool_call_payload(request, target_thread_id, target_turn_id) { + Ok(payload) => payload, + Err(dispatch) => return *dispatch, + }; + let Some(dynamic_tool_handler) = dynamic_tool_handler else { + return DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + String::from("Dynamic tool bridge is unavailable for this run attempt."), + ); + }; + let tool_specs = dynamic_tool_handler.tool_specs(); + let spec_matches_namespace = tool_specs.iter().any(|spec| { + spec.name == payload.tool && spec.namespace.as_deref() == payload.namespace.as_deref() + }); + + if !spec_matches_namespace { + let message = match payload.namespace.as_deref() { + Some(namespace) => format!( + "Dynamic tool `{}` was called under namespace `{namespace}`, but this run did not declare that tool namespace.", + payload.tool + ), + None => + format!("Dynamic tool `{}` is not declared for this run attempt.", payload.tool), + }; + + return DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + message, + ); + } + + let response = dynamic_tool_handler.handle_call_with_namespace( + payload.namespace.as_deref(), + &payload.tool, + payload.arguments, + ); + + if let Err(message) = validate_dynamic_tool_call_response(&response) { + return DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + message, + ); + } + + if !response.success { + return DynamicToolCallDispatch::tool_failure( + response, + Some(payload.tool), + payload.namespace, + ); + } + + DynamicToolCallDispatch::success(response) +} + +fn validated_dynamic_tool_call_payload( + request: &JsonRpcRequest, + target_thread_id: &str, + target_turn_id: &str, +) -> std::result::Result> { + let payload = serde_json::from_value::(request.params.clone()).map_err( + |error| { + Box::new(DynamicToolCallDispatch::protocol_failure( + None, + None, + format!("Invalid `item/tool/call` payload: {error}"), + )) + }, + )?; + + if payload.call_id.trim().is_empty() { + return Err(Box::new(DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + String::from("Dynamic tool call payload included an empty `callId`."), + ))); + } + if !tracker_tool_bridge::dynamic_tool_identifier_is_valid(&payload.tool) { + return Err(Box::new(DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + String::from( + "Dynamic tool call payload included a tool name outside the Codex app-server identifier pattern `^[a-zA-Z0-9_-]+$`.", + ), + ))); + } + + if let Some(namespace) = payload.namespace.as_deref() + && !tracker_tool_bridge::dynamic_tool_identifier_is_valid(namespace) + { + return Err(Box::new(DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + String::from( + "Dynamic tool call payload included a namespace outside the Codex app-server identifier pattern `^[a-zA-Z0-9_-]+$`.", + ), + ))); + } + + if payload.thread_id != target_thread_id { + return Err(Box::new(DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + format!( + "Dynamic tool call targeted thread `{}`, but the active thread is `{target_thread_id}`.", + payload.thread_id + ), + ))); + } + if payload.turn_id != target_turn_id { + return Err(Box::new(DynamicToolCallDispatch::protocol_failure( + Some(payload.tool), + payload.namespace, + format!( + "Dynamic tool call targeted turn `{}`, but the active turn is `{target_turn_id}`.", + payload.turn_id + ), + ))); + } + + Ok(payload) +} + +fn validate_dynamic_tool_call_response(response: &DynamicToolCallResponse) -> Result<(), String> { + if response.content_items.is_empty() { + return Err(String::from( + "Dynamic tool handler returned an invalid response with no `contentItems`.", + )); + } + + Ok(()) +} + +fn dynamic_tool_response_text(response: &DynamicToolCallResponse) -> String { + let text_items = response + .content_items + .iter() + .map(|item| match item { + DynamicToolContentItem::InputText { text } => text.trim(), + }) + .filter(|text| !text.is_empty()) + .collect::>(); + + if text_items.is_empty() { + String::from("Dynamic tool call failed without a text response.") + } else { + text_items.join("\n") + } +} + +fn message_type(message: &WireMessage) -> &str { + match &message.message { + JsonRpcMessage::Notification(notification) => notification.method.as_str(), + JsonRpcMessage::Request(request) => request.method.as_str(), + JsonRpcMessage::Response(_) => "json-rpc/response", + JsonRpcMessage::Error(_) => "json-rpc/error", + } +} + +fn targets_thread(message: &WireMessage, target_thread_id: Option<&str>) -> bool { + let Some(target_thread_id) = target_thread_id else { + return true; + }; + + match &message.message { + JsonRpcMessage::Notification(notification) => thread_id_from_notification(notification) + .is_none_or(|thread_id| thread_id == target_thread_id), + JsonRpcMessage::Request(request) => thread_id_from_value(&request.params) + .is_none_or(|thread_id| thread_id == target_thread_id), + JsonRpcMessage::Response(_) | JsonRpcMessage::Error(_) => true, + } +} + +fn thread_id_from_notification(notification: &JsonRpcNotification) -> Option<&str> { + thread_id_from_value(¬ification.params) +} + +fn thread_id_from_value(value: &Value) -> Option<&str> { + value + .get("threadId") + .and_then(Value::as_str) + .or_else(|| value.get("thread").and_then(|thread| thread.get("id")).and_then(Value::as_str)) +} + +fn turn_id_from_value(value: &Value) -> Option<&str> { + value + .get("turnId") + .and_then(Value::as_str) + .or_else(|| value.get("turn").and_then(|turn| turn.get("id")).and_then(Value::as_str)) +} + +#[cfg(test)] mod tests; diff --git a/apps/decodex/src/agent/app_server/protocol.rs b/apps/decodex/src/agent/app_server/protocol.rs new file mode 100644 index 00000000..1e576c1b --- /dev/null +++ b/apps/decodex/src/agent/app_server/protocol.rs @@ -0,0 +1,851 @@ +use std::{ + collections::{BTreeMap, HashMap}, + env, + time::Duration, +}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + agent::{ + app_server::REQUEST_TIMEOUT, + json_rpc::{AppServerProcessEnv, JsonRpcConnection, JsonRpcRequest, WireMessage}, + tracker_tool_bridge::{DynamicToolCallResponse, DynamicToolHandler, DynamicToolSpec}, + }, + prelude::Result, +}; + +pub(super) struct AppServerClient { + pub(super) connection: JsonRpcConnection, +} +impl AppServerClient { + pub(super) fn spawn(listen: &str, process_env: &AppServerProcessEnv) -> Result { + Ok(Self { connection: JsonRpcConnection::spawn_app_server(listen, process_env)? }) + } + + #[allow(dead_code)] + pub(super) fn initialize( + &mut self, + enable_experimental_api: bool, + ) -> Result { + self.initialize_with_handler(enable_experimental_api, |_connection, _message, request| { + color_eyre::eyre::bail!( + "Unexpected inbound JSON-RPC request `{}` while waiting for `initialize`.", + request.method + ); + }) + } + + pub(super) fn initialize_with_handler( + &mut self, + enable_experimental_api: bool, + handler: H, + ) -> Result + where + H: FnMut(&mut JsonRpcConnection, &WireMessage, &JsonRpcRequest) -> Result<()>, + { + self.connection.request_with_handler( + "initialize", + &InitializeParams { + client_info: ClientInfo { + name: env!("CARGO_PKG_NAME").to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + }, + capabilities: enable_experimental_api.then_some(InitializeCapabilities { + experimental_api: Some(true), + opt_out_notification_methods: Vec::new(), + }), + }, + REQUEST_TIMEOUT, + handler, + ) + } + + pub(super) fn mark_initialized(&mut self) -> Result<()> { + self.connection.notify::("initialized", None) + } + + pub(super) fn login_account_with_handler( + &mut self, + params: LoginAccountParams, + handler: H, + ) -> Result + where + H: FnMut(&mut JsonRpcConnection, &WireMessage, &JsonRpcRequest) -> Result<()>, + { + self.connection.request_with_handler( + "account/login/start", + ¶ms, + REQUEST_TIMEOUT, + handler, + ) + } + + #[allow(dead_code)] + pub(super) fn start_thread( + &mut self, + params: ThreadStartRequest, + ) -> Result { + self.start_thread_with_handler(params, |_connection, _message, request| { + color_eyre::eyre::bail!( + "Unexpected inbound JSON-RPC request `{}` while waiting for `thread/start`.", + request.method + ); + }) + } + + pub(super) fn start_thread_with_handler( + &mut self, + params: ThreadStartRequest, + handler: H, + ) -> Result + where + H: FnMut(&mut JsonRpcConnection, &WireMessage, &JsonRpcRequest) -> Result<()>, + { + self.connection.request_with_handler("thread/start", ¶ms, REQUEST_TIMEOUT, handler) + } + + #[allow(dead_code)] + pub(super) fn resume_thread( + &mut self, + params: ThreadResumeRequest, + ) -> Result { + self.resume_thread_with_handler(params, |_connection, _message, request| { + color_eyre::eyre::bail!( + "Unexpected inbound JSON-RPC request `{}` while waiting for `thread/resume`.", + request.method + ); + }) + } + + pub(super) fn resume_thread_with_handler( + &mut self, + params: ThreadResumeRequest, + handler: H, + ) -> Result + where + H: FnMut(&mut JsonRpcConnection, &WireMessage, &JsonRpcRequest) -> Result<()>, + { + self.connection.request_with_handler("thread/resume", ¶ms, REQUEST_TIMEOUT, handler) + } + + #[allow(dead_code)] + pub(super) fn start_turn(&mut self, params: TurnStartRequest) -> Result { + self.start_turn_with_handler(params, |_connection, _message, request| { + color_eyre::eyre::bail!( + "Unexpected inbound JSON-RPC request `{}` while waiting for `turn/start`.", + request.method + ); + }) + } + + pub(super) fn start_turn_with_handler( + &mut self, + params: TurnStartRequest, + handler: H, + ) -> Result + where + H: FnMut(&mut JsonRpcConnection, &WireMessage, &JsonRpcRequest) -> Result<()>, + { + self.connection.request_with_handler("turn/start", ¶ms, REQUEST_TIMEOUT, handler) + } + + pub(super) fn command_exec( + &mut self, + params: &CommandExecParams, + ) -> Result { + self.connection.request("command/exec", params, params.request_timeout()) + } + + pub(super) fn read_config(&mut self, params: &ConfigReadParams) -> Result { + self.connection.request("config/read", params, REQUEST_TIMEOUT) + } + + pub(super) fn list_models(&mut self, params: &ModelListParams) -> Result { + self.connection.request("model/list", params, REQUEST_TIMEOUT) + } + + pub(super) fn read_model_provider_capabilities( + &mut self, + ) -> Result { + self.connection.request( + "modelProvider/capabilities/read", + &ModelProviderCapabilitiesReadParams {}, + REQUEST_TIMEOUT, + ) + } + + pub(super) fn list_skills(&mut self, params: &SkillsListParams) -> Result { + self.connection.request("skills/list", params, REQUEST_TIMEOUT) + } + + pub(super) fn list_plugins(&mut self, params: &PluginListParams) -> Result { + self.connection.request("plugin/list", params, REQUEST_TIMEOUT) + } + + pub(super) fn list_mcp_server_status( + &mut self, + params: &ListMcpServerStatusParams, + ) -> Result { + self.connection.request("mcpServerStatus/list", params, REQUEST_TIMEOUT) + } + + pub(super) fn recv(&mut self, timeout: Option) -> Result { + self.connection.recv(timeout) + } + + #[allow(dead_code)] + pub(super) fn respond(&mut self, id: &Value, result: &R) -> Result<()> + where + R: Serialize, + { + self.connection.respond(id, result) + } + + #[allow(dead_code)] + pub(super) fn respond_error(&mut self, id: &Value, code: i64, message: &str) -> Result<()> { + self.connection.respond_error(id, code, message) + } + + pub(super) fn drain_pending(&mut self) -> Vec { + self.connection.drain_pending() + } +} + +#[derive(Serialize)] +#[serde(tag = "type")] +pub(super) enum LoginAccountParams { + #[serde(rename = "chatgptAuthTokens", rename_all = "camelCase")] + ChatgptAuthTokens { + access_token: String, + chatgpt_account_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + chatgpt_plan_type: Option, + }, +} + +#[derive(Debug, Eq, PartialEq, Deserialize)] +#[serde(tag = "type")] +pub(super) enum LoginAccountResponse { + #[serde(rename = "chatgptAuthTokens")] + ChatgptAuthTokens {}, +} + +#[derive(Default)] +pub(super) struct RunOutcome { + pub(super) final_output: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct InitializeParams { + #[serde(rename = "clientInfo")] + pub(super) client_info: ClientInfo, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) capabilities: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct ClientInfo { + pub(super) name: String, + pub(super) version: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct InitializeCapabilities { + #[serde(rename = "experimentalApi", skip_serializing_if = "Option::is_none")] + pub(super) experimental_api: Option, + #[serde(default, rename = "optOutNotificationMethods", skip_serializing_if = "Vec::is_empty")] + pub(super) opt_out_notification_methods: Vec, +} + +#[derive(Debug, Deserialize)] +pub(super) struct InitializeResponse { + #[serde(rename = "userAgent")] + pub(super) user_agent: String, + #[allow(dead_code)] + #[serde(rename = "codexHome")] + pub(super) codex_home: String, + #[allow(dead_code)] + #[serde(rename = "platformFamily")] + pub(super) platform_family: String, + #[allow(dead_code)] + #[serde(rename = "platformOs")] + pub(super) platform_os: String, +} + +#[derive(Debug, Default, Serialize)] +pub(super) struct ThreadStartRequest { + #[serde(rename = "baseInstructions", skip_serializing_if = "Option::is_none")] + pub(super) base_instructions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) config: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cwd: Option, + #[serde(rename = "dynamicTools", skip_serializing_if = "Option::is_none")] + pub(super) dynamic_tools: Option>, + #[serde(rename = "developerInstructions", skip_serializing_if = "Option::is_none")] + pub(super) developer_instructions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) ephemeral: Option, + #[serde(rename = "modelProvider", skip_serializing_if = "Option::is_none")] + pub(super) model_provider: Option, + #[serde(rename = "serviceName", skip_serializing_if = "Option::is_none")] + pub(super) service_name: Option, +} + +#[derive(Debug, Default, Serialize)] +pub(super) struct ThreadResumeRequest { + #[serde(rename = "threadId")] + pub(super) thread_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) model: Option, + #[serde(rename = "modelProvider", skip_serializing_if = "Option::is_none")] + pub(super) model_provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cwd: Option, + #[serde(rename = "approvalPolicy", skip_serializing_if = "Option::is_none")] + pub(super) approval_policy: Option, + #[serde(rename = "approvalsReviewer", skip_serializing_if = "Option::is_none")] + pub(super) approvals_reviewer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) sandbox: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) config: Option, + #[serde(rename = "baseInstructions", skip_serializing_if = "Option::is_none")] + pub(super) base_instructions: Option, + #[serde(rename = "developerInstructions", skip_serializing_if = "Option::is_none")] + pub(super) developer_instructions: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(super) struct ThreadSessionResponse { + pub(super) thread: Thread, + pub(super) model: String, + #[serde(rename = "modelProvider")] + pub(super) model_provider: String, + #[serde(rename = "serviceTier")] + pub(super) _service_tier: Option, + pub(super) cwd: String, + #[serde(default, rename = "instructionSources")] + pub(super) _instruction_sources: Vec, + #[serde(rename = "approvalPolicy")] + pub(super) approval_policy: Value, + #[serde(rename = "approvalsReviewer")] + pub(super) approvals_reviewer: String, + pub(super) sandbox: Value, + #[serde(rename = "reasoningEffort")] + pub(super) _reasoning_effort: Option, +} +impl ThreadSessionResponse { + pub(super) fn effective_config(&self) -> EffectiveThreadConfig { + EffectiveThreadConfig { + model: self.model.clone(), + model_provider: self.model_provider.clone(), + cwd: self.cwd.clone(), + approval_policy: externally_tagged_value_name(&self.approval_policy) + .unwrap_or_else(|| String::from("unknown")), + approvals_reviewer: self.approvals_reviewer.clone(), + sandbox_mode: externally_tagged_value_name(&self.sandbox) + .unwrap_or_else(|| String::from("unknown")), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct EffectiveThreadConfig { + pub(super) model: String, + pub(super) model_provider: String, + pub(super) cwd: String, + pub(super) approval_policy: String, + pub(super) approvals_reviewer: String, + pub(super) sandbox_mode: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub(super) struct Thread { + pub(super) id: String, +} + +#[derive(Debug, Default, Serialize)] +pub(super) struct TurnStartRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cwd: Option, + pub(super) input: Vec, + #[serde(rename = "outputSchema", skip_serializing_if = "Option::is_none")] + pub(super) output_schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) summary: Option, + #[serde(rename = "threadId")] + pub(super) thread_id: String, +} + +#[derive(Debug, Deserialize)] +pub(super) struct TurnStartResponse { + pub(super) turn: TurnStatusPayload, +} + +#[derive(Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct CommandExecParams { + pub(super) command: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cwd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) timeout_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) output_bytes_cap: Option, +} +impl CommandExecParams { + fn request_timeout(&self) -> Duration { + self.timeout_ms + .map(Duration::from_millis) + .map(|timeout| timeout.saturating_add(REQUEST_TIMEOUT)) + .unwrap_or(REQUEST_TIMEOUT) + } +} + +#[derive(Debug, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct CommandExecResponse { + pub(super) exit_code: i32, + pub(super) stdout: String, + pub(super) stderr: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ConfigReadParams { + pub(super) cwd: Option, + pub(super) include_layers: bool, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ConfigReadResponse { + pub(super) config: RuntimeConfigSummary, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +pub(super) struct RuntimeConfigSummary { + pub(super) model: Option, + #[serde(rename = "model_provider")] + pub(super) model_provider: Option, + pub(super) approval_policy: Option, + pub(super) sandbox_mode: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ModelListParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cursor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) include_hidden: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ModelListResponse { + pub(super) data: Vec, + #[serde(rename = "nextCursor")] + pub(super) next_cursor: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct ModelSummary { + pub(super) id: String, + pub(super) model: String, + #[serde(rename = "displayName")] + pub(super) display_name: String, + #[serde(rename = "isDefault")] + pub(super) is_default: bool, + pub(super) hidden: bool, +} + +#[derive(Debug, Serialize)] +pub(super) struct ModelProviderCapabilitiesReadParams {} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct ModelProviderCapabilitiesReadResponse { + #[serde(rename = "imageGeneration")] + pub(super) image_generation: bool, + #[serde(rename = "namespaceTools")] + pub(super) namespace_tools: bool, + #[serde(rename = "webSearch")] + pub(super) web_search: bool, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SkillsListParams { + pub(super) cwds: Vec, + pub(super) force_reload: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) per_cwd_extra_user_roots: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct SkillsListResponse { + pub(super) data: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct SkillsListEntry { + pub(super) cwd: String, + pub(super) errors: Vec, + pub(super) skills: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct SkillErrorInfo { + pub(super) message: String, + pub(super) path: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct SkillMetadata { + pub(super) enabled: bool, + pub(super) name: String, + pub(super) scope: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct PluginListParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cwds: Option>, +} + +#[derive(Debug, Deserialize)] +pub(super) struct PluginListResponse { + pub(super) marketplaces: Vec, + #[serde(default, rename = "marketplaceLoadErrors")] + pub(super) marketplace_load_errors: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct PluginMarketplaceEntry { + pub(super) name: String, + pub(super) plugins: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct PluginSummary { + pub(super) enabled: bool, + pub(super) id: String, + pub(super) installed: bool, + pub(super) name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub(super) struct MarketplaceLoadErrorInfo { + #[serde(rename = "marketplacePath")] + pub(super) marketplace_path: String, + pub(super) message: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ListMcpServerStatusParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) cursor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) detail: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ListMcpServerStatusResponse { + pub(super) data: Vec, + #[serde(rename = "nextCursor")] + pub(super) next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub(super) struct McpServerStatusSummary { + #[serde(rename = "authStatus")] + pub(super) auth_status: String, + pub(super) name: String, + pub(super) tools: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(super) struct TurnStatusPayload { + pub(super) id: String, + pub(super) status: String, + pub(super) error: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct TurnError { + pub(super) message: String, + #[serde(rename = "codexErrorInfo")] + pub(super) codex_error_info: Option, + #[allow(dead_code)] + #[serde(rename = "additionalDetails")] + pub(super) additional_details: Option, +} + +#[derive(Debug, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ChatgptAuthTokensRefreshParams { + pub(super) reason: Option, + pub(super) previous_account_id: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ChatgptAuthTokensRefreshResponse { + pub(super) access_token: String, + pub(super) chatgpt_account_id: String, + pub(super) chatgpt_plan_type: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ThreadStatusChangedNotification { + #[serde(rename = "threadId")] + pub(super) thread_id: String, + pub(super) status: ThreadStatus, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ThreadStatus { + #[serde(rename = "type")] + pub(super) kind: String, + #[serde(default, rename = "activeFlags")] + pub(super) active_flags: Vec, +} + +#[derive(Debug, Deserialize)] +pub(super) struct AgentMessageDeltaNotification { + pub(super) delta: String, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ItemCompletedNotification { + pub(super) item: CompletedItem, +} + +#[derive(Debug, Deserialize)] +pub(super) struct CompletedItem { + #[serde(rename = "type")] + pub(super) kind: String, + pub(super) text: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct TurnCompletedNotification { + #[allow(dead_code)] + #[serde(rename = "threadId")] + pub(super) thread_id: Option, + pub(super) turn: TurnStatusPayload, +} + +#[derive(Debug, Deserialize)] +pub(super) struct ErrorNotification { + pub(super) error: TurnError, + #[serde(rename = "willRetry")] + pub(super) will_retry: Option, + #[serde(rename = "threadId")] + pub(super) thread_id: Option, + #[serde(rename = "turnId")] + pub(super) turn_id: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct DynamicToolCallParams { + pub(super) arguments: Value, + #[serde(rename = "callId")] + pub(super) call_id: String, + pub(super) namespace: Option, + #[serde(rename = "threadId")] + pub(super) thread_id: String, + pub(super) tool: String, + #[serde(rename = "turnId")] + pub(super) turn_id: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct CommandExecutionRequestApprovalResponse { + pub(super) decision: CommandExecutionApprovalDecision, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum CommandExecutionApprovalDecision { + Decline, +} + +#[derive(Debug, Serialize)] +pub(super) struct FileChangeRequestApprovalResponse { + pub(super) decision: FileChangeApprovalDecision, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum FileChangeApprovalDecision { + Decline, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ToolRequestUserInputResponse { + pub(super) answers: HashMap, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ToolRequestUserInputAnswer { + pub(super) answers: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct McpServerElicitationRequestResponse { + pub(super) action: McpServerElicitationAction, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) content: Option, + #[serde(rename = "_meta", skip_serializing_if = "Option::is_none")] + pub(super) meta: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum McpServerElicitationAction { + Decline, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct PermissionsRequestApprovalResponse { + pub(super) permissions: GrantedPermissionProfile, + pub(super) scope: PermissionGrantScope, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct GrantedPermissionProfile {} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum PermissionGrantScope { + #[default] + Turn, +} + +pub(super) struct ProbeDynamicToolHandler; +impl DynamicToolHandler for ProbeDynamicToolHandler { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "echo_probe", + "Echo the provided text back to the model.", + serde_json::json!({ + "type": "object", + "properties": { + "text": { "type": "string" } + }, + "required": ["text"], + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, tool_name: &str, arguments: Value) -> DynamicToolCallResponse { + if tool_name != "echo_probe" { + return DynamicToolCallResponse::failure(format!( + "Unexpected probe tool `{tool_name}`." + )); + } + + let Some(text) = arguments.get("text").and_then(Value::as_str) else { + return DynamicToolCallResponse::failure(String::from( + "`echo_probe` requires a string `text` argument.", + )); + }; + + DynamicToolCallResponse::success(text.to_owned()) + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type")] +pub(super) enum UserInput { + #[serde(rename = "text")] + Text { text: String }, +} + +fn externally_tagged_value_name(value: &Value) -> Option { + match value { + Value::String(value) if !value.is_empty() => Some(value.clone()), + Value::Object(object) => object + .get("type") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| (object.len() == 1).then(|| object.keys().next().cloned()).flatten()), + _ => None, + } +} + +#[cfg(test)] +mod tests { + #[test] + fn externally_tagged_values_prefer_explicit_type_field() { + assert_eq!( + super::externally_tagged_value_name(&serde_json::json!({ "type": "dangerFullAccess" })), + Some(String::from("dangerFullAccess")) + ); + } + + #[test] + fn error_notifications_keep_codex_error_info_without_retry_flag() { + let notification: super::ErrorNotification = serde_json::from_value(serde_json::json!({ + "error": { + "message": "usage limit exceeded", + "codexErrorInfo": "usageLimitExceeded" + }, + "threadId": "thread-1", + "turnId": "turn-1" + })) + .expect("error notification should parse"); + + assert_eq!(notification.error.codex_error_info.as_deref(), Some("usageLimitExceeded")); + assert_eq!(notification.will_retry, None); + } + + #[test] + fn chatgpt_auth_tokens_login_uses_app_server_protocol_shape() { + let value = serde_json::to_value(super::LoginAccountParams::ChatgptAuthTokens { + access_token: String::from("access"), + chatgpt_account_id: String::from("acct_1"), + chatgpt_plan_type: Some(String::from("pro")), + }) + .expect("login params should serialize"); + + assert_eq!( + value, + serde_json::json!({ + "type": "chatgptAuthTokens", + "accessToken": "access", + "chatgptAccountId": "acct_1", + "chatgptPlanType": "pro" + }) + ); + } + + #[test] + fn command_exec_request_timeout_includes_process_timeout() { + let params = super::CommandExecParams { + command: vec![String::from("/bin/sh")], + cwd: None, + timeout_ms: Some(1_000), + output_bytes_cap: Some(128), + }; + + assert_eq!( + params.request_timeout(), + std::time::Duration::from_millis(1_000) + super::REQUEST_TIMEOUT + ); + } +} diff --git a/apps/decodex/src/agent/app_server/tests.rs b/apps/decodex/src/agent/app_server/tests.rs new file mode 100644 index 00000000..14b6c8f2 --- /dev/null +++ b/apps/decodex/src/agent/app_server/tests.rs @@ -0,0 +1,1285 @@ +use std::{ + cell::RefCell, + collections::BTreeMap, + path::PathBuf, + time::{Duration, Instant}, +}; + +use serde_json::{self, Value}; +use tempfile::TempDir; + +use crate::{ + agent::{ + app_server::{ + AppServerCapabilityPreflightFailure, AppServerCapabilityPreflightReport, + AppServerDynamicToolFailure, AppServerRunResult, AppServerTurnFailure, + CommandExecHealthCheck, CommandExecResponse, EffectiveThreadConfig, InitializeResponse, + ModelProviderCapabilitiesReadResponse, PluginListResponse, ProbeDynamicToolHandler, + RequestWaitPhase, RunRecorder, RuntimeConfigSummary, SkillsListResponse, + TurnContinuationGuard, UserInput, + }, + json_rpc::{ + AppServerHomePreflightFailure, AppServerProcessEnv, JsonRpcMessage, + JsonRpcNotification, JsonRpcRequest, ResolvedAppServerCodexHomeEnv, WireMessage, + }, + tracker_tool_bridge::{ + DynamicToolCallResponse, DynamicToolContentItem, DynamicToolHandler, DynamicToolSpec, + TurnCompletionStatus, + }, + }, + prelude::{Result, eyre}, + state::{self, StateStore}, +}; + +struct RejectingCompletionHandler; +impl DynamicToolHandler for RejectingCompletionHandler { + fn tool_specs(&self) -> Vec { + Vec::new() + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::failure(String::from("unused")) + } + + fn validate_turn_completion(&self, _final_output: &str) -> Result<()> { + Err(eyre::eyre!("terminal finalization missing")) + } +} + +struct ContinuingCompletionHandler; +impl DynamicToolHandler for ContinuingCompletionHandler { + fn tool_specs(&self) -> Vec { + Vec::new() + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::failure(String::from("unused")) + } + + fn classify_turn_completion(&self, _final_output: &str) -> Result { + Ok(TurnCompletionStatus::Continue) + } + + fn validate_turn_completion(&self, _final_output: &str) -> Result<()> { + Err(eyre::eyre!("terminal finalization missing")) + } +} + +struct InvalidToolNameHandler; +impl DynamicToolHandler for InvalidToolNameHandler { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "invalid.tool", + "Invalid test tool.", + serde_json::json!({ + "type": "object", + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::success(String::from("unused")) + } +} + +struct EmptyToolResponseHandler; +impl DynamicToolHandler for EmptyToolResponseHandler { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "empty_response", + "Return an invalid empty response.", + serde_json::json!({ + "type": "object", + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse { content_items: Vec::new(), success: true } + } +} + +struct FailingToolHandler; +impl DynamicToolHandler for FailingToolHandler { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "failing_tool", + "Return a normal tool failure response.", + serde_json::json!({ + "type": "object", + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::failure(String::from("tool rejected the request")) + } +} + +struct YieldingContinuationGuard; +impl TurnContinuationGuard for YieldingContinuationGuard { + fn should_continue_turn(&self, _turn_count: u32) -> Result { + Ok(false) + } +} + +struct RejectingContinuationGuard; +impl TurnContinuationGuard for RejectingContinuationGuard { + fn should_continue_turn(&self, _turn_count: u32) -> Result { + Ok(false) + } + + fn validate_continuation_boundary(&self, turn_count: u32) -> Result<()> { + Err(eyre::eyre!("turn {turn_count} hit an invalid continuation boundary")) + } +} + +struct NamespacedDynamicToolHandler { + seen_namespace: RefCell>, +} +impl DynamicToolHandler for NamespacedDynamicToolHandler { + fn tool_specs(&self) -> Vec { + let mut spec = DynamicToolSpec::new( + "tracker_tool", + "Test namespaced tool.", + serde_json::json!({ + "type": "object", + "additionalProperties": false + }), + ); + + spec.namespace = Some(String::from("tracker")); + + vec![spec] + } + + fn handle_call(&self, _tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::failure(String::from("namespace should be forwarded")) + } + + fn handle_call_with_namespace( + &self, + namespace: Option<&str>, + _tool_name: &str, + _arguments: Value, + ) -> DynamicToolCallResponse { + self.seen_namespace.replace(namespace.map(str::to_owned)); + + DynamicToolCallResponse::success(String::from("ok")) + } +} + +struct LiveResumeDynamicToolHandler; +impl DynamicToolHandler for LiveResumeDynamicToolHandler { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "echo_resume", + "Echo the provided integration text.", + serde_json::json!({ + "type": "object", + "properties": { + "text": { "type": "string" } + }, + "required": ["text"], + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, tool_name: &str, arguments: Value) -> DynamicToolCallResponse { + if tool_name != "echo_resume" { + return DynamicToolCallResponse::failure(format!( + "Unexpected live integration tool `{tool_name}`." + )); + } + + let Some(text) = arguments.get("text").and_then(Value::as_str) else { + return DynamicToolCallResponse::failure(String::from( + "`echo_resume` requires a string `text` argument.", + )); + }; + + DynamicToolCallResponse::success(text.to_owned()) + } + + fn classify_turn_completion(&self, final_output: &str) -> Result { + Ok(match final_output.trim() { + "CONTINUE" => TurnCompletionStatus::Continue, + _ => TurnCompletionStatus::Complete, + }) + } +} + +struct LiveResumeBoundaryGuard; +impl TurnContinuationGuard for LiveResumeBoundaryGuard { + fn should_continue_turn(&self, _turn_count: u32) -> Result { + Ok(false) + } +} + +fn notification_message(method: &str, params: Value) -> WireMessage { + WireMessage { + raw: params.to_string(), + message: JsonRpcMessage::Notification(JsonRpcNotification { + method: method.to_owned(), + params, + }), + } +} + +#[test] +fn matches_thread_id_from_supported_notification_shapes() { + for message in [ + notification_message( + "thread/started", + serde_json::json!({ + "thread": { + "id": "thread-1", + } + }), + ), + notification_message( + "turn/completed", + serde_json::json!({ + "threadId": "thread-1", + "turn": { + "id": "turn-1", + "status": "completed", + "error": null, + } + }), + ), + ] { + assert!(super::targets_thread(&message, Some("thread-1"))); + assert!(!super::targets_thread(&message, Some("thread-2"))); + } +} + +#[test] +fn probe_result_shape_is_stable() { + let result = AppServerRunResult { + user_agent: String::from("ua"), + thread_id: String::from("thread"), + turn_id: String::from("turn"), + turn_count: 1, + event_count: 3, + final_output: String::from("PROBE_OK"), + continuation_pending: false, + }; + + assert_eq!(result.final_output, "PROBE_OK"); + assert_eq!(result.turn_count, 1); +} + +#[test] +fn turn_start_request_uses_default_runtime_settings() { + let request = super::build_turn_start_request("thread-1", "hello"); + + assert_eq!(request.thread_id, "thread-1"); + assert!(matches!( + request.input.as_slice(), + [UserInput::Text{ text }] if text == "hello" + )); +} + +#[test] +fn thread_start_and_resume_requests_inherit_runtime_config() { + fn assert_runtime_config(value: &Value) { + assert_eq!(value["cwd"], "/tmp/worktree"); + assert_eq!(value["developerInstructions"], "Follow the workflow."); + assert!(value.get("model").is_none()); + assert!(value.get("modelProvider").is_none()); + assert!(value.get("personality").is_none()); + assert!(value.get("serviceTier").is_none()); + assert!(value.get("approvalPolicy").is_none()); + assert!(value.get("sandbox").is_none()); + assert!(value.get("config").is_none()); + } + + let start = + super::build_thread_start_request(&minimal_run_request()).expect("request should build"); + let start_value = serde_json::to_value(&start).expect("thread start request should serialize"); + let resume = super::build_thread_resume_request("thread-1", &minimal_run_request()); + let resume_value = + serde_json::to_value(&resume).expect("thread resume request should serialize"); + + assert_runtime_config(&start_value); + assert_runtime_config(&resume_value); + + assert_eq!(resume_value["threadId"], "thread-1"); +} + +#[test] +fn thread_start_rejects_invalid_dynamic_tool_names() { + let handler = InvalidToolNameHandler; + let mut request = minimal_run_request(); + + request.dynamic_tool_handler = Some(&handler); + + let error = super::build_thread_start_request(&request) + .expect_err("invalid dynamic tool name should fail before thread/start"); + let failure = error + .downcast_ref::() + .expect("invalid dynamic tool should classify as a dynamic tool failure"); + + assert_eq!(failure.error_class(), "app_server_dynamic_tool_protocol_failure"); + assert!(error.to_string().contains("identifier pattern")); +} + +#[test] +fn turn_start_request_omits_execution_policy_overrides() { + let request = super::build_turn_start_request("thread-1", "hello"); + let value = serde_json::to_value(&request).expect("turn start request should serialize"); + + assert_eq!(value["threadId"], "thread-1"); + assert!(value.get("model").is_none()); + assert!(value.get("modelProvider").is_none()); + assert!(value.get("personality").is_none()); + assert!(value.get("serviceTier").is_none()); + assert!(value.get("approvalPolicy").is_none()); + assert!(value.get("sandboxPolicy").is_none()); + assert!(value.get("config").is_none()); +} + +fn minimal_run_request<'a>() -> super::AppServerRunRequest<'a> { + super::AppServerRunRequest { + run_id: String::from("run-1"), + issue_id: String::from("issue-1"), + attempt_number: 1, + listen: String::from("stdio://"), + cwd: String::from("/tmp/worktree"), + developer_instructions: String::from("Follow the workflow."), + user_input: String::from("Work the issue."), + max_turns: 1, + timeout: Duration::from_secs(30), + process_env: AppServerProcessEnv::default(), + continuation_user_input: None, + activity_marker_path: None, + resume_thread_id: None, + command_exec_health_check: None, + dynamic_tool_handler: None, + continuation_guard: None, + codex_account_provider: None, + } +} + +#[test] +fn command_exec_health_check_uses_bounded_standalone_request() { + let health_check = CommandExecHealthCheck { + command: vec![String::from("/bin/sh"), String::from("-c"), String::from("printf ok")], + expected_stdout: String::from("ok"), + timeout_ms: 1_000, + output_bytes_cap: 128, + }; + let params = super::build_command_exec_health_check_params(&health_check, "/tmp/worktree"); + let value = serde_json::to_value(¶ms).expect("command exec params should serialize"); + + assert_eq!(value["command"], serde_json::json!(["/bin/sh", "-c", "printf ok"])); + assert_eq!(value["cwd"], "/tmp/worktree"); + assert_eq!(value["timeoutMs"], 1_000); + assert_eq!(value["outputBytesCap"], 128); + assert!(value.get("threadId").is_none()); + assert!(value.get("sandboxPolicy").is_none()); + assert!(value.get("permissionProfile").is_none()); +} + +#[test] +fn command_exec_health_check_validates_exact_buffered_result() { + let health_check = CommandExecHealthCheck { + command: vec![String::from("/bin/sh"), String::from("-c"), String::from("printf ok")], + expected_stdout: String::from("ok"), + timeout_ms: 1_000, + output_bytes_cap: 128, + }; + let response = + CommandExecResponse { exit_code: 0, stdout: String::from("ok"), stderr: String::new() }; + + super::validate_command_exec_health_check_result(&health_check, &response) + .expect("matching command exec result should pass"); + + let bad_response = + CommandExecResponse { exit_code: 0, stdout: String::from("wrong"), stderr: String::new() }; + let error = super::validate_command_exec_health_check_result(&health_check, &bad_response) + .expect_err("mismatched stdout should fail health check"); + + assert!(error.to_string().contains("expected \"ok\"")); +} + +#[test] +fn capability_preflight_report_accepts_available_runtime_state() { + let config = RuntimeConfigSummary { + model: Some(String::from("gpt-5.4")), + model_provider: Some(String::from("openai")), + approval_policy: Some(serde_json::json!("never")), + sandbox_mode: Some(serde_json::json!("workspaceWrite")), + }; + let models = vec![super::ModelSummary { + id: String::from("model-gpt-5.4"), + model: String::from("gpt-5.4"), + display_name: String::from("GPT-5.4"), + is_default: true, + hidden: false, + }]; + let capabilities = ModelProviderCapabilitiesReadResponse { + image_generation: true, + namespace_tools: true, + web_search: true, + }; + let skills = SkillsListResponse { + data: vec![super::protocol::SkillsListEntry { + cwd: String::from("/tmp/worktree"), + errors: Vec::new(), + skills: vec![super::protocol::SkillMetadata { + enabled: true, + name: String::from("playbook:rust"), + scope: String::from("user"), + }], + }], + }; + let plugins = PluginListResponse { + marketplaces: vec![super::protocol::PluginMarketplaceEntry { + name: String::from("curated"), + plugins: vec![super::protocol::PluginSummary { + enabled: true, + id: String::from("github"), + installed: true, + name: String::from("GitHub"), + }], + }], + marketplace_load_errors: Vec::new(), + }; + let mcp = vec![super::McpServerStatusSummary { + auth_status: String::from("bearerToken"), + name: String::from("linear"), + tools: BTreeMap::from([(String::from("issue_transition"), serde_json::json!({}))]), + }]; + let mut report = AppServerCapabilityPreflightReport::new(); + + super::record_config_preflight(&mut report, &config); + super::record_model_preflight(&mut report, &config, &models); + super::record_model_provider_preflight(&mut report, &capabilities); + super::record_skills_preflight(&mut report, "/tmp/worktree", &skills); + super::record_plugin_preflight(&mut report, &plugins); + super::record_mcp_preflight(&mut report, &mcp); + + assert!(!report.has_blockers()); + assert_eq!(report.checks().len(), 6); + assert!( + report + .checks() + .iter() + .all(|check| { check.status == super::AppServerCapabilityPreflightStatus::Ok }) + ); + + let serialized = serde_json::to_value(&report).expect("report should serialize"); + + assert_eq!(serialized["checks"][0]["status"], "ok"); + assert_eq!(serialized["checks"][1]["details"]["configured_model"], "gpt-5.4"); +} + +#[test] +fn capability_preflight_report_blocks_missing_runtime_state() { + let config = RuntimeConfigSummary { + model: Some(String::from("missing-model")), + model_provider: Some(String::from("openai")), + approval_policy: None, + sandbox_mode: None, + }; + let models = vec![super::ModelSummary { + id: String::from("model-gpt-5.4"), + model: String::from("gpt-5.4"), + display_name: String::from("GPT-5.4"), + is_default: true, + hidden: false, + }]; + let skills = SkillsListResponse { + data: vec![super::protocol::SkillsListEntry { + cwd: String::from("/tmp/worktree"), + errors: vec![super::protocol::SkillErrorInfo { + message: String::from("bad skill metadata"), + path: String::from("/tmp/worktree/.codex/skills/bad/SKILL.md"), + }], + skills: Vec::new(), + }], + }; + let plugins = PluginListResponse { + marketplaces: Vec::new(), + marketplace_load_errors: vec![super::protocol::MarketplaceLoadErrorInfo { + marketplace_path: String::from("/tmp/plugins.json"), + message: String::from("invalid marketplace"), + }], + }; + let mcp = vec![super::McpServerStatusSummary { + auth_status: String::from("notLoggedIn"), + name: String::from("linear"), + tools: BTreeMap::new(), + }]; + let mut report = AppServerCapabilityPreflightReport::new(); + + super::record_model_preflight(&mut report, &config, &models); + super::record_skills_preflight(&mut report, "/tmp/worktree", &skills); + super::record_plugin_preflight(&mut report, &plugins); + super::record_mcp_preflight(&mut report, &mcp); + + assert!(report.has_blockers()); + assert_eq!( + report.blocker_summary(), + "model: configured model was not present in model/list.; skills: skills/list returned skill scan errors.; plugins: plugin/list returned marketplace load errors.; mcp: mcpServerStatus/list returned MCP servers that are not logged in." + ); +} + +#[test] +fn capability_preflight_method_error_is_typed_operator_blocker() { + let mut report = AppServerCapabilityPreflightReport::new(); + + report.push_ok( + "config", + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + + let failure = AppServerCapabilityPreflightFailure::method_failed( + "model/list", + String::from("`model/list` failed with -32601: Method not found"), + report, + ); + + assert_eq!(failure.error_class(), "app_server_introspection_method_failed"); + assert!(failure.to_string().contains("model/list")); + assert!(failure.to_string().contains("Method not found")); + assert_eq!(failure.report().checks().len(), 1); +} + +#[test] +fn capability_preflight_request_error_records_method_blocker() { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, None); + let report = AppServerCapabilityPreflightReport::new(); + let error = super::preflight_request::<(), _>(&mut recorder, &report, "model/list", || { + Err(eyre::eyre!("JSON-RPC error -32601: Method not found")) + }) + .expect_err("unsupported app-server method should fail preflight"); + let failure = error + .downcast_ref::() + .expect("preflight request error should be typed"); + + assert_eq!(failure.error_class(), "app_server_introspection_method_failed"); + assert!(failure.to_string().contains("model/list")); + assert!(failure.to_string().contains("Method not found")); + assert!(failure.report().has_blockers()); + assert_eq!(state_store.event_count("run-1").expect("event count should load"), 1); +} + +#[test] +fn remaining_idle_budget_resets_from_latest_activity() { + let now = Instant::now(); + let timeout = Duration::from_secs(300); + let last_activity_at = now.checked_sub(Duration::from_secs(12)).expect("instant math"); + let remaining = + super::remaining_idle_budget(last_activity_at, now, timeout).expect("budget should remain"); + + assert!(remaining <= timeout); + assert!(remaining >= Duration::from_secs(287)); +} + +#[test] +fn remaining_idle_budget_expires_after_idle_timeout() { + let now = Instant::now(); + let timeout = Duration::from_secs(300); + let last_activity_at = now.checked_sub(Duration::from_secs(301)).expect("instant math"); + + assert!(super::remaining_idle_budget(last_activity_at, now, timeout).is_none()); +} + +#[test] +fn run_recorder_keeps_events_when_marker_write_fails() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let missing_worktree = PathBuf::from(temp_dir.path()).join("missing-worktree"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&missing_worktree)); + + recorder.mark_activity().expect("marker failures should be non-fatal"); + recorder.record("turn/started", "{\"turn\":\"1\"}").expect("event should record"); + + assert_eq!(state_store.event_count("run-1").expect("event count should load"), 1); +} + +#[test] +fn completion_classification_uses_dynamic_tool_handler() { + let error = super::classify_turn_completion(Some(&RejectingCompletionHandler), "finished") + .expect_err("completion classifier should be consulted"); + + assert!(error.to_string().contains("terminal finalization missing")); +} + +#[test] +fn completion_classification_defaults_to_complete_without_handler() { + assert_eq!( + super::classify_turn_completion(None, "finished") + .expect("missing dynamic handler should not fail completion"), + TurnCompletionStatus::Complete + ); +} + +#[test] +fn probe_handler_allows_completion_classification() { + assert_eq!( + super::classify_turn_completion(Some(&ProbeDynamicToolHandler), "PROBE_OK") + .expect("probe handler should not override completion validation"), + TurnCompletionStatus::Complete + ); +} + +#[test] +fn nonterminal_single_turn_completion_stays_invalid() { + let error = super::reject_nonterminal_single_turn_completion( + Some(&ContinuingCompletionHandler), + "unfinished", + ) + .expect_err("single-turn mode should preserve terminal completion validation"); + + assert!(error.to_string().contains("terminal finalization missing")); +} + +#[test] +fn continuation_boundary_reached_yields_when_guard_allows_it() { + assert!( + super::continuation_boundary_reached(Some(&YieldingContinuationGuard), 2) + .expect("yielding guard should allow a clean continuation boundary") + ); +} + +#[test] +fn continuation_boundary_reached_rejects_invalid_boundary() { + let error = super::continuation_boundary_reached(Some(&RejectingContinuationGuard), 1) + .expect_err("invalid continuation boundaries should surface as errors"); + + assert!(error.to_string().contains("turn 1 hit an invalid continuation boundary")); +} + +#[test] +fn validate_effective_thread_config_accepts_noninteractive_runtime() { + let runtime = EffectiveThreadConfig { + model: String::from("gpt-5.4"), + model_provider: String::from("openai"), + cwd: String::from("/tmp/worktree"), + approval_policy: String::from("never"), + approvals_reviewer: String::from("human"), + sandbox_mode: String::from("workspaceWrite"), + }; + + super::validate_effective_thread_config("/tmp/worktree", &runtime) + .expect("matching non-interactive config should validate"); +} + +#[test] +fn validate_effective_thread_config_rejects_interactive_runtime_policies() { + for (case_name, approval_policy, sandbox_mode, expected) in [ + ( + "interactive approval policy", + "onRequest", + "workspaceWrite", + "approval policy `onRequest`", + ), + ("read-only sandbox", "never", "readOnly", "readOnly"), + ] { + let runtime = EffectiveThreadConfig { + model: String::from("gpt-5.4"), + model_provider: String::from("openai"), + cwd: String::from("/tmp/worktree"), + approval_policy: String::from(approval_policy), + approvals_reviewer: String::from("human"), + sandbox_mode: String::from(sandbox_mode), + }; + let error = super::validate_effective_thread_config("/tmp/worktree", &runtime) + .expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } +} + +#[test] +fn initialize_codex_home_assertion_accepts_expected_home() { + let expected = ResolvedAppServerCodexHomeEnv::new( + PathBuf::from("/Users/test/.codex"), + PathBuf::from("/Users/test/.codex"), + ) + .expect("test Codex home should validate"); + let response = InitializeResponse { + user_agent: String::from("codex-cli-test"), + codex_home: String::from("/Users/test/.codex"), + platform_family: String::from("unix"), + platform_os: String::from("macos"), + }; + + super::validate_initialize_codex_home(&expected, &response) + .expect("matching Codex home should pass"); +} + +#[test] +fn initialize_codex_home_assertion_blocks_before_thread_start_on_mismatch() { + let expected = ResolvedAppServerCodexHomeEnv::new( + PathBuf::from("/Users/test/.codex"), + PathBuf::from("/Users/test/.codex"), + ) + .expect("test Codex home should validate"); + let response = InitializeResponse { + user_agent: String::from("codex-cli-test"), + codex_home: String::from("/tmp/per-account-codex-home"), + platform_family: String::from("unix"), + platform_os: String::from("macos"), + }; + let error = super::validate_initialize_codex_home(&expected, &response) + .expect_err("mismatched Codex home should fail before thread start"); + + assert!(error.downcast_ref::().is_some()); + assert!(error.to_string().contains("initialize codexHome `/tmp/per-account-codex-home`")); + assert!(error.to_string().contains("expected shared Codex home `/Users/test/.codex`")); + assert!(error.to_string().contains("before thread/start")); +} + +#[test] +fn app_server_turn_failure_classifies_operator_attention() { + for (case_name, message, code, requires_attention, error_class) in [ + ( + "usage limit", + "You've hit your usage limit.", + Some(String::from("usageLimitExceeded")), + true, + "app_server_usage_limit_exceeded", + ), + ("generic failure", "transient model failure", None, false, "app_server_turn_failed"), + ] { + let failure = AppServerTurnFailure::new( + "thread-1", + Some(String::from("turn-1")), + "failed", + message, + code.clone(), + ); + + assert_eq!(failure.requires_operator_attention(), requires_attention, "{case_name}"); + assert_eq!(failure.error_class(), error_class); + + if let Some(code) = code { + assert!(failure.to_string().contains(&code)); + } + } +} + +#[test] +fn thread_resume_fallback_only_allows_missing_thread_errors() { + assert!(super::thread_resume_error_allows_fallback(&eyre::eyre!("thread not found"))); + assert!(super::thread_resume_error_allows_fallback(&eyre::eyre!( + "failed to load rollout from disk" + ))); + assert!(!super::thread_resume_error_allows_fallback(&eyre::eyre!( + "thread belongs to another cwd" + ))); +} + +#[test] +fn dynamic_tool_call_enforces_declared_namespace() { + for (case_name, namespace, expected_success, expected_seen_namespace, expected_error) in [ + ( + "unknown namespace", + Some("other"), + false, + None, + Some( + "Dynamic tool `tracker_tool` was called under namespace `other`, but this run did not declare that tool namespace.", + ), + ), + ("declared namespace", Some("tracker"), true, Some("tracker"), None), + ( + "missing namespace", + None, + false, + None, + Some("Dynamic tool `tracker_tool` is not declared for this run attempt."), + ), + ] { + let handler = NamespacedDynamicToolHandler { seen_namespace: RefCell::new(None) }; + let mut params = serde_json::json!({ + "arguments": {}, + "callId": "call-1", + "threadId": "thread-1", + "tool": "tracker_tool", + "turnId": "turn-1" + }); + + if let Some(namespace) = namespace { + params["namespace"] = serde_json::json!(namespace); + } + + let request = JsonRpcRequest { + id: serde_json::json!(1), + method: String::from("item/tool/call"), + params, + }; + let dispatch = + super::handle_dynamic_tool_call(Some(&handler), &request, "thread-1", "turn-1"); + + assert_eq!(dispatch.response.success, expected_success, "{case_name}"); + assert_eq!( + *handler.seen_namespace.borrow(), + expected_seen_namespace.map(String::from), + "{case_name}" + ); + + if let Some(expected_error) = expected_error { + assert_eq!( + dispatch.response.content_items, + vec![DynamicToolContentItem::InputText { text: String::from(expected_error) }], + "{case_name}" + ); + assert_eq!( + dispatch + .terminal_failure + .as_ref() + .map(super::AppServerDynamicToolFailure::error_class), + Some("app_server_dynamic_tool_protocol_failure"), + "{case_name}" + ); + } else { + assert!(dispatch.terminal_failure.is_none(), "{case_name}"); + } + } +} + +#[test] +fn dynamic_tool_call_rejects_invalid_response_shape() { + let handler = EmptyToolResponseHandler; + let request = JsonRpcRequest { + id: serde_json::json!(1), + method: String::from("item/tool/call"), + params: serde_json::json!({ + "arguments": {}, + "callId": "call-1", + "threadId": "thread-1", + "tool": "empty_response", + "turnId": "turn-1" + }), + }; + let dispatch = super::handle_dynamic_tool_call(Some(&handler), &request, "thread-1", "turn-1"); + + assert!(!dispatch.response.success); + assert!(matches!( + dispatch.response.content_items.as_slice(), + [DynamicToolContentItem::InputText { text }] + if text.contains("invalid response with no `contentItems`") + )); + assert_eq!( + dispatch.terminal_failure.as_ref().map(super::AppServerDynamicToolFailure::error_class), + Some("app_server_dynamic_tool_protocol_failure") + ); +} + +#[test] +fn dynamic_tool_call_records_tool_failures_without_terminal_protocol_failure() { + let handler = FailingToolHandler; + let request = JsonRpcRequest { + id: serde_json::json!(1), + method: String::from("item/tool/call"), + params: serde_json::json!({ + "arguments": {}, + "callId": "call-1", + "threadId": "thread-1", + "tool": "failing_tool", + "turnId": "turn-1" + }), + }; + let dispatch = super::handle_dynamic_tool_call(Some(&handler), &request, "thread-1", "turn-1"); + + assert!(!dispatch.response.success); + assert!(dispatch.terminal_failure.is_none()); + + let diagnostic = dispatch.diagnostic.expect("tool failure should publish a diagnostic"); + + assert_eq!(diagnostic.failure_class, "app_server_dynamic_tool_failed"); + assert_eq!(diagnostic.tool.as_deref(), Some("failing_tool")); + assert_eq!(diagnostic.message, "tool rejected the request"); +} + +#[test] +fn dynamic_tool_call_unavailable_outside_turn_execution_is_protocol_diagnostic() { + let dispatch = super::dynamic_tool_call_unavailable_for_phase(RequestWaitPhase::TurnStart); + + assert!(!dispatch.response.success); + assert!(matches!( + dispatch.response.content_items.as_slice(), + [DynamicToolContentItem::InputText { text }] + if text.contains("unavailable while waiting for turn/start") + )); + assert_eq!( + dispatch.terminal_failure.as_ref().map(super::AppServerDynamicToolFailure::error_class), + Some("app_server_dynamic_tool_protocol_failure") + ); + + let diagnostic = dispatch.diagnostic.expect("protocol failure should publish a diagnostic"); + + assert_eq!(diagnostic.failure_class, "app_server_dynamic_tool_protocol_failure"); + assert!(diagnostic.message.contains("unavailable while waiting for turn/start")); + assert_eq!( + diagnostic.next_action, + "inspect the declared dynamic tool surface and item/tool/call payload before retrying the lane" + ); +} + +#[test] +fn interactive_request_updates_marker_turn_id_to_current_turn() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let marker_path = temp_dir.path().to_path_buf(); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&marker_path)); + + recorder.set_thread_id("thread-1").expect("thread marker should write"); + recorder.set_turn_id("turn-old").expect("initial turn marker should write"); + + let request = JsonRpcRequest { + id: serde_json::json!(1), + method: String::from("item/tool/requestUserInput"), + params: serde_json::json!({ + "threadId": "thread-1", + "turnId": "turn-new", + }), + }; + + super::record_interactive_request_state(&mut recorder, &request) + .expect("interactive request state should record"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.thread_id(), Some("thread-1")); + assert_eq!(marker.turn_id(), Some("turn-new")); + assert_eq!(marker.thread_status(), Some("active")); + assert_eq!(marker.thread_active_flags(), &[String::from("waitingOnUserInput")]); +} + +#[test] +fn recorder_aggregates_child_agent_activity_breakdown() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let marker_path = temp_dir.path().to_path_buf(); + let large_output = "x".repeat(100_500); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&marker_path)); + + recorder + .record( + "thread/status/changed", + r#"{"method":"thread/status/changed","params":{"threadId":"thread-1","status":{"type":"active"}}}"#, + ) + .expect("thread status should record"); + recorder + .record( + "item/tool/call", + r#"{"method":"item/tool/call","params":{"tool":"functions.exec_command","arguments":{"cmd":"cargo make test"},"threadId":"thread-1","turnId":"turn-1","callId":"call-1"}}"#, + ) + .expect("shell tool call should record"); + recorder + .record( + "item/tool/call/response", + r#"{"contentItems":[{"type":"inputText","text":"tests passed"}],"success":true}"#, + ) + .expect("shell tool response should record"); + + for call_id in ["call-2", "call-3"] { + recorder + .record( + "item/tool/call", + &format!( + r#"{{"method":"item/tool/call","params":{{"tool":"view_image","arguments":{{"detail":"original"}},"threadId":"thread-1","turnId":"turn-1","callId":"{call_id}"}}}}"# + ), + ) + .expect("image tool call should record"); + recorder + .record( + "item/tool/call/response", + &format!( + r#"{{"contentItems":[{{"type":"inputText","text":"{large_output}"}}],"success":true}}"# + ), + ) + .expect("image tool response should record"); + } + + recorder + .record( + "turn/completed", + r#"{"method":"turn/completed","params":{"turn":{"id":"turn-1","status":"completed"},"usage":{"input_tokens":105000,"output_tokens":12000}}}"#, + ) + .expect("turn completion should record"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + let summary = marker.child_agent_activity().expect("child activity should be captured"); + let protocol_activity = + marker.protocol_activity().expect("protocol activity should be captured"); + + assert_eq!(summary.event_count, 8); + assert_eq!(summary.tool_call_count, 3); + assert_eq!(summary.current_bucket, None); + assert_eq!(summary.input_tokens_current, Some(105_000)); + assert_eq!(summary.input_tokens_cumulative, 105_000); + assert_eq!(summary.output_tokens_cumulative, 12_000); + assert_eq!(summary.largest_tool_output_tool.as_deref(), Some("view_image")); + assert!( + summary + .large_output_warnings + .iter() + .any(|warning| warning.contains("view_image repeated 2 large outputs")) + ); + assert!(summary.buckets.iter().any(|bucket| { + bucket.name == "Shell" && bucket.tool_call_count == 1 && bucket.event_count >= 2 + })); + assert!(summary.buckets.iter().any(|bucket| { + bucket.name == "Browser/Image" + && bucket.tool_call_count == 2 + && bucket.output_bytes > 200_000 + })); + assert!(summary.buckets.iter().any(|bucket| { + bucket.name == "Model" && bucket.input_tokens == 105_000 && bucket.output_tokens == 12_000 + })); + assert_eq!(protocol_activity.turn_status.as_deref(), Some("completed")); + assert_eq!(protocol_activity.waiting_reason.as_deref(), Some("turn_completed")); + assert_eq!(protocol_activity.recent_events.len(), 8); + assert!(protocol_activity.recent_events.iter().any(|event| { + event.event_type == "item/tool/call" + && event.detail.as_deref() == Some("functions.exec_command") + })); + assert!(protocol_activity.recent_events.iter().any(|event| { + event.event_type == "turn/completed" + && event.category == "turn" + && event.detail.as_deref() == Some("completed") + })); +} + +#[test] +fn recorder_summarizes_high_value_protocol_activity() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let marker_path = temp_dir.path().to_path_buf(); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&marker_path)); + + for (event_type, payload) in [ + ( + "turn/started", + r#"{"method":"turn/started","params":{"turn":{"id":"turn-1","status":"running"}}}"#, + ), + ("plan/update", r#"{"method":"plan/update","params":{"step":"verify"}}"#), + ("diff/update", r#"{"method":"diff/update","params":{"filesChanged":2}}"#), + ( + "item/tool/call/failure", + r#"{"failureClass":"app_server_dynamic_tool_failed","tool":"issue_comment","message":"tool rejected","nextAction":"retry"}"#, + ), + ("command/output/delta", r#"{"method":"command/output/delta","params":{"delta":"ok"}}"#), + ("item/tool/requestUserInput/response", r#"{"answers":{}}"#), + ( + "item/tool/requestUserInput", + r#"{"method":"item/tool/requestUserInput","params":{"threadId":"thread-1","turnId":"turn-1"}}"#, + ), + ( + "account/rateLimit/update", + r#"{"rateLimitReachedType":"primary","primaryRemainingPercent":0}"#, + ), + ] { + recorder.record(event_type, payload).expect("protocol event should record"); + } + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + let summary = marker.protocol_activity().expect("protocol activity should be captured"); + let categories = + summary.recent_events.iter().map(|event| event.category.as_str()).collect::>(); + + assert_eq!(summary.turn_status.as_deref(), Some("running")); + assert_eq!(summary.waiting_reason.as_deref(), Some("approval_or_user_input")); + assert_eq!(summary.rate_limit_status.as_deref(), Some("primary")); + assert!(categories.contains(&"turn")); + assert!(categories.contains(&"plan")); + assert!(categories.contains(&"diff")); + assert!(categories.contains(&"item")); + assert!(categories.contains(&"command_output")); + assert!(categories.contains(&"protocol_error")); + assert!(categories.contains(&"server_request_resolution")); + assert!(categories.contains(&"rate_limit")); + assert!(summary.recent_events.iter().any(|event| { + event.event_type == "item/tool/call/failure" + && event.detail.as_deref() == Some("app_server_dynamic_tool_failed") + })); +} + +#[test] +fn recorder_summarizes_wrapped_account_protocol_activity() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let marker_path = temp_dir.path().to_path_buf(); + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&marker_path)); + + recorder + .record( + "account/update", + r#"{"method":"account/update","params":{"planType":"pro","refreshStatus":"refreshed"}}"#, + ) + .expect("account protocol event should record"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + let summary = marker.protocol_activity().expect("protocol activity should be captured"); + let event = summary.recent_events.first().expect("recent account event should render"); + + assert_eq!(event.category, "account"); + assert_eq!(event.detail.as_deref(), Some("pro/refreshed")); +} + +#[test] +fn turn_execution_records_dynamic_tool_call_before_response() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let marker_path = temp_dir.path().to_path_buf(); + let request = JsonRpcRequest { + id: serde_json::json!(7), + method: String::from("item/tool/call"), + params: serde_json::json!({ + "tool": "issue_progress_checkpoint", + "arguments": {"phase": "verifying"}, + "threadId": "thread-1", + "turnId": "turn-1", + "callId": "call-1", + }), + }; + let mut recorder = RunRecorder::new(&state_store, "run-1", 1, Some(&marker_path)); + + super::record_server_request(&mut recorder, &request) + .expect("tool call request should record before handler execution"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + let activity = marker.child_agent_activity().expect("child activity should be captured"); + + assert_eq!(marker.last_event_type(), Some("item/tool/call")); + assert_eq!(activity.current_bucket.as_deref(), Some("Tracker")); + assert_eq!(activity.current_detail.as_deref(), Some("issue_progress_checkpoint")); + assert!(activity.buckets.iter().any(|bucket| { + bucket.name == "Tracker" && bucket.tool_call_count == 1 && bucket.event_count == 1 + })); +} + +#[test] +#[ignore = "requires a live local codex app-server binary"] +fn live_app_server_resume_round_trip_updates_marker_and_state() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let marker_path = temp_dir.path().to_path_buf(); + let first_state_store = StateStore::open_in_memory().expect("state store should open"); + let handler = LiveResumeDynamicToolHandler; + let guard = LiveResumeBoundaryGuard; + let cwd = marker_path.display().to_string(); + let developer_instructions = String::from( + "You are a live resume integration test. On the first turn, call the dynamic tool `echo_resume` exactly once with the JSON argument `{\"text\":\"FIRST_OK\"}` and then reply with the exact text CONTINUE. If the thread is later resumed and the user asks for `SECOND_OK`, call `echo_resume` exactly once with `{\"text\":\"SECOND_OK\"}` and then reply with the exact text DONE. Do not use shell. Do not inspect files.", + ); + let first_result = super::execute_app_server_run( + &super::AppServerRunRequest { + run_id: String::from("live-resume-run"), + issue_id: String::from("live-resume-issue"), + attempt_number: 1, + listen: String::from("stdio://"), + cwd: cwd.clone(), + developer_instructions: developer_instructions.clone(), + user_input: String::from( + "Call `echo_resume` with `{\\\"text\\\":\\\"FIRST_OK\\\"}`. After the tool succeeds, reply with the exact text CONTINUE.", + ), + max_turns: 3, + timeout: Duration::from_secs(30), + process_env: AppServerProcessEnv::default(), + continuation_user_input: Some(String::from( + "Call `echo_resume` with `{\\\"text\\\":\\\"SECOND_OK\\\"}`. After the tool succeeds, reply with the exact text DONE.", + )), + activity_marker_path: Some(marker_path.clone()), + resume_thread_id: None, + command_exec_health_check: None, + dynamic_tool_handler: Some(&handler), + continuation_guard: Some(&guard), + codex_account_provider: None, + }, + &first_state_store, + ) + .expect("first live app-server run should succeed"); + + assert!(first_result.continuation_pending); + assert_eq!(first_result.final_output.trim(), "CONTINUE"); + + let first_marker = state::read_run_activity_marker_snapshot(&marker_path) + .expect("first marker snapshot should load") + .expect("first marker snapshot should exist"); + + assert_eq!(first_marker.run_id(), "live-resume-run"); + assert_eq!(first_marker.attempt_number(), 1); + assert_eq!(first_marker.thread_id(), Some(first_result.thread_id.as_str())); + assert_eq!(first_marker.turn_id(), Some(first_result.turn_id.as_str())); + assert_eq!(first_marker.effective_cwd(), Some(cwd.as_str())); + assert_eq!(first_marker.effective_approval_policy(), Some("never")); + assert!(first_marker.last_protocol_activity_unix_epoch().is_some()); + + let resumed_state_store = + StateStore::open_in_memory().expect("resumed state store should open"); + let second_result = super::execute_app_server_run( + &super::AppServerRunRequest { + run_id: String::from("live-resume-run"), + issue_id: String::from("live-resume-issue"), + attempt_number: 1, + listen: String::from("stdio://"), + cwd: cwd.clone(), + developer_instructions, + user_input: String::from( + "Call `echo_resume` with `{\\\"text\\\":\\\"SECOND_OK\\\"}`. After the tool succeeds, reply with the exact text DONE.", + ), + max_turns: 1, + timeout: Duration::from_secs(30), + process_env: AppServerProcessEnv::default(), + continuation_user_input: None, + activity_marker_path: Some(marker_path.clone()), + resume_thread_id: Some(first_result.thread_id.clone()), + command_exec_health_check: None, + dynamic_tool_handler: Some(&handler), + continuation_guard: None, + codex_account_provider: None, + }, + &resumed_state_store, + ) + .expect("resumed live app-server run should succeed"); + + assert!(!second_result.continuation_pending); + assert_eq!(second_result.thread_id, first_result.thread_id); + assert_ne!(second_result.turn_id, first_result.turn_id); + assert_eq!(second_result.final_output.trim(), "DONE"); + + let second_marker = state::read_run_activity_marker_snapshot(&marker_path) + .expect("second marker snapshot should load") + .expect("second marker snapshot should exist"); + + assert_eq!(second_marker.thread_id(), Some(first_result.thread_id.as_str())); + assert_eq!(second_marker.turn_id(), Some(second_result.turn_id.as_str())); + assert_eq!(second_marker.effective_model_provider(), Some("openai")); + assert_eq!(second_marker.effective_cwd(), Some(cwd.as_str())); + assert!(second_marker.last_protocol_activity_unix_epoch().is_some()); + assert!(second_marker.event_count() > 0); + + let resumed_attempt = resumed_state_store + .run_attempt("live-resume-run") + .expect("resumed run attempt should load") + .expect("resumed run attempt should exist"); + + assert_eq!(resumed_attempt.thread_id(), Some(first_result.thread_id.as_str())); + assert_eq!(resumed_attempt.turn_id(), Some(second_result.turn_id.as_str())); +} diff --git a/apps/decodex/src/agent/app_server/turn_failure.rs b/apps/decodex/src/agent/app_server/turn_failure.rs new file mode 100644 index 00000000..d319a19f --- /dev/null +++ b/apps/decodex/src/agent/app_server/turn_failure.rs @@ -0,0 +1,81 @@ +use std::{ + error::Error, + fmt::{Display, Formatter}, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AppServerTurnFailure { + thread_id: String, + turn_id: Option, + status: String, + message: String, + codex_error_info: Option, +} +impl AppServerTurnFailure { + pub(crate) fn new( + thread_id: impl Into, + turn_id: Option, + status: impl Into, + message: impl Into, + codex_error_info: Option, + ) -> Self { + Self { + thread_id: thread_id.into(), + turn_id, + status: status.into(), + message: message.into(), + codex_error_info, + } + } + + pub(super) fn from_system_error(thread_id: &str) -> Self { + Self::new( + thread_id, + None, + "systemError", + "thread entered systemError before a turn error was reported", + None, + ) + } + + pub(crate) fn requires_operator_attention(&self) -> bool { + matches!(self.codex_error_info.as_deref(), Some("usageLimitExceeded")) + } + + pub(crate) fn error_class(&self) -> &'static str { + match self.codex_error_info.as_deref() { + Some("usageLimitExceeded") => "app_server_usage_limit_exceeded", + _ => "app_server_turn_failed", + } + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + match self.codex_error_info.as_deref() { + Some("usageLimitExceeded") => format!( + "inspect Codex account usage and retry after credits or the usage reset are available, {recovery_gate}" + ), + _ => format!( + "inspect the app-server turn error and worktree, resolve the blocker manually, {recovery_gate}" + ), + } + } +} +impl Display for AppServerTurnFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + write!(formatter, "Codex app-server turn failed on thread `{}`", self.thread_id)?; + + if let Some(turn_id) = self.turn_id.as_deref() { + write!(formatter, ", turn `{turn_id}`")?; + } + + write!(formatter, " with status `{}`: {}", self.status, self.message)?; + + if let Some(codex_error_info) = self.codex_error_info.as_deref() { + write!(formatter, " ({codex_error_info})")?; + } + + Ok(()) + } +} + +impl Error for AppServerTurnFailure {} diff --git a/apps/decodex/src/agent/codex_accounts.rs b/apps/decodex/src/agent/codex_accounts.rs new file mode 100644 index 00000000..1eee1958 --- /dev/null +++ b/apps/decodex/src/agent/codex_accounts.rs @@ -0,0 +1,1141 @@ +use std::{ + cmp::Ordering, + error::Error, + fmt::{self, Display, Formatter}, + fs, + path::{Path, PathBuf}, + process, + sync::Mutex, + time::Duration, +}; + +use reqwest::{StatusCode, blocking::Client}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + config::ProjectCodexAccountsConfig, prelude::eyre, state::CodexAccountActivitySummary, +}; + +const DEFAULT_USAGE_ENDPOINT: &str = "https://chatgpt.com/backend-api/wham/usage"; +const DEFAULT_REFRESH_ENDPOINT: &str = "https://auth.openai.com/oauth/token"; +const CODEX_USER_AGENT: &str = "codex-cli"; +const CHATGPT_OAUTH_CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); + +pub(crate) trait CodexAccountProvider { + fn select_account(&self) -> crate::prelude::Result; + fn refresh_account( + &self, + previous_account_id: Option<&str>, + ) -> crate::prelude::Result; +} + +pub(crate) struct CodexAccountPool { + path: PathBuf, + usage_endpoint: String, + refresh_endpoint: String, + client: Client, + selected_account_id: Mutex>, +} +impl CodexAccountPool { + pub(crate) fn from_config(config: &ProjectCodexAccountsConfig) -> crate::prelude::Result { + Self::new( + config.path(), + config.usage_endpoint().unwrap_or(DEFAULT_USAGE_ENDPOINT), + config.refresh_endpoint().unwrap_or(DEFAULT_REFRESH_ENDPOINT), + ) + } + + pub(crate) fn new( + path: impl AsRef, + usage_endpoint: impl Into, + refresh_endpoint: impl Into, + ) -> crate::prelude::Result { + let client = Client::builder().timeout(HTTP_TIMEOUT).build()?; + + Ok(Self { + path: path.as_ref().to_path_buf(), + usage_endpoint: usage_endpoint.into(), + refresh_endpoint: refresh_endpoint.into(), + client, + selected_account_id: Mutex::new(None), + }) + } + + fn load_records(&self) -> crate::prelude::Result> { + let input = fs::read_to_string(&self.path).map_err(|error| { + eyre::eyre!("Failed to read Codex accounts `{}`: {error}", self.path.display()) + })?; + + parse_account_records(&input, &self.path) + } + + pub(crate) fn account_activity_summaries( + &self, + ) -> crate::prelude::Result> { + let mut records = self.load_records()?; + + self.probe_account_activity_summaries(&mut records) + } + + fn save_records(&self, records: &[AccountPoolRecord]) -> crate::prelude::Result<()> { + let parent = self.path.parent().ok_or_else(|| { + eyre::eyre!( + "Codex accounts path `{}` must have a parent directory.", + self.path.display() + ) + })?; + let file_name = self + .path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| eyre::eyre!("Codex accounts path must end in a valid file name."))?; + let temp_path = parent.join(format!(".{file_name}.tmp-{}", process::id())); + let mut body = String::new(); + + for record in records { + body.push_str(&serde_json::to_string(record)?); + body.push('\n'); + } + + fs::write(&temp_path, body)?; + fs::rename(temp_path, &self.path)?; + + Ok(()) + } + + fn select_from_records( + &self, + records: &mut [AccountPoolRecord], + ) -> crate::prelude::Result { + let now = OffsetDateTime::now_utc().unix_timestamp(); + let mut candidates = Vec::new(); + let mut skipped = Vec::new(); + let mut records_changed = false; + + for (index, record) in records.iter_mut().enumerate() { + if record.disabled { + skipped.push(format!("line {} disabled", index + 1)); + + continue; + } + if record.cooldown_until_unix_epoch.is_some_and(|cooldown| cooldown > now) { + skipped.push(format!("line {} cooling down", index + 1)); + + continue; + } + if record.account_id().is_none() { + skipped.push(format!("line {} missing account id", index + 1)); + + continue; + } + if record.access_token().is_none() { + skipped.push(format!("line {} missing access token", index + 1)); + + continue; + } + + match self.probe_record_usage(record) { + Ok(usage) => candidates.push(record.login_from_usage(usage, "not_needed")?), + Err(error) if error.unauthorized && record.refresh_token().is_some() => { + self.refresh_record(record)?; + + records_changed = true; + + let usage = self.probe_record_usage(record).map_err(|retry_error| { + eyre::eyre!( + "Codex account `{}` refreshed but usage probe still failed: {retry_error}", + record.display_name() + ) + })?; + + candidates.push(record.login_from_usage(usage, "succeeded")?); + }, + Err(error) => { + skipped.push(format!("{} usage probe failed: {error}", record.display_name())); + }, + } + } + + if records_changed { + self.save_records(records)?; + } + if candidates.is_empty() { + eyre::bail!( + "No usable Codex account was available from `{}`. Skipped entries: {}", + self.path.display(), + if skipped.is_empty() { String::from("none") } else { skipped.join("; ") } + ); + } + + candidates.sort_by(compare_account_candidates); + + let mut selected = candidates.remove(0); + + selected.mark_selected(now); + + let account_summaries = account_summaries(&selected, &candidates); + let selected = selected.with_account_summaries(account_summaries); + + self.remember_selected_account(&selected.account_id)?; + + Ok(selected) + } + + fn probe_account_activity_summaries( + &self, + records: &mut [AccountPoolRecord], + ) -> crate::prelude::Result> { + let now = OffsetDateTime::now_utc().unix_timestamp(); + let mut summaries = Vec::new(); + let mut records_changed = false; + + for record in records.iter_mut() { + let Some(configured_summary) = record.configured_activity_summary(now) else { + continue; + }; + + if configured_summary.status != "available" { + summaries.push(configured_summary); + + continue; + } + + match self.probe_record_usage(record) { + Ok(usage) => { + summaries.push(record.activity_summary_from_usage(usage, "not_needed")?); + }, + Err(error) if error.unauthorized && record.refresh_token().is_some() => { + match self.refresh_record(record) { + Ok(()) => { + records_changed = true; + + match self.probe_record_usage(record) { + Ok(usage) => { + summaries.push( + record.activity_summary_from_usage(usage, "succeeded")?, + ); + }, + Err(retry_error) => { + summaries.push(record.probe_failed_activity_summary( + now, + "failed", + &retry_error, + )); + }, + } + }, + Err(refresh_error) => { + summaries.push(record.probe_failed_activity_summary( + now, + "failed", + refresh_error.as_ref(), + )); + }, + } + }, + Err(error) => { + summaries.push(record.probe_failed_activity_summary( + now, + "probe_failed", + &error, + )); + }, + } + } + + if records_changed { + self.save_records(records)?; + } + + Ok(summaries) + } + + fn refresh_from_records( + &self, + records: &mut [AccountPoolRecord], + previous_account_id: Option<&str>, + ) -> crate::prelude::Result { + let selected_account_id = self.selected_account_id()?; + let target_account_id = previous_account_id.or(selected_account_id.as_deref()); + let Some(record_index) = records.iter().position(|record| { + target_account_id.is_none_or(|target| record.account_id() == Some(target)) + }) else { + eyre::bail!( + "Codex account refresh requested an account that is not in the configured accounts." + ); + }; + + self.refresh_record(&mut records[record_index])?; + + let usage = self.probe_record_usage(&records[record_index])?; + let now = OffsetDateTime::now_utc().unix_timestamp(); + let mut selected = records[record_index].login_from_usage(usage, "succeeded")?; + + selected.mark_selected(now); + + let selected_summary = selected.summary().clone(); + let selected = selected.with_account_summaries(vec![selected_summary]); + + self.save_records(records)?; + self.remember_selected_account(&selected.account_id)?; + + Ok(selected) + } + + fn probe_record_usage( + &self, + record: &AccountPoolRecord, + ) -> std::result::Result { + let access_token = record + .access_token() + .ok_or_else(|| UsageProbeError::other("account is missing an access token"))?; + let account_id = record + .account_id() + .ok_or_else(|| UsageProbeError::other("account is missing an account id"))?; + let response = self + .client + .get(&self.usage_endpoint) + .bearer_auth(access_token) + .header("ChatGPT-Account-Id", account_id) + .header("User-Agent", CODEX_USER_AGENT) + .send() + .map_err(|error| UsageProbeError::other(error.to_string()))?; + let status = response.status(); + + if status == StatusCode::UNAUTHORIZED { + return Err(UsageProbeError::unauthorized()); + } + if !status.is_success() { + return Err(UsageProbeError::other(format!("usage endpoint returned {status}"))); + } + + let payload = response.json::().map_err(|error| { + UsageProbeError::other(format!("usage JSON did not parse: {error}")) + })?; + + Ok(usage_snapshot_from_payload(&payload, OffsetDateTime::now_utc().unix_timestamp())) + } + + fn refresh_record(&self, record: &mut AccountPoolRecord) -> crate::prelude::Result<()> { + let display_name = record.display_name(); + let refresh_token = record + .refresh_token() + .ok_or_else(|| { + eyre::eyre!( + "Codex account `{}` cannot refresh because no refresh token is present.", + display_name + ) + })? + .to_owned(); + let response = self + .client + .post(&self.refresh_endpoint) + .header("Content-Type", "application/json") + .json(&RefreshRequest { + client_id: CHATGPT_OAUTH_CLIENT_ID, + grant_type: "refresh_token", + refresh_token, + }) + .send()?; + let status = response.status(); + + if !status.is_success() { + eyre::bail!( + "Codex account `{}` token refresh failed with HTTP {status}.", + display_name + ); + } + + let refresh_response = response.json::()?; + let tokens = record.tokens.as_mut().ok_or_else(|| { + eyre::eyre!("Codex account `{display_name}` is missing token storage.") + })?; + + if let Some(id_token) = refresh_response.id_token { + tokens.id_token = Some(id_token); + } + if let Some(access_token) = refresh_response.access_token { + tokens.access_token = access_token; + } + if let Some(refresh_token) = refresh_response.refresh_token { + tokens.refresh_token = refresh_token; + } + + if tokens.access_token.trim().is_empty() { + eyre::bail!( + "Codex account `{}` token refresh did not produce a usable access token.", + display_name + ); + } + + record.last_refresh = Some(OffsetDateTime::now_utc().format(&Rfc3339)?); + + Ok(()) + } + + fn remember_selected_account(&self, account_id: &str) -> crate::prelude::Result<()> { + let mut selected = self + .selected_account_id + .lock() + .map_err(|_| eyre::eyre!("Codex accounts selection lock was poisoned."))?; + + *selected = Some(account_id.to_owned()); + + Ok(()) + } + + fn selected_account_id(&self) -> crate::prelude::Result> { + self.selected_account_id + .lock() + .map(|selected| selected.clone()) + .map_err(|_| eyre::eyre!("Codex accounts selection lock was poisoned.")) + } +} + +impl CodexAccountProvider for CodexAccountPool { + fn select_account(&self) -> crate::prelude::Result { + let mut records = self.load_records()?; + + self.select_from_records(&mut records) + } + + fn refresh_account( + &self, + previous_account_id: Option<&str>, + ) -> crate::prelude::Result { + let mut records = self.load_records()?; + + self.refresh_from_records(&mut records, previous_account_id) + } +} + +pub(crate) struct CodexAccountLogin { + access_token: String, + account_id: String, + plan_type: Option, + summary: CodexAccountActivitySummary, + account_summaries: Vec, +} +impl CodexAccountLogin { + pub(crate) fn access_token(&self) -> &str { + &self.access_token + } + + pub(crate) fn account_id(&self) -> &str { + &self.account_id + } + + pub(crate) fn plan_type(&self) -> Option<&str> { + self.plan_type.as_deref() + } + + pub(crate) fn summary(&self) -> &CodexAccountActivitySummary { + &self.summary + } + + pub(crate) fn account_summaries(&self) -> &[CodexAccountActivitySummary] { + &self.account_summaries + } + + fn mark_selected(&mut self, selected_at_unix_epoch: i64) { + if self.summary.status == "available" { + self.summary.status = String::from("selected"); + } + + self.summary.selected_at_unix_epoch = Some(selected_at_unix_epoch); + } + + fn with_account_summaries( + mut self, + account_summaries: Vec, + ) -> Self { + self.account_summaries = account_summaries; + + self + } +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(untagged)] +enum AccountPoolLine { + Wrapped { + #[serde(skip_serializing_if = "Option::is_none")] + email: Option, + #[serde(default, skip_serializing_if = "is_false")] + disabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + cooldown_until_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + cooldown_until: Option, + auth: AuthDotJson, + }, + Flat(AccountPoolRecord), +} +impl AccountPoolLine { + fn into_record(self) -> AccountPoolRecord { + match self { + Self::Flat(record) => record, + Self::Wrapped { email, disabled, cooldown_until_unix_epoch, cooldown_until, auth } => + AccountPoolRecord { + email: first_nonblank_string(email, auth.email), + disabled, + cooldown_until_unix_epoch, + cooldown_until, + auth_mode: auth.auth_mode, + openai_api_key: auth.openai_api_key, + tokens: auth.tokens, + last_refresh: auth.last_refresh, + }, + } + } +} + +#[derive(Clone, Deserialize, Serialize)] +struct AuthDotJson { + #[serde(skip_serializing_if = "Option::is_none")] + email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_mode: Option, + #[serde(rename = "OPENAI_API_KEY", skip_serializing_if = "Option::is_none")] + openai_api_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + last_refresh: Option, +} + +#[derive(Clone, Deserialize, Serialize)] +struct AccountPoolRecord { + #[serde(skip_serializing_if = "Option::is_none")] + email: Option, + #[serde(default, skip_serializing_if = "is_false")] + disabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + cooldown_until_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + cooldown_until: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_mode: Option, + #[serde(rename = "OPENAI_API_KEY", skip_serializing_if = "Option::is_none")] + openai_api_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + last_refresh: Option, +} +impl AccountPoolRecord { + fn display_name(&self) -> String { + self.email() + .or_else(|| self.account_id().map(redact_account_id)) + .unwrap_or_else(|| String::from("unnamed account")) + } + + fn access_token(&self) -> Option<&str> { + self.tokens + .as_ref() + .map(|tokens| tokens.access_token.as_str()) + .filter(|token| !token.trim().is_empty()) + } + + fn refresh_token(&self) -> Option { + self.tokens + .as_ref() + .map(|tokens| tokens.refresh_token.as_str()) + .filter(|token| !token.trim().is_empty()) + .map(str::to_owned) + } + + fn account_id(&self) -> Option<&str> { + self.tokens + .as_ref() + .and_then(|tokens| tokens.account_id.as_deref()) + .filter(|account_id| !account_id.trim().is_empty()) + } + + fn email(&self) -> Option { + nonblank_string(self.email.as_deref()) + .or_else(|| { + self.tokens.as_ref().and_then(|tokens| nonblank_string(tokens.email.as_deref())) + }) + .or_else(|| { + self.tokens.as_ref().and_then(|tokens| jwt_email_claim(tokens.id_token.as_deref())) + }) + } + + fn configured_activity_summary( + &self, + now_unix_epoch: i64, + ) -> Option { + let account_fingerprint = + self.account_id().map(redact_account_id).or_else(|| self.email())?; + let status = if self.disabled { + "disabled" + } else if self + .cooldown_until_unix_epoch + .is_some_and(|cooldown_until| cooldown_until > now_unix_epoch) + { + "cooldown" + } else if self.access_token().is_none() { + "unusable" + } else { + "available" + }; + + Some(CodexAccountActivitySummary { + account_fingerprint, + email: self.email(), + status: String::from(status), + refresh_status: String::from("not_checked"), + cooldown_until_unix_epoch: self.cooldown_until_unix_epoch, + note: Some(String::from("configured account")), + ..CodexAccountActivitySummary::default() + }) + } + + fn login_from_usage( + &self, + usage: AccountUsageSnapshot, + refresh_status: &str, + ) -> crate::prelude::Result { + let access_token = self + .access_token() + .ok_or_else(|| { + eyre::eyre!("Codex account `{}` is missing an access token.", self.display_name()) + })? + .to_owned(); + let account_id = self + .account_id() + .ok_or_else(|| { + eyre::eyre!("Codex account `{}` is missing an account id.", self.display_name()) + })? + .to_owned(); + let summary = CodexAccountActivitySummary { + account_fingerprint: redact_account_id(&account_id), + email: self.email(), + plan_type: usage.plan_type.clone(), + status: if usage.is_limited() { + String::from("usage_limited") + } else { + String::from("available") + }, + refresh_status: refresh_status.to_owned(), + checked_at_unix_epoch: Some(usage.checked_at_unix_epoch), + selected_at_unix_epoch: None, + primary_window_seconds: usage.primary.as_ref().and_then(|window| window.window_seconds), + primary_remaining_percent: usage + .primary + .as_ref() + .map(|window| window.remaining_percent), + primary_resets_at_unix_epoch: usage + .primary + .as_ref() + .and_then(|window| window.resets_at_unix_epoch), + secondary_window_seconds: usage + .secondary + .as_ref() + .and_then(|window| window.window_seconds), + secondary_remaining_percent: usage + .secondary + .as_ref() + .map(|window| window.remaining_percent), + secondary_resets_at_unix_epoch: usage + .secondary + .as_ref() + .and_then(|window| window.resets_at_unix_epoch), + credits_has_credits: usage.credits.as_ref().map(|credits| credits.has_credits), + credits_unlimited: usage.credits.as_ref().map(|credits| credits.unlimited), + credits_balance: usage.credits.and_then(|credits| credits.balance), + rate_limit_reached_type: usage.rate_limit_reached_type, + cooldown_until_unix_epoch: self.cooldown_until_unix_epoch, + note: Some(String::from("usage probe ok")), + }; + + Ok(CodexAccountLogin { + access_token, + account_id, + plan_type: summary.plan_type.clone(), + summary, + account_summaries: Vec::new(), + }) + } + + fn activity_summary_from_usage( + &self, + usage: AccountUsageSnapshot, + refresh_status: &str, + ) -> crate::prelude::Result { + Ok(self.login_from_usage(usage, refresh_status)?.summary) + } + + fn probe_failed_activity_summary( + &self, + now_unix_epoch: i64, + refresh_status: &str, + error: &(dyn Error + '_), + ) -> CodexAccountActivitySummary { + let mut summary = self.configured_activity_summary(now_unix_epoch).unwrap_or_default(); + + summary.status = if refresh_status == "failed" { + String::from("unusable") + } else { + String::from("probe_failed") + }; + summary.refresh_status = refresh_status.to_owned(); + summary.note = Some(format!("usage probe failed: {error}")); + + summary + } +} + +#[derive(Clone, Deserialize, Serialize)] +struct CodexTokenData { + #[serde(skip_serializing_if = "Option::is_none")] + email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + id_token: Option, + access_token: String, + refresh_token: String, + #[serde(skip_serializing_if = "Option::is_none")] + account_id: Option, +} + +#[derive(Serialize)] +struct RefreshRequest { + client_id: &'static str, + grant_type: &'static str, + refresh_token: String, +} + +#[derive(Deserialize)] +struct RefreshResponse { + id_token: Option, + access_token: Option, + refresh_token: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct AccountUsageSnapshot { + plan_type: Option, + primary: Option, + secondary: Option, + credits: Option, + rate_limit_reached_type: Option, + checked_at_unix_epoch: i64, +} +impl AccountUsageSnapshot { + fn is_limited(&self) -> bool { + self.rate_limit_reached_type.is_some() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct UsageWindow { + window_seconds: Option, + remaining_percent: i64, + resets_at_unix_epoch: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct CreditsSnapshot { + has_credits: bool, + unlimited: bool, + balance: Option, +} + +#[derive(Debug)] +struct UsageProbeError { + unauthorized: bool, + message: String, +} +impl UsageProbeError { + fn unauthorized() -> Self { + Self { unauthorized: true, message: String::from("usage endpoint returned 401") } + } + + fn other(message: impl Into) -> Self { + Self { unauthorized: false, message: message.into() } + } +} +impl Display for UsageProbeError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} +impl Error for UsageProbeError {} + +fn parse_account_records( + input: &str, + path: &Path, +) -> crate::prelude::Result> { + let mut records = Vec::new(); + + for (line_index, line) in input.lines().enumerate() { + let line_number = line_index + 1; + let trimmed = line.trim(); + + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + let parsed = serde_json::from_str::(trimmed).map_err(|error| { + eyre::eyre!( + "Codex accounts `{}` line {line_number} is not a valid auth JSONL entry: {error}", + path.display() + ) + })?; + + records.push(parsed.into_record()); + } + + if records.is_empty() { + eyre::bail!("Codex accounts `{}` does not contain any account records.", path.display()); + } + + Ok(records) +} + +fn usage_snapshot_from_payload( + payload: &Value, + checked_at_unix_epoch: i64, +) -> AccountUsageSnapshot { + let rate_limit = payload.get("rate_limit").filter(|value| !value.is_null()); + + AccountUsageSnapshot { + plan_type: payload.get("plan_type").and_then(json_scalar_to_string), + primary: rate_limit + .and_then(|details| usage_window_from_value(details.get("primary_window"))), + secondary: rate_limit + .and_then(|details| usage_window_from_value(details.get("secondary_window"))), + credits: payload.get("credits").and_then(credits_from_value), + rate_limit_reached_type: rate_limit_reached_type_from_payload(payload), + checked_at_unix_epoch, + } +} + +fn usage_window_from_value(value: Option<&Value>) -> Option { + let value = value.filter(|value| !value.is_null())?; + let used_percent = number_as_i64(value.get("used_percent")?)?; + let remaining_percent = 100_i64.saturating_sub(used_percent).clamp(0, 100); + + Some(UsageWindow { + window_seconds: value.get("limit_window_seconds").and_then(number_as_i64), + remaining_percent, + resets_at_unix_epoch: value.get("reset_at").and_then(number_as_i64), + }) +} + +fn credits_from_value(value: &Value) -> Option { + if value.is_null() { + return None; + } + + Some(CreditsSnapshot { + has_credits: value.get("has_credits").and_then(Value::as_bool).unwrap_or(true), + unlimited: value.get("unlimited").and_then(Value::as_bool).unwrap_or(false), + balance: value.get("balance").and_then(json_scalar_to_string), + }) +} + +fn rate_limit_reached_type_from_payload(payload: &Value) -> Option { + let reached = payload.get("rate_limit_reached_type").filter(|value| !value.is_null())?; + + if let Some(kind) = reached.get("kind").and_then(json_scalar_to_string) { + return Some(kind); + } + + json_scalar_to_string(reached) +} + +fn number_as_i64(value: &Value) -> Option { + value + .as_i64() + .or_else(|| value.as_u64().and_then(|number| i64::try_from(number).ok())) + .or_else(|| value.as_f64().map(|number| number.round() as i64)) +} + +fn json_scalar_to_string(value: &Value) -> Option { + match value { + Value::String(text) if !text.is_empty() => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(value) => Some(value.to_string()), + _ => None, + } +} + +fn first_nonblank_string(left: Option, right: Option) -> Option { + left.filter(|value| !value.trim().is_empty()) + .or_else(|| right.filter(|value| !value.trim().is_empty())) +} + +fn nonblank_string(value: Option<&str>) -> Option { + value.map(str::trim).filter(|value| !value.is_empty()).map(str::to_owned) +} + +fn jwt_email_claim(id_token: Option<&str>) -> Option { + let payload = id_token?.split('.').nth(1)?; + let payload_bytes = parse_base64_url(payload)?; + let claims = serde_json::from_slice::(&payload_bytes).ok()?; + + claims.get("email").and_then(json_scalar_to_string) +} + +fn parse_base64_url(input: &str) -> Option> { + let mut output = Vec::with_capacity(input.len() * 3 / 4); + let mut accumulator = 0_u32; + let mut bits = 0_u32; + + for byte in input.bytes().take_while(|byte| *byte != b'=') { + accumulator = (accumulator << 6) | u32::from(base64_url_value(byte)?); + bits += 6; + + if bits >= 8 { + bits -= 8; + + output.push(((accumulator >> bits) & 0xff) as u8); + } + } + + Some(output) +} + +const fn base64_url_value(byte: u8) -> Option { + match byte { + b'A'..=b'Z' => Some(byte - b'A'), + b'a'..=b'z' => Some(byte - b'a' + 26), + b'0'..=b'9' => Some(byte - b'0' + 52), + b'-' => Some(62), + b'_' => Some(63), + _ => None, + } +} + +fn compare_account_candidates(left: &CodexAccountLogin, right: &CodexAccountLogin) -> Ordering { + account_candidate_score(right) + .cmp(&account_candidate_score(left)) + .then_with(|| left.summary.account_fingerprint.cmp(&right.summary.account_fingerprint)) +} + +fn account_candidate_score(candidate: &CodexAccountLogin) -> i64 { + let summary = candidate.summary(); + let primary = summary.primary_remaining_percent.unwrap_or(0); + let secondary = summary.secondary_remaining_percent.unwrap_or(primary); + let mut score = primary.saturating_mul(1_000).saturating_add(secondary.saturating_mul(10)); + + if summary.rate_limit_reached_type.is_some() { + score = score.saturating_sub(50_000); + } + + score +} + +fn account_summaries( + selected: &CodexAccountLogin, + candidates: &[CodexAccountLogin], +) -> Vec { + let mut summaries = Vec::with_capacity(candidates.len() + 1); + + summaries.push(selected.summary().clone()); + summaries.extend(candidates.iter().map(|candidate| candidate.summary().clone())); + + summaries +} + +fn redact_account_id(account_id: &str) -> String { + let tail = + account_id.chars().rev().take(6).collect::>().into_iter().rev().collect::(); + + if tail.is_empty() { String::from("unknown") } else { format!("...{tail}") } +} + +const fn is_false(value: &bool) -> bool { + !*value +} + +#[cfg(test)] +mod tests { + use crate::agent::codex_accounts::{ + self, CodexAccountActivitySummary, CodexAccountLogin, CreditsSnapshot, Path, UsageWindow, + compare_account_candidates, + }; + + #[test] + fn accounts_accept_flat_and_wrapped_auth_jsonl_records() { + let input = r#" + {"email":"primary@example.com","auth_mode":"chatgpt","tokens":{"id_token":"id","access_token":"access","refresh_token":"refresh","account_id":"acct_primary"}} + {"auth":{"auth_mode":"chatgpt","tokens":{"id_token":"x.eyJlbWFpbCI6IndyYXBwZWRAZXhhbXBsZS5jb20ifQ.y","access_token":"access-2","refresh_token":"refresh-2","account_id":"acct_wrapped"}}} + "#; + let records = + codex_accounts::parse_account_records(input, Path::new("/tmp/accounts.jsonl")) + .expect("records should parse"); + + assert_eq!(records.len(), 2); + assert_eq!(records[0].account_id(), Some("acct_primary")); + assert_eq!(records[0].email().as_deref(), Some("primary@example.com")); + assert_eq!(records[1].account_id(), Some("acct_wrapped")); + assert_eq!(records[1].email().as_deref(), Some("wrapped@example.com")); + } + + #[test] + fn usage_summary_parses_codex_rate_limit_payload() { + let payload = serde_json::json!({ + "plan_type": "pro", + "rate_limit": { + "primary_window": { + "used_percent": 42, + "limit_window_seconds": 18_000, + "reset_at": 1_800_018_000 + }, + "secondary_window": { + "used_percent": 84, + "limit_window_seconds": 604_800, + "reset_at": 1_800_604_800 + } + }, + "credits": { + "has_credits": true, + "unlimited": false, + "balance": "9.99" + }, + "rate_limit_reached_type": { + "kind": "workspace_member_credits_depleted" + } + }); + let summary = codex_accounts::usage_snapshot_from_payload(&payload, 1_800_000_000); + + assert_eq!(summary.plan_type.as_deref(), Some("pro")); + assert_eq!( + summary.primary, + Some(UsageWindow { + window_seconds: Some(18_000), + remaining_percent: 58, + resets_at_unix_epoch: Some(1_800_018_000), + }) + ); + assert_eq!( + summary.secondary, + Some(UsageWindow { + window_seconds: Some(604_800), + remaining_percent: 16, + resets_at_unix_epoch: Some(1_800_604_800), + }) + ); + assert_eq!( + summary.credits, + Some(CreditsSnapshot { + has_credits: true, + unlimited: false, + balance: Some(String::from("9.99")), + }) + ); + assert_eq!( + summary.rate_limit_reached_type.as_deref(), + Some("workspace_member_credits_depleted") + ); + } + + #[test] + fn usage_limit_requires_explicit_rate_limit_signal() { + let payload = serde_json::json!({ + "plan_type": "pro", + "rate_limit": { + "primary_window": { + "used_percent": 0, + "limit_window_seconds": 18_000, + "reset_at": 1_800_018_000 + }, + "secondary_window": { + "used_percent": 0, + "limit_window_seconds": 604_800, + "reset_at": 1_800_604_800 + } + }, + "credits": { + "has_credits": false, + "unlimited": false, + "balance": "0" + }, + "rate_limit_reached_type": null + }); + let summary = codex_accounts::usage_snapshot_from_payload(&payload, 1_800_000_000); + + assert_eq!(summary.primary.as_ref().map(|window| window.remaining_percent), Some(100)); + assert_eq!(summary.secondary.as_ref().map(|window| window.remaining_percent), Some(100)); + assert_eq!(summary.credits.as_ref().map(|credits| credits.has_credits), Some(false)); + assert!(!summary.is_limited()); + } + + #[test] + fn account_candidate_sort_prefers_remaining_usage() { + let mut candidates = [ + CodexAccountLogin { + access_token: String::from("a"), + account_id: String::from("acct_a"), + plan_type: Some(String::from("pro")), + summary: CodexAccountActivitySummary { + account_fingerprint: String::from("...acct_a"), + primary_remaining_percent: Some(10), + secondary_remaining_percent: Some(90), + ..CodexAccountActivitySummary::default() + }, + account_summaries: Vec::new(), + }, + CodexAccountLogin { + access_token: String::from("b"), + account_id: String::from("acct_b"), + plan_type: Some(String::from("pro")), + summary: CodexAccountActivitySummary { + account_fingerprint: String::from("...acct_b"), + primary_remaining_percent: Some(70), + secondary_remaining_percent: Some(40), + ..CodexAccountActivitySummary::default() + }, + account_summaries: Vec::new(), + }, + ]; + + candidates.sort_by(compare_account_candidates); + + assert_eq!(candidates[0].account_id(), "acct_b"); + } + + #[test] + fn account_candidate_sort_does_not_penalize_missing_credits() { + let mut candidates = [ + CodexAccountLogin { + access_token: String::from("a"), + account_id: String::from("acct_a"), + plan_type: Some(String::from("pro")), + summary: CodexAccountActivitySummary { + account_fingerprint: String::from("...acct_a"), + primary_remaining_percent: Some(86), + secondary_remaining_percent: Some(97), + credits_has_credits: Some(true), + credits_unlimited: Some(false), + ..CodexAccountActivitySummary::default() + }, + account_summaries: Vec::new(), + }, + CodexAccountLogin { + access_token: String::from("b"), + account_id: String::from("acct_b"), + plan_type: Some(String::from("pro")), + summary: CodexAccountActivitySummary { + account_fingerprint: String::from("...acct_b"), + primary_remaining_percent: Some(100), + secondary_remaining_percent: Some(100), + credits_has_credits: Some(false), + credits_unlimited: Some(false), + ..CodexAccountActivitySummary::default() + }, + account_summaries: Vec::new(), + }, + ]; + + candidates.sort_by(compare_account_candidates); + + assert_eq!(candidates[0].account_id(), "acct_b"); + } +} diff --git a/apps/decodex/src/agent/decodex_tool_bridge.rs b/apps/decodex/src/agent/decodex_tool_bridge.rs new file mode 100644 index 00000000..46823eb1 --- /dev/null +++ b/apps/decodex/src/agent/decodex_tool_bridge.rs @@ -0,0 +1,233 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{self, Value}; + +use crate::{ + agent::tracker_tool_bridge::{ + DynamicToolCallResponse, DynamicToolHandler, DynamicToolSpec, TurnCompletionStatus, + }, + prelude::Result, +}; + +pub(crate) const DECODEX_RUN_CONTEXT_TOOL_NAME: &str = "decodex_run_context"; +pub(crate) const DECODEX_RUN_CONTEXT_NAMESPACE: &str = "decodex"; + +/// Client-side Decodex tools that are local to one app-server run attempt. +pub(crate) struct DecodexToolBridge<'a> { + tracker_tools: &'a dyn DynamicToolHandler, + run_context: DecodexRunContext, +} +impl<'a> DecodexToolBridge<'a> { + pub(crate) fn new( + tracker_tools: &'a dyn DynamicToolHandler, + run_context: DecodexRunContext, + ) -> Self { + Self { tracker_tools, run_context } + } + + fn handle_run_context(&self, arguments: Value) -> DynamicToolCallResponse { + if let Err(error) = serde_json::from_value::(arguments) { + return DynamicToolCallResponse::failure(format!( + "Invalid `{DECODEX_RUN_CONTEXT_TOOL_NAME}` arguments: {error}" + )); + } + + match serde_json::to_string(&self.run_context) { + Ok(response) => DynamicToolCallResponse::success(response), + Err(error) => DynamicToolCallResponse::failure(format!( + "Failed to serialize `{DECODEX_RUN_CONTEXT_TOOL_NAME}` response: {error}" + )), + } + } +} + +impl DynamicToolHandler for DecodexToolBridge<'_> { + fn tool_specs(&self) -> Vec { + let mut specs = self.tracker_tools.tool_specs(); + let mut run_context_spec = DynamicToolSpec::new( + DECODEX_RUN_CONTEXT_TOOL_NAME, + "Return the current Decodex run, issue, branch, worktree, and repo-gate context for this app-server attempt.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ) + .deferred(); + + run_context_spec.namespace = Some(String::from(DECODEX_RUN_CONTEXT_NAMESPACE)); + + specs.push(run_context_spec); + + specs + } + + fn handle_call(&self, tool_name: &str, arguments: Value) -> DynamicToolCallResponse { + self.handle_call_with_namespace(None, tool_name, arguments) + } + + fn handle_call_with_namespace( + &self, + namespace: Option<&str>, + tool_name: &str, + arguments: Value, + ) -> DynamicToolCallResponse { + if namespace == Some(DECODEX_RUN_CONTEXT_NAMESPACE) + && tool_name == DECODEX_RUN_CONTEXT_TOOL_NAME + { + return self.handle_run_context(arguments); + } + + self.tracker_tools.handle_call_with_namespace(namespace, tool_name, arguments) + } + + fn classify_turn_completion(&self, final_output: &str) -> Result { + self.tracker_tools.classify_turn_completion(final_output) + } + + fn validate_turn_completion(&self, final_output: &str) -> Result<()> { + self.tracker_tools.validate_turn_completion(final_output) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DecodexRunContext { + pub(crate) run_id: String, + pub(crate) attempt_number: i64, + pub(crate) issue_id: String, + pub(crate) issue_identifier: String, + pub(crate) branch: String, + pub(crate) worktree_path: String, + pub(crate) max_turns: u32, + pub(crate) default_canonicalize_commands: Vec, + pub(crate) default_verify_commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct EmptyToolArgs {} + +#[cfg(test)] +mod tests { + use serde_json::Value; + + use crate::agent::{ + decodex_tool_bridge::{ + DECODEX_RUN_CONTEXT_NAMESPACE, DECODEX_RUN_CONTEXT_TOOL_NAME, DecodexRunContext, + DecodexToolBridge, + }, + tracker_tool_bridge::{ + DynamicToolCallResponse, DynamicToolContentItem, DynamicToolHandler, DynamicToolSpec, + }, + }; + + struct FakeTrackerTools; + impl DynamicToolHandler for FakeTrackerTools { + fn tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + "issue_comment", + "Add a comment.", + serde_json::json!({ + "type": "object", + "additionalProperties": false + }), + )] + } + + fn handle_call(&self, tool_name: &str, _arguments: Value) -> DynamicToolCallResponse { + DynamicToolCallResponse::success(format!("delegated {tool_name}")) + } + } + + fn sample_bridge() -> DecodexToolBridge<'static> { + DecodexToolBridge::new( + &FakeTrackerTools, + DecodexRunContext { + run_id: String::from("run-1"), + attempt_number: 2, + issue_id: String::from("issue-1"), + issue_identifier: String::from("XY-449"), + branch: String::from("y/decodex-xy-449"), + worktree_path: String::from("/tmp/worktree"), + max_turns: 3, + default_canonicalize_commands: vec![String::from("cargo make fmt")], + default_verify_commands: vec![String::from("cargo make test")], + }, + ) + } + + #[test] + fn publishes_deferred_run_context_tool_with_protocol_safe_name() { + let specs = sample_bridge().tool_specs(); + let run_context = specs + .iter() + .find(|spec| spec.name == DECODEX_RUN_CONTEXT_TOOL_NAME) + .expect("run context tool should be published"); + let protocol_identifier_is_safe = |name: &str| { + !name.is_empty() + && name.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + }; + + assert!(run_context.defer_loading); + assert_eq!(run_context.namespace.as_deref(), Some(DECODEX_RUN_CONTEXT_NAMESPACE)); + assert!(protocol_identifier_is_safe(&run_context.name)); + assert!(protocol_identifier_is_safe( + run_context.namespace.as_deref().expect("run context tool should be namespaced") + )); + assert!(specs.iter().all(|spec| protocol_identifier_is_safe(&spec.name))); + assert!( + specs + .iter() + .filter_map(|spec| spec.namespace.as_deref()) + .all(protocol_identifier_is_safe) + ); + } + + #[test] + fn returns_run_context_json() { + let response = sample_bridge().handle_call_with_namespace( + Some(DECODEX_RUN_CONTEXT_NAMESPACE), + DECODEX_RUN_CONTEXT_TOOL_NAME, + serde_json::json!({}), + ); + + assert!(response.success); + + let [DynamicToolContentItem::InputText { text }] = response.content_items.as_slice() else { + panic!("run context should return one text item"); + }; + let value: Value = serde_json::from_str(text).expect("run context should be JSON"); + + assert_eq!(value["runId"], "run-1"); + assert_eq!(value["issueIdentifier"], "XY-449"); + assert_eq!(value["defaultVerifyCommands"][0], "cargo make test"); + } + + #[test] + fn validates_run_context_arguments() { + let response = sample_bridge().handle_call_with_namespace( + Some(DECODEX_RUN_CONTEXT_NAMESPACE), + DECODEX_RUN_CONTEXT_TOOL_NAME, + serde_json::json!({ "unexpected": true }), + ); + + assert!(!response.success); + assert!(matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText { text }] + if text.contains("Invalid `decodex_run_context` arguments") + )); + } + + #[test] + fn delegates_tracker_tools() { + let response = sample_bridge().handle_call("issue_comment", serde_json::json!({})); + + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: String::from("delegated issue_comment") + }] + ); + } +} diff --git a/apps/decodex/src/agent/json_rpc.rs b/apps/decodex/src/agent/json_rpc.rs new file mode 100644 index 00000000..525e8cb6 --- /dev/null +++ b/apps/decodex/src/agent/json_rpc.rs @@ -0,0 +1,939 @@ +use std::{ + collections::VecDeque, + env, + ffi::OsString, + fmt::{self, Display, Formatter}, + io::{self, BufRead as _, BufReader, Write as _}, + path::{Path, PathBuf}, + process::{Child, ChildStdin, Command, Stdio}, + sync::{ + Arc, Mutex, + mpsc::{self, Receiver, RecvTimeoutError}, + }, + thread, + time::Duration, +}; + +use color_eyre::{Report, eyre}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{self, Value}; + +use crate::git_credentials::{GitCredentialEnvironment, GitSigningConfig}; + +const APP_SERVER_STDERR_TAIL_LINES: usize = 20; +const CODEX_HOME_ENV_VAR: &str = "CODEX_HOME"; +const CODEX_SQLITE_HOME_ENV_VAR: &str = "CODEX_SQLITE_HOME"; +const CODEX_HOME_DIR_NAME: &str = ".codex"; + +#[derive(Clone, Eq, PartialEq)] +pub(crate) struct AppServerProcessEnv { + git: GitCredentialEnvironment, + codex_home_policy: AppServerCodexHomePolicy, +} +impl AppServerProcessEnv { + #[cfg(test)] + pub(crate) fn with_github_credentials( + github_token_env_var: String, + github_token: String, + git_askpass_path: PathBuf, + ) -> Self { + Self { + git: GitCredentialEnvironment::with_github_credentials( + github_token_env_var, + github_token, + git_askpass_path, + ), + codex_home_policy: AppServerCodexHomePolicy::SharedDefault, + } + } + + pub(crate) fn with_github_credentials_and_signing_config( + github_token_env_var: String, + github_token: String, + git_askpass_path: PathBuf, + signing_config: GitSigningConfig, + ) -> Self { + Self { + git: GitCredentialEnvironment::with_github_credentials_and_signing_config( + github_token_env_var, + github_token, + git_askpass_path, + signing_config, + ), + codex_home_policy: AppServerCodexHomePolicy::SharedDefault, + } + } + + pub(crate) fn resolve_codex_home_env( + &self, + ) -> crate::prelude::Result { + match &self.codex_home_policy { + AppServerCodexHomePolicy::SharedDefault => resolve_shared_codex_home_env(), + #[cfg(test)] + AppServerCodexHomePolicy::Explicit(home_env) => Ok(home_env.clone()), + } + } + + pub(crate) fn apply_to( + &self, + command: &mut Command, + ) -> crate::prelude::Result { + self.git.apply_to(command); + + let codex_home_env = self.resolve_codex_home_env()?; + + codex_home_env.apply_to(command)?; + + Ok(codex_home_env) + } + + #[cfg(test)] + fn with_codex_home_for_test(home_env: ResolvedAppServerCodexHomeEnv) -> Self { + Self { + git: GitCredentialEnvironment::default(), + codex_home_policy: AppServerCodexHomePolicy::Explicit(home_env), + } + } +} + +impl Default for AppServerProcessEnv { + fn default() -> Self { + Self { + git: GitCredentialEnvironment::default(), + codex_home_policy: AppServerCodexHomePolicy::SharedDefault, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ResolvedAppServerCodexHomeEnv { + codex_home: PathBuf, + sqlite_home: PathBuf, +} +impl ResolvedAppServerCodexHomeEnv { + pub(crate) fn new(codex_home: PathBuf, sqlite_home: PathBuf) -> crate::prelude::Result { + validate_codex_home_path(CODEX_HOME_ENV_VAR, &codex_home)?; + validate_codex_home_path(CODEX_SQLITE_HOME_ENV_VAR, &sqlite_home)?; + + Ok(Self { codex_home, sqlite_home }) + } + + pub(crate) fn codex_home(&self) -> &Path { + &self.codex_home + } + + #[cfg(test)] + fn sqlite_home(&self) -> &Path { + &self.sqlite_home + } + + fn apply_to(&self, command: &mut Command) -> crate::prelude::Result<()> { + let codex_home = path_env_value(CODEX_HOME_ENV_VAR, &self.codex_home)?; + let sqlite_home = path_env_value(CODEX_SQLITE_HOME_ENV_VAR, &self.sqlite_home)?; + + command.env_remove(CODEX_HOME_ENV_VAR); + command.env_remove(CODEX_SQLITE_HOME_ENV_VAR); + command.env(CODEX_HOME_ENV_VAR, codex_home); + command.env(CODEX_SQLITE_HOME_ENV_VAR, sqlite_home); + + Ok(()) + } +} + +#[derive(Debug)] +pub(crate) struct AppServerHomePreflightFailure { + details: String, + kind: AppServerHomePreflightFailureKind, +} +impl AppServerHomePreflightFailure { + pub(crate) fn resolution_failed(details: String) -> Self { + Self { details, kind: AppServerHomePreflightFailureKind::ResolutionFailed } + } + + pub(crate) fn initialize_mismatch(resolved_home: String, expected_home: String) -> Self { + Self { + details: format!( + "app_server_protocol_failure: initialize codexHome `{resolved_home}` did not match expected shared Codex home `{expected_home}`; Decodex blocked dispatch before thread/start so Codex state is not split across homes." + ), + kind: AppServerHomePreflightFailureKind::InitializeMismatch, + } + } + + pub(crate) fn error_class(&self) -> &'static str { + match self.kind { + AppServerHomePreflightFailureKind::ResolutionFailed => + "app_server_codex_home_preflight_failed", + AppServerHomePreflightFailureKind::InitializeMismatch => + "app_server_codex_home_mismatch", + } + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + format!( + "inspect the local `decodex serve` HOME and app-server Codex home resolution, keep CODEX_HOME/CODEX_SQLITE_HOME shared instead of per-account, restart `decodex serve`, {recovery_gate}" + ) + } +} + +impl Display for AppServerHomePreflightFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.details) + } +} + +impl std::error::Error for AppServerHomePreflightFailure {} + +#[derive(Debug)] +pub(crate) struct AppServerOutputTimeout; +impl Display for AppServerOutputTimeout { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + formatter.write_str("Timed out while waiting for app-server output.") + } +} + +impl std::error::Error for AppServerOutputTimeout {} + +#[derive(Debug)] +pub(crate) struct AppServerTransportFailure { + details: String, +} +impl AppServerTransportFailure { + pub(crate) fn new(details: String) -> Self { + Self { details } + } + + pub(crate) fn error_class(&self) -> &'static str { + "app_server_transport_disconnected" + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + format!( + "inspect the local app-server stderr tail and process exit status, resolve the Codex app-server transport failure manually, {recovery_gate}" + ) + } +} + +impl Display for AppServerTransportFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.details) + } +} + +impl std::error::Error for AppServerTransportFailure {} + +pub(crate) struct JsonRpcConnection { + child: Child, + stdin: ChildStdin, + stdout_rx: Receiver, + stderr_tail: Arc>>, + pending_messages: VecDeque, + next_request_id: i64, +} +impl JsonRpcConnection { + pub(crate) fn spawn_app_server( + listen: &str, + process_env: &AppServerProcessEnv, + ) -> crate::prelude::Result { + let mut command = Command::new("codex"); + let _codex_home_env = configure_app_server_command(&mut command, listen, process_env)?; + let mut child = command.spawn()?; + let stdin = + child.stdin.take().ok_or_else(|| eyre::eyre!("Failed to capture app-server stdin."))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| eyre::eyre!("Failed to capture app-server stdout."))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| eyre::eyre!("Failed to capture app-server stderr."))?; + let (stdout_tx, stdout_rx) = mpsc::channel(); + let stderr_tail = Arc::new(Mutex::new(VecDeque::new())); + let _stdout_task = thread::spawn(move || { + let reader = BufReader::new(stdout); + + for line in reader.lines() { + match line { + Ok(line) => { + let line: String = line; + + if line.trim().is_empty() { + continue; + } + if stdout_tx.send(line).is_err() { + break; + } + }, + Err(error) => { + tracing::warn!(?error, "Failed to read app-server stdout."); + + break; + }, + } + } + }); + let stderr_tail_writer = Arc::clone(&stderr_tail); + let _stderr_task = thread::spawn(move || { + let reader = BufReader::new(stderr); + + for line in reader.lines() { + match line { + Ok(line) => { + let line: String = line; + let trimmed_line = line.trim().to_owned(); + + if trimmed_line.is_empty() { + continue; + } + + match stderr_tail_writer.lock() { + Ok(mut tail) => { + if tail.len() == APP_SERVER_STDERR_TAIL_LINES { + tail.pop_front(); + } + + tail.push_back(trimmed_line); + }, + Err(error) => { + tracing::warn!(?error, "Failed to retain app-server stderr tail."); + }, + } + + tracing::warn!(stderr = %line, "codex app-server stderr"); + }, + Err(error) => { + tracing::warn!(?error, "Failed to read app-server stderr."); + + break; + }, + } + } + }); + + Ok(Self { + child, + stdin, + stdout_rx, + stderr_tail, + pending_messages: VecDeque::new(), + next_request_id: 1, + }) + } + + #[allow(dead_code)] + pub(crate) fn request( + &mut self, + method: &str, + params: &P, + timeout: Duration, + ) -> crate::prelude::Result + where + P: Serialize, + T: DeserializeOwned, + { + self.request_with_handler(method, params, timeout, |_connection, _message, request| { + eyre::bail!( + "Unexpected inbound JSON-RPC request `{}` while waiting for `{method}`.", + request.method + ); + }) + } + + pub(crate) fn request_with_handler( + &mut self, + method: &str, + params: &P, + timeout: Duration, + mut handle_request: F, + ) -> crate::prelude::Result + where + P: Serialize, + T: DeserializeOwned, + F: FnMut(&mut Self, &WireMessage, &JsonRpcRequest) -> crate::prelude::Result<()>, + { + let request_id = self.next_request_id; + let expected_id = Value::from(request_id); + + self.next_request_id += 1; + + self.send_value(&serde_json::json!({ + "id": request_id, + "method": method, + "params": params, + }))?; + + loop { + let wire_message = self.read_message(Some(timeout))?; + + match &wire_message.message { + JsonRpcMessage::Notification(_) => self.pending_messages.push_back(wire_message), + JsonRpcMessage::Response(response) if response.id == expected_id => { + return Ok(serde_json::from_value(response.result.clone())?); + }, + JsonRpcMessage::Error(error) if error.id == expected_id => { + return Err(eyre::eyre!( + "`{method}` failed with {}: {}", + error.error.code, + error.error.message + )); + }, + JsonRpcMessage::Request(request) => handle_request(self, &wire_message, request)?, + JsonRpcMessage::Response(_) | JsonRpcMessage::Error(_) => { + return Err(eyre::eyre!( + "Received an unexpected JSON-RPC response while waiting for `{method}`." + )); + }, + } + } + } + + pub(crate) fn notify

( + &mut self, + method: &str, + params: Option<&P>, + ) -> crate::prelude::Result<()> + where + P: Serialize, + { + let value = match params { + Some(params) => serde_json::json!({ + "method": method, + "params": params, + }), + None => serde_json::json!({ "method": method }), + }; + + self.send_value(&value) + } + + pub(crate) fn recv( + &mut self, + timeout: Option, + ) -> crate::prelude::Result { + if let Some(message) = self.pending_messages.pop_front() { + return Ok(message); + } + + self.read_message(timeout) + } + + pub(crate) fn respond(&mut self, id: &Value, result: &R) -> crate::prelude::Result<()> + where + R: Serialize, + { + self.send_value(&serde_json::json!({ + "id": id, + "result": result, + })) + } + + pub(crate) fn respond_error( + &mut self, + id: &Value, + code: i64, + message: &str, + ) -> crate::prelude::Result<()> { + self.send_value(&serde_json::json!({ + "id": id, + "error": { + "code": code, + "message": message, + }, + })) + } + + pub(crate) fn drain_pending(&mut self) -> Vec { + self.pending_messages.drain(..).collect() + } + + fn send_value(&mut self, value: &Value) -> crate::prelude::Result<()> { + let payload = serde_json::to_string(value)?; + + if let Err(error) = writeln!(self.stdin, "{payload}") { + return Err(self.app_server_stdin_error("write", error)); + } + if let Err(error) = self.stdin.flush() { + return Err(self.app_server_stdin_error("flush", error)); + } + + Ok(()) + } + + fn read_message(&mut self, timeout: Option) -> crate::prelude::Result { + let raw = match timeout { + Some(timeout) => match self.stdout_rx.recv_timeout(timeout) { + Ok(raw) => raw, + Err(RecvTimeoutError::Timeout) => { + return Err(Report::new(AppServerOutputTimeout)); + }, + Err(RecvTimeoutError::Disconnected) => { + return Err(self.app_server_disconnect_error()); + }, + }, + None => self.stdout_rx.recv().map_err(|_| self.app_server_disconnect_error())?, + }; + + WireMessage::parse(raw) + } + + fn app_server_disconnect_error(&mut self) -> Report { + let details = self.app_server_transport_error_details( + "App-server stdout disconnected unexpectedly".to_owned(), + ); + + Report::new(AppServerTransportFailure::new(details)) + } + + fn app_server_stdin_error(&mut self, operation: &str, error: io::Error) -> Report { + let details = self.app_server_transport_error_details(format!( + "App-server stdin {operation} failed: {error}" + )); + + Report::new(AppServerTransportFailure::new(details)) + } + + fn app_server_transport_error_details(&mut self, summary: String) -> String { + let process_status = match self.child.try_wait() { + Ok(Some(status)) => format!("process exited with `{status}`"), + Ok(None) => String::from("process was still running"), + Err(error) => format!("failed to inspect process status: {error}"), + }; + let stderr_tail = self.stderr_tail_snapshot(); + let mut details = format!("{summary} ({process_status})."); + + if !stderr_tail.is_empty() { + details.push_str(" Recent app-server stderr tail:"); + + for line in stderr_tail { + details.push_str("\n "); + details.push_str(&line); + } + } + + details + } + + fn stderr_tail_snapshot(&self) -> Vec { + match self.stderr_tail.lock() { + Ok(tail) => tail.iter().cloned().collect(), + Err(error) => { + tracing::warn!(?error, "Failed to read app-server stderr tail."); + + Vec::new() + }, + } + } +} + +impl Drop for JsonRpcConnection { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +#[derive(Clone, Debug)] +pub(crate) struct WireMessage { + pub(crate) raw: String, + pub(crate) message: JsonRpcMessage, +} +impl WireMessage { + fn parse(raw: String) -> crate::prelude::Result { + let value: Value = serde_json::from_str(&raw)?; + let message = if value.get("method").is_some() && value.get("id").is_some() { + JsonRpcMessage::Request(serde_json::from_value(value)?) + } else if value.get("method").is_some() { + JsonRpcMessage::Notification(serde_json::from_value(value)?) + } else if value.get("error").is_some() { + JsonRpcMessage::Error(serde_json::from_value(value)?) + } else if value.get("result").is_some() { + JsonRpcMessage::Response(serde_json::from_value(value)?) + } else { + return Err(eyre::eyre!("Received an unrecognized JSON-RPC payload: {raw}")); + }; + + Ok(Self { raw, message }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct JsonRpcRequest { + pub(crate) id: Value, + pub(crate) method: String, + #[serde(default)] + pub(crate) params: Value, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct JsonRpcNotification { + pub(crate) method: String, + #[serde(default)] + pub(crate) params: Value, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct JsonRpcResponse { + pub(crate) id: Value, + pub(crate) result: Value, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct JsonRpcError { + pub(crate) id: Value, + pub(crate) error: JsonRpcErrorPayload, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct JsonRpcErrorPayload { + pub(crate) code: i64, + pub(crate) message: String, +} + +#[derive(Clone, Debug)] +pub(crate) enum JsonRpcMessage { + Request(JsonRpcRequest), + Notification(JsonRpcNotification), + Response(JsonRpcResponse), + Error(JsonRpcError), +} + +#[derive(Debug)] +enum AppServerHomePreflightFailureKind { + ResolutionFailed, + InitializeMismatch, +} + +#[derive(Clone, Eq, PartialEq)] +enum AppServerCodexHomePolicy { + SharedDefault, + #[cfg(test)] + Explicit(ResolvedAppServerCodexHomeEnv), +} + +fn resolve_shared_codex_home_env() -> crate::prelude::Result { + resolve_shared_codex_home_env_from_home(env::var_os("HOME")) +} + +fn resolve_shared_codex_home_env_from_home( + home: Option, +) -> crate::prelude::Result { + let Some(home) = home else { + return Err(Report::new(AppServerHomePreflightFailure::resolution_failed(String::from( + "app_server_preflight_failed: HOME is not set, so Decodex cannot resolve the shared Codex home for app-server dispatch.", + )))); + }; + let home = PathBuf::from(home); + + if home.as_os_str().is_empty() { + return Err(Report::new(AppServerHomePreflightFailure::resolution_failed(String::from( + "app_server_preflight_failed: HOME is empty, so Decodex cannot resolve the shared Codex home for app-server dispatch.", + )))); + } + if !home.is_absolute() { + return Err(Report::new(AppServerHomePreflightFailure::resolution_failed(format!( + "app_server_preflight_failed: HOME `{}` is not absolute, so Decodex cannot resolve the shared Codex home for app-server dispatch.", + home.display() + )))); + } + + let codex_home = home.join(CODEX_HOME_DIR_NAME); + + ResolvedAppServerCodexHomeEnv::new(codex_home.clone(), codex_home) +} + +fn validate_codex_home_path(name: &str, path: &Path) -> crate::prelude::Result<()> { + if path.as_os_str().is_empty() { + return Err(Report::new(AppServerHomePreflightFailure::resolution_failed(format!( + "app_server_preflight_failed: {name} resolved to an empty path." + )))); + } + if !path.is_absolute() { + return Err(Report::new(AppServerHomePreflightFailure::resolution_failed(format!( + "app_server_preflight_failed: {name} `{}` is not absolute.", + path.display() + )))); + } + + path_env_value(name, path).map(|_| ()) +} + +fn path_env_value(name: &str, path: &Path) -> crate::prelude::Result { + path.to_str().map(str::to_owned).ok_or_else(|| { + Report::new(AppServerHomePreflightFailure::resolution_failed(format!( + "app_server_preflight_failed: {name} `{}` is not valid UTF-8.", + path.display() + ))) + }) +} + +fn configure_app_server_command( + command: &mut Command, + listen: &str, + process_env: &AppServerProcessEnv, +) -> crate::prelude::Result { + command + .args(["app-server", "--listen", listen]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + process_env.apply_to(command) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{HashMap, VecDeque}, + ffi::OsString, + path::PathBuf, + process::{Command, Stdio}, + sync::{Arc, Mutex, mpsc}, + }; + + use serde_json::json; + + use crate::agent::json_rpc::{ + AppServerHomePreflightFailure, AppServerOutputTimeout, AppServerProcessEnv, + AppServerTransportFailure, JsonRpcConnection, JsonRpcMessage, + ResolvedAppServerCodexHomeEnv, WireMessage, + }; + + #[test] + fn parses_notification_messages() { + let message = WireMessage::parse( + r#"{"method":"thread/status/changed","params":{"threadId":"thread-1"}}"#.to_owned(), + ) + .expect("notification should parse"); + + match message.message { + JsonRpcMessage::Notification(notification) => { + assert_eq!(notification.method, "thread/status/changed"); + assert_eq!(notification.params["threadId"], json!("thread-1")); + }, + other => panic!("unexpected message: {other:?}"), + } + } + + #[test] + fn parses_response_messages() { + let message = + WireMessage::parse(r#"{"id":1,"result":{"userAgent":"decodex-test"}}"#.to_owned()) + .expect("response should parse"); + + match message.message { + JsonRpcMessage::Response(response) => { + assert_eq!(response.id, json!(1)); + assert_eq!(response.result["userAgent"], json!("decodex-test")); + }, + other => panic!("unexpected message: {other:?}"), + } + } + + #[test] + fn app_server_command_inherits_noninteractive_git_environment() { + let process_env = AppServerProcessEnv::with_github_credentials( + String::from("GITHUB_PAT_Y"), + String::from("ghp_test_token"), + PathBuf::from("/tmp/decodex-askpass.sh"), + ); + let mut command = Command::new("codex"); + + super::configure_app_server_command(&mut command, "stdio://", &process_env) + .expect("app-server command should configure"); + + let args = + command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect::>(); + let envs = command + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .collect::>(); + + assert_eq!(args, ["app-server", "--listen", "stdio://"]); + assert_eq!(envs.get("GH_TOKEN").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GITHUB_TOKEN").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GITHUB_PAT_Y").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GH_PROMPT_DISABLED").map(String::as_str), Some("1")); + assert_eq!(envs.get("GIT_TERMINAL_PROMPT").map(String::as_str), Some("0")); + assert_eq!(envs.get("GCM_INTERACTIVE").map(String::as_str), Some("never")); + assert_eq!(envs.get("GIT_ASKPASS").map(String::as_str), Some("/tmp/decodex-askpass.sh")); + assert_eq!(envs.get("GIT_CONFIG_COUNT").map(String::as_str), Some("9")); + assert_eq!( + envs.get("GIT_CONFIG_KEY_1").map(String::as_str), + Some("url.https://github.com/.insteadOf") + ); + assert_eq!(envs.get("GIT_CONFIG_VALUE_1").map(String::as_str), Some("git@github.com-x:")); + assert_eq!( + envs.get("GIT_CONFIG_KEY_5").map(String::as_str), + Some("url.https://github.com/.insteadOf") + ); + assert_eq!( + envs.get("GIT_CONFIG_VALUE_5").map(String::as_str), + Some("ssh://git@github.com-y/") + ); + assert_eq!(envs.get("GIT_CONFIG_KEY_6").map(String::as_str), Some("commit.gpgsign")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_6").map(String::as_str), Some("false")); + assert_eq!(envs.get("GIT_CONFIG_KEY_7").map(String::as_str), Some("tag.gpgsign")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_7").map(String::as_str), Some("false")); + assert_eq!(envs.get("GIT_CONFIG_KEY_8").map(String::as_str), Some("user.signingkey")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_8").map(String::as_str), Some("")); + } + + #[test] + fn app_server_command_does_not_rewrite_git_urls_without_credentials() { + let mut command = Command::new("codex"); + + super::configure_app_server_command( + &mut command, + "stdio://", + &AppServerProcessEnv::default(), + ) + .expect("app-server command should configure"); + + let envs = command + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .collect::>(); + + assert_eq!(envs.get("GH_PROMPT_DISABLED").map(String::as_str), Some("1")); + assert_eq!(envs.get("GIT_TERMINAL_PROMPT").map(String::as_str), Some("0")); + assert_eq!(envs.get("GCM_INTERACTIVE").map(String::as_str), Some("never")); + assert!(!envs.contains_key("GIT_CONFIG_COUNT")); + assert!(!envs.keys().any(|key| key.starts_with("GIT_CONFIG_KEY_"))); + assert!(!envs.keys().any(|key| key.starts_with("GIT_CONFIG_VALUE_"))); + } + + #[test] + fn app_server_command_sets_shared_codex_homes() { + let shared_home = PathBuf::from("/Users/test/.codex"); + let home_env = ResolvedAppServerCodexHomeEnv::new(shared_home.clone(), shared_home.clone()) + .expect("test home should validate"); + let process_env = AppServerProcessEnv::with_codex_home_for_test(home_env); + let mut command = Command::new("codex"); + let resolved = super::configure_app_server_command(&mut command, "stdio://", &process_env) + .expect("app-server command should configure"); + let envs = command + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .collect::>(); + + assert_eq!(resolved.codex_home(), shared_home.as_path()); + assert_eq!(envs.get("CODEX_HOME").map(String::as_str), Some("/Users/test/.codex")); + assert_eq!(envs.get("CODEX_SQLITE_HOME").map(String::as_str), Some("/Users/test/.codex")); + } + + #[test] + fn shared_codex_home_resolution_uses_home_dot_codex_for_state() { + let resolved = + super::resolve_shared_codex_home_env_from_home(Some(OsString::from("/Users/test"))) + .expect("absolute HOME should resolve"); + + assert_eq!(resolved.codex_home(), PathBuf::from("/Users/test/.codex").as_path()); + assert_eq!(resolved.sqlite_home(), PathBuf::from("/Users/test/.codex").as_path()); + } + + #[test] + fn app_server_command_overrides_ambient_codex_home_leakage() { + let shared_home = PathBuf::from("/Users/test/.codex"); + let home_env = ResolvedAppServerCodexHomeEnv::new(shared_home.clone(), shared_home) + .expect("test home should validate"); + let process_env = AppServerProcessEnv::with_codex_home_for_test(home_env); + let mut command = Command::new("codex"); + + command.env("CODEX_HOME", "/tmp/per-account-codex-home"); + command.env("CODEX_SQLITE_HOME", "/tmp/per-account-codex-state"); + + super::configure_app_server_command(&mut command, "stdio://", &process_env) + .expect("app-server command should configure"); + + let envs = command + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .collect::>(); + + assert_eq!(envs.get("CODEX_HOME").map(String::as_str), Some("/Users/test/.codex")); + assert_eq!(envs.get("CODEX_SQLITE_HOME").map(String::as_str), Some("/Users/test/.codex")); + } + + #[test] + fn shared_codex_home_resolution_requires_home() { + let error = super::resolve_shared_codex_home_env_from_home(None) + .expect_err("missing HOME should fail"); + + assert!(error.downcast_ref::().is_some()); + assert!(error.to_string().contains("HOME is not set")); + } + + #[test] + fn shared_codex_home_resolution_rejects_invalid_home() { + for (case_name, home, expected) in [ + ("empty", OsString::from(""), "HOME is empty"), + ("relative", OsString::from("relative-home"), "is not absolute"), + ] { + let error = + super::resolve_shared_codex_home_env_from_home(Some(home)).expect_err(case_name); + + assert!(error.downcast_ref::().is_some()); + assert!( + error.to_string().contains(expected), + "unexpected error for {case_name}: {error:?}" + ); + } + } + + #[test] + fn stdin_write_failures_classify_as_app_server_transport_failures() { + let mut child = Command::new("sh") + .args(["-c", "exit 17"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("child process should spawn"); + let stdin = child.stdin.take().expect("child stdin should be captured"); + let _status = child.wait().expect("child should exit"); + let (_stdout_tx, stdout_rx) = mpsc::channel(); + let stderr_tail = + Arc::new(Mutex::new(VecDeque::from([String::from("fatal app-server transport test")]))); + let mut connection = JsonRpcConnection { + child, + stdin, + stdout_rx, + stderr_tail, + pending_messages: VecDeque::new(), + next_request_id: 1, + }; + let error = connection + .notify::("thread/test", None) + .expect_err("closed stdin should fail as transport"); + + assert!(error.downcast_ref::().is_some()); + assert!(error.to_string().contains("App-server stdin write failed")); + assert!(error.to_string().contains("fatal app-server transport test")); + } + + #[test] + fn output_timeouts_downcast_to_timeout_class() { + let error = color_eyre::Report::new(AppServerOutputTimeout); + + assert!(error.downcast_ref::().is_some()); + assert_eq!(error.to_string(), "Timed out while waiting for app-server output."); + } + + #[test] + fn wrapped_transport_failures_still_downcast_to_transport_class() { + let error = color_eyre::Report::new(AppServerTransportFailure::new(String::from( + "App-server stdout disconnected unexpectedly.", + ))) + .wrap_err("outer context"); + + assert!(error.downcast_ref::().is_some()); + } +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge.rs b/apps/decodex/src/agent/tracker_tool_bridge.rs new file mode 100644 index 00000000..4bb649ed --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge.rs @@ -0,0 +1,1167 @@ +mod review; +mod tools; + +use std::{ + cell::RefCell, + env, + error::Error, + fmt::{Display, Formatter}, + path::{Component, PathBuf}, + process::Command, +}; + +use color_eyre::Report; +use serde::{Deserialize, Serialize}; +use serde_json::{self, Value}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + config::InternalReviewMode, + github, + prelude::eyre, + state::StateStore, + tracker::{IssueTracker, TrackerIssue}, + workflow::WorkflowDocument, +}; + +pub(crate) const ISSUE_TRANSITION_TOOL_NAME: &str = "issue_transition"; +pub(crate) const ISSUE_COMMENT_TOOL_NAME: &str = "issue_comment"; +pub(crate) const ISSUE_LABEL_ADD_TOOL_NAME: &str = "issue_label_add"; +pub(crate) const ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME: &str = "issue_progress_checkpoint"; +pub(crate) const ISSUE_REVIEW_CHECKPOINT_TOOL_NAME: &str = "issue_review_checkpoint"; +pub(crate) const ISSUE_REVIEW_HANDOFF_TOOL_NAME: &str = "issue_review_handoff"; +pub(crate) const ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME: &str = "issue_review_repair_complete"; +pub(crate) const ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME: &str = "issue_closeout_complete"; +pub(crate) const ISSUE_TERMINAL_FINALIZE_TOOL_NAME: &str = "issue_terminal_finalize"; + +const REVIEW_POLICY_CONVERGENCE_BUDGET: i64 = 3; + +static GH_PULL_REQUEST_INSPECTOR: GhPullRequestInspector = GhPullRequestInspector; +static LOCAL_GIT_REPO_INSPECTOR: LocalGitRepoInspector = LocalGitRepoInspector; + +pub(crate) trait DynamicToolHandler { + fn tool_specs(&self) -> Vec; + fn handle_call(&self, tool_name: &str, arguments: Value) -> DynamicToolCallResponse; + fn handle_call_with_namespace( + &self, + namespace: Option<&str>, + tool_name: &str, + arguments: Value, + ) -> DynamicToolCallResponse { + let _ = namespace; + + self.handle_call(tool_name, arguments) + } + fn classify_turn_completion( + &self, + final_output: &str, + ) -> crate::prelude::Result { + self.validate_turn_completion(final_output)?; + + Ok(TurnCompletionStatus::Complete) + } + fn validate_turn_completion(&self, _final_output: &str) -> crate::prelude::Result<()> { + Ok(()) + } +} + +pub(crate) trait PullRequestInspector { + fn inspect_pull_request( + &self, + cwd: &std::path::Path, + pr_url: &str, + github_token: &str, + ) -> std::result::Result; +} + +pub(crate) trait LocalRepoInspector { + fn inspect_local_repo( + &self, + cwd: &std::path::Path, + ) -> std::result::Result; +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct DynamicToolSpec { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) namespace: Option, + pub(crate) description: String, + #[serde(rename = "deferLoading", default, skip_serializing_if = "std::ops::Not::not")] + pub(crate) defer_loading: bool, + #[serde(rename = "inputSchema")] + pub(crate) input_schema: Value, + pub(crate) name: String, +} +impl DynamicToolSpec { + pub(crate) fn new( + name: impl Into, + description: impl Into, + input_schema: Value, + ) -> Self { + Self { + namespace: None, + description: description.into(), + defer_loading: false, + input_schema, + name: name.into(), + } + } + + pub(crate) fn deferred(mut self) -> Self { + self.defer_loading = true; + + self + } +} + +pub(crate) struct TrackerToolBridge<'a> { + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: Option, + state_store: Option<&'a StateStore>, + pull_request_inspector: &'a dyn PullRequestInspector, + local_repo_inspector: &'a dyn LocalRepoInspector, + local_issue_state_name: RefCell, + local_opt_out_requested: RefCell, + manual_attention_requested: RefCell, + manual_attention_comment_recorded: RefCell, + continuation_blocking_tracker_write: RefCell>, + pending_review_completion: RefCell>, + finalized_completion_path: RefCell>, +} +impl<'a> TrackerToolBridge<'a> { + #[cfg(test)] + pub(crate) fn new( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + ) -> Self { + Self { + tracker, + issue, + workflow, + review_context: None, + state_store: None, + pull_request_inspector: &GH_PULL_REQUEST_INSPECTOR, + local_repo_inspector: &LOCAL_GIT_REPO_INSPECTOR, + local_issue_state_name: RefCell::new(issue.state.name.clone()), + local_opt_out_requested: RefCell::new( + issue.has_label(workflow.frontmatter().tracker().opt_out_label()), + ), + manual_attention_requested: RefCell::new(false), + manual_attention_comment_recorded: RefCell::new(false), + continuation_blocking_tracker_write: RefCell::new(None), + pending_review_completion: RefCell::new(None), + finalized_completion_path: RefCell::new(None), + } + } + + fn with_review_handoff_inspectors( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: ReviewHandoffContext, + state_store: Option<&'a StateStore>, + pull_request_inspector: &'a dyn PullRequestInspector, + local_repo_inspector: &'a dyn LocalRepoInspector, + ) -> Self { + Self { + tracker, + issue, + workflow, + review_context: Some(review_context), + state_store, + pull_request_inspector, + local_repo_inspector, + local_issue_state_name: RefCell::new(issue.state.name.clone()), + local_opt_out_requested: RefCell::new( + issue.has_label(workflow.frontmatter().tracker().opt_out_label()), + ), + manual_attention_requested: RefCell::new(false), + manual_attention_comment_recorded: RefCell::new(false), + continuation_blocking_tracker_write: RefCell::new(None), + pending_review_completion: RefCell::new(None), + finalized_completion_path: RefCell::new(None), + } + } + + #[cfg(test)] + fn leaked_test_state_store() -> &'static StateStore { + Box::leak(Box::new( + StateStore::open_in_memory().expect("test runtime state store should open"), + )) + } + + #[cfg(test)] + pub(crate) fn with_review_handoff_for_test( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: ReviewHandoffContext, + pull_request_inspector: &'a dyn PullRequestInspector, + local_repo_inspector: &'a dyn LocalRepoInspector, + ) -> Self { + Self::with_review_handoff_inspectors( + tracker, + issue, + workflow, + review_context, + Some(Self::leaked_test_state_store()), + pull_request_inspector, + local_repo_inspector, + ) + } + + #[cfg(test)] + pub(crate) fn with_review_repair_for_test( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: ReviewHandoffContext, + pull_request_inspector: &'a dyn PullRequestInspector, + local_repo_inspector: &'a dyn LocalRepoInspector, + ) -> Self { + Self::with_review_handoff_inspectors( + tracker, + issue, + workflow, + review_context, + Some(Self::leaked_test_state_store()), + pull_request_inspector, + local_repo_inspector, + ) + } + + #[cfg(test)] + pub(crate) fn with_run_context( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: ReviewHandoffContext, + ) -> Self { + Self::with_review_handoff_inspectors( + tracker, + issue, + workflow, + review_context, + Some(Self::leaked_test_state_store()), + &GH_PULL_REQUEST_INSPECTOR, + &LOCAL_GIT_REPO_INSPECTOR, + ) + } + + pub(crate) fn with_run_context_and_state_store( + tracker: &'a dyn IssueTracker, + issue: &'a TrackerIssue, + workflow: &'a WorkflowDocument, + review_context: ReviewHandoffContext, + state_store: &'a StateStore, + ) -> Self { + Self::with_review_handoff_inspectors( + tracker, + issue, + workflow, + review_context, + Some(state_store), + &GH_PULL_REQUEST_INSPECTOR, + &LOCAL_GIT_REPO_INSPECTOR, + ) + } + + pub(crate) fn review_context(&self) -> Option<&ReviewHandoffContext> { + self.review_context.as_ref() + } +} + +impl DynamicToolHandler for TrackerToolBridge<'_> { + fn tool_specs(&self) -> Vec { + self.build_tool_specs() + } + + fn handle_call(&self, tool_name: &str, arguments: Value) -> DynamicToolCallResponse { + self.handle_call_inner(tool_name, arguments) + } + + fn classify_turn_completion( + &self, + _final_output: &str, + ) -> crate::prelude::Result { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + let manual_attention_requested = *self.manual_attention_requested.borrow(); + let manual_attention_comment_recorded = *self.manual_attention_comment_recorded.borrow(); + let review_completion = self.pending_review_completion.borrow().clone(); + + match (manual_attention_requested, manual_attention_comment_recorded, review_completion) { + (false, false, None) => { + if let Some(review_policy_stop) = + self.review_policy_stop_requested(review_context)? + { + return Err(Report::new(review_policy_stop)); + } + if let Some(reason) = self.continuation_blocking_write_reason()? { + eyre::bail!( + "Run `{}` changed issue `{}` via {} without recording a terminal path. Continuation turns may only yield cleanly while the leased issue remains active.", + review_context.run_id, + self.issue.identifier, + reason + ); + } + + if review_context.mode == ReviewExecutionMode::Closeout { + eyre::bail!( + "Run `{}` reached a clean continuation boundary for retained closeout on issue `{}`, but closeout is a deterministic tail. Finish the same turn with `{}` plus `{}` or take the manual-attention path instead of yielding another clean continuation boundary.", + review_context.run_id, + self.issue.identifier, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ); + } + + Ok(TurnCompletionStatus::Continue) + }, + (false, false, Some(_)) | (true, true, None) => { + self.validate_turn_completion("")?; + + Ok(TurnCompletionStatus::Complete) + }, + (true, false, None) => eyre::bail!( + "Run `{}` requested human attention with label `{}`, but issue `{}` never recorded the required explanatory comment.", + review_context.run_id, + self.workflow.frontmatter().tracker().needs_attention_label(), + self.issue.identifier + ), + (true, _, Some(_)) => eyre::bail!( + "Run `{}` recorded both `issue_review_handoff` and label `{}`. Use exactly one final handoff path.", + review_context.run_id, + self.workflow.frontmatter().tracker().needs_attention_label() + ), + (false, true, None) | (false, true, Some(_)) => eyre::bail!( + "Run `{}` recorded a human-attention comment for issue `{}`, but never recorded label `{}`.", + review_context.run_id, + self.issue.identifier, + self.workflow.frontmatter().tracker().needs_attention_label() + ), + } + } + + fn validate_turn_completion(&self, _final_output: &str) -> crate::prelude::Result<()> { + let completion_path = self.completion_disposition()?; + let Some(finalized_path) = *self.finalized_completion_path.borrow() else { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + + eyre::bail!( + "Run `{}` completed, but issue `{}` never called `{}` for terminal path `{}`.", + review_context.run_id, + self.issue.identifier, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + completion_path.as_str() + ); + }; + + if finalized_path != completion_path { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + + eyre::bail!( + "Run `{}` finalized terminal path `{}`, but the recorded terminal path resolved to `{}` at turn completion.", + review_context.run_id, + finalized_path.as_str(), + completion_path.as_str() + ); + } + + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReviewHandoffContext { + pub(crate) attempt_number: i64, + pub(crate) branch_name: String, + pub(crate) run_id: String, + pub(crate) service_id: String, + pub(crate) worktree_path: String, + pub(crate) cwd: PathBuf, + pub(crate) github_token_env_var: Option, + pub(crate) internal_review_mode: InternalReviewMode, + pub(crate) mode: ReviewExecutionMode, + pub(crate) recorded_pr_url: Option, +} +impl ReviewHandoffContext { + pub(crate) fn internal_review_checkpoint_enabled(&self) -> bool { + self.internal_review_mode.requires_review_checkpoint() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReviewHandoffWritebackFailed { + pub(crate) issue_identifier: String, + pub(crate) run_id: String, + pub(crate) pr_url: String, + pub(crate) success_state: String, + pub(crate) source: String, +} +impl Display for ReviewHandoffWritebackFailed { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Run `{}` failed to finalize the review handoff for issue `{}` around target state `{}` and PR `{}`: {}", + self.run_id, self.issue_identifier, self.success_state, self.pr_url, self.source + ) + } +} + +impl Error for ReviewHandoffWritebackFailed {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PullRequestDetails { + base_ref_name: String, + head_ref_name: String, + head_ref_oid: String, + head_repository_name: String, + head_repository_owner: String, + is_draft: bool, + state: String, + url: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct LocalRepoDetails { + default_branch: String, + head_oid: String, + repository_name: String, + repository_owner: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct DynamicToolCallResponse { + #[serde(rename = "contentItems")] + pub(crate) content_items: Vec, + pub(crate) success: bool, +} +impl DynamicToolCallResponse { + pub(crate) fn success(message: String) -> Self { + Self { content_items: vec![DynamicToolContentItem::text(message)], success: true } + } + + pub(crate) fn failure(message: String) -> Self { + Self { content_items: vec![DynamicToolContentItem::text(message)], success: false } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReviewPolicyStopRequested { + pub(crate) head_sha: String, + pub(crate) issue_identifier: String, + pub(crate) nonclean_rounds: Option, + pub(crate) reason: ReviewPolicyStopReason, + pub(crate) run_id: String, +} +impl Display for ReviewPolicyStopRequested { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.reason { + ReviewPolicyStopReason::Exhausted => write!( + f, + "Run `{}` for issue `{}` exhausted the runtime-owned review convergence budget at HEAD `{}` after {} non-clean rounds.", + self.run_id, + self.issue_identifier, + self.head_sha, + self.nonclean_rounds.unwrap_or_default() + ), + ReviewPolicyStopReason::ArchitectureReviewRequired => write!( + f, + "Run `{}` for issue `{}` recorded `needs_architecture_review` at HEAD `{}` and now requires human architecture review.", + self.run_id, self.issue_identifier, self.head_sha + ), + ReviewPolicyStopReason::Blocked => write!( + f, + "Run `{}` for issue `{}` recorded `blocked` at HEAD `{}` and now requires human intervention.", + self.run_id, self.issue_identifier, self.head_sha + ), + } + } +} + +impl Error for ReviewPolicyStopRequested {} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PendingReviewAction { + pr_url: String, + summary: String, +} + +struct GhPullRequestInspector; +impl PullRequestInspector for GhPullRequestInspector { + fn inspect_pull_request( + &self, + cwd: &std::path::Path, + pr_url: &str, + github_token: &str, + ) -> std::result::Result { + let mut command = Command::new("gh"); + + command.args([ + "pr", + "view", + pr_url, + "--json", + "url,baseRefName,headRefName,headRefOid,state,isDraft,headRepository,headRepositoryOwner", + ]); + command.current_dir(cwd); + + github::configure_gh_command(&mut command, github_token); + + let output = command + .output() + .map_err(|error| format!("Failed to inspect pull request `{pr_url}`: {error}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + return Err(format!("Failed to inspect pull request `{pr_url}`: {}", stderr.trim())); + } + + let response: PullRequestViewResponse = + serde_json::from_slice(&output.stdout).map_err(|error| { + format!("Failed to parse pull request details for `{pr_url}`: {error}") + })?; + let Some(head_repository) = response.head_repository else { + return Err(format!( + "Pull request `{pr_url}` does not expose a head repository for review handoff validation." + )); + }; + + Ok(PullRequestDetails { + base_ref_name: response.base_ref_name, + head_ref_name: response.head_ref_name, + head_ref_oid: response.head_ref_oid, + head_repository_name: head_repository.name, + head_repository_owner: response.head_repository_owner.login, + is_draft: response.is_draft, + state: response.state, + url: response.url, + }) + } +} + +struct LocalGitRepoInspector; +impl LocalRepoInspector for LocalGitRepoInspector { + fn inspect_local_repo( + &self, + cwd: &std::path::Path, + ) -> std::result::Result { + let head_oid = + run_command_for_stdout("git", &["rev-parse", "HEAD"], cwd, "inspect lane HEAD")?; + let default_branch = resolve_lane_default_branch(cwd)?; + let origin_url = run_command_for_stdout( + "git", + &["config", "--get", "remote.origin.url"], + cwd, + "inspect lane origin repository", + )?; + let repository = parse_github_repository_identity(origin_url.trim())?; + + Ok(LocalRepoDetails { + default_branch: default_branch + .strip_prefix("origin/") + .unwrap_or(default_branch.as_str()) + .to_owned(), + head_oid, + repository_name: repository.name, + repository_owner: repository.owner, + }) + } +} + +#[derive(Debug, Deserialize)] +struct ScopeArgs { + issue_id: Option, + + issue_identifier: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TransitionArgs { + #[serde(flatten)] + scope: ScopeArgs, + state: String, +} + +#[derive(Debug, Deserialize)] +struct CommentArgs { + #[serde(flatten)] + scope: ScopeArgs, + body: String, +} + +#[derive(Debug, Deserialize)] +struct ReviewHandoffArgs { + #[serde(flatten)] + scope: ScopeArgs, + pr_url: String, + summary: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ProgressCheckpointArgs { + #[serde(flatten)] + scope: ScopeArgs, + phase: String, + focus: String, + next_action: String, + #[serde(default)] + blockers: Vec, + #[serde(default)] + evidence: Vec, + #[serde(default)] + verification: Vec, + head_sha: Option, + branch: Option, + pr_url: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct LabelArgs { + #[serde(flatten)] + scope: ScopeArgs, + label: String, +} + +#[derive(Debug, Deserialize)] +struct TerminalFinalizeArgs { + #[serde(flatten)] + scope: ScopeArgs, + path: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ReviewCheckpointArgs { + #[serde(flatten)] + scope: ScopeArgs, + status: String, + head_sha: String, + #[serde(default)] + evidence: Vec, +} + +#[derive(Debug, Deserialize)] +struct PullRequestViewResponse { + #[serde(rename = "baseRefName")] + base_ref_name: String, + #[serde(rename = "headRefName")] + head_ref_name: String, + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "headRepository")] + head_repository: Option, + #[serde(rename = "headRepositoryOwner")] + head_repository_owner: PullRequestRepositoryOwnerResponse, + #[serde(rename = "isDraft")] + is_draft: bool, + state: String, + url: String, +} + +#[derive(Debug, Deserialize)] +struct PullRequestRepositoryResponse { + name: String, +} + +#[derive(Debug, Deserialize)] +struct PullRequestRepositoryOwnerResponse { + login: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RepositoryIdentity { + name: String, + owner: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ReviewPolicyState { + phase: ReviewPolicyPhase, + status: ReviewPolicyStatus, + head_sha: String, + nonclean_rounds: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReviewExecutionMode { + Handoff, + Repair, + Closeout, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TurnCompletionStatus { + Continue, + Complete, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RunCompletionDisposition { + ManualAttention, + ReviewHandoff, + ReviewRepair, + Closeout, +} +impl RunCompletionDisposition { + fn as_str(self) -> &'static str { + match self { + Self::ManualAttention => "manual_attention", + Self::ReviewHandoff => "review_handoff", + Self::ReviewRepair => "review_repair", + Self::Closeout => "closeout", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "type")] +pub(crate) enum DynamicToolContentItem { + #[serde(rename = "inputText")] + InputText { text: String }, +} +impl DynamicToolContentItem { + fn text(text: String) -> Self { + Self::InputText { text } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReviewPolicyStopReason { + Exhausted, + ArchitectureReviewRequired, + Blocked, +} +impl ReviewPolicyStopReason { + pub(crate) fn error_class(self) -> &'static str { + match self { + Self::Exhausted => "review_policy_exhausted", + Self::ArchitectureReviewRequired => "architecture_review_required", + Self::Blocked => "review_policy_blocked", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExecutionProgressPhase { + Probing, + Implementing, + Verifying, + Blocked, + ReadyForReview, + ReviewRepair, + ReadyToLand, + Closeout, +} +impl ExecutionProgressPhase { + fn as_str(self) -> &'static str { + match self { + Self::Probing => "probing", + Self::Implementing => "implementing", + Self::Verifying => "verifying", + Self::Blocked => "blocked", + Self::ReadyForReview => "ready_for_review", + Self::ReviewRepair => "review_repair", + Self::ReadyToLand => "ready_to_land", + Self::Closeout => "closeout", + } + } + + fn parse(value: &str) -> std::result::Result { + match value { + "probing" => Ok(Self::Probing), + "implementing" => Ok(Self::Implementing), + "verifying" => Ok(Self::Verifying), + "blocked" => Ok(Self::Blocked), + "ready_for_review" => Ok(Self::ReadyForReview), + "review_repair" => Ok(Self::ReviewRepair), + "ready_to_land" => Ok(Self::ReadyToLand), + "closeout" => Ok(Self::Closeout), + other => Err(format!( + "`issue_progress_checkpoint` phase must be `probing`, `implementing`, `verifying`, `blocked`, `ready_for_review`, `review_repair`, `ready_to_land`, or `closeout`, not `{other}`." + )), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ReviewPolicyPhase { + Handoff, + Repair, +} +impl ReviewPolicyPhase { + fn as_str(self) -> &'static str { + match self { + Self::Handoff => "handoff", + Self::Repair => "repair", + } + } + + fn for_mode(mode: ReviewExecutionMode) -> Option { + match mode { + ReviewExecutionMode::Handoff => Some(Self::Handoff), + ReviewExecutionMode::Repair => Some(Self::Repair), + ReviewExecutionMode::Closeout => None, + } + } + + fn parse(value: &str) -> std::result::Result { + match value { + "handoff" => Ok(Self::Handoff), + "repair" => Ok(Self::Repair), + other => Err(format!( + "Unsupported review policy phase `{other}` in the run activity marker." + )), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ReviewPolicyStatus { + Clean, + Findings, + NeedsArchitectureReview, + Blocked, +} +impl ReviewPolicyStatus { + fn as_str(self) -> &'static str { + match self { + Self::Clean => "clean", + Self::Findings => "findings", + Self::NeedsArchitectureReview => "needs_architecture_review", + Self::Blocked => "blocked", + } + } + + fn parse(value: &str) -> std::result::Result { + match value { + "clean" => Ok(Self::Clean), + "findings" => Ok(Self::Findings), + "needs_architecture_review" => Ok(Self::NeedsArchitectureReview), + "blocked" => Ok(Self::Blocked), + other => Err(format!( + "`issue_review_checkpoint` status must be `clean`, `findings`, `needs_architecture_review`, or `blocked`, not `{other}`." + )), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum PendingReviewCompletion { + Handoff(PendingReviewAction), + Repair(PendingReviewAction), + Closeout(PendingReviewAction), +} + +pub(crate) fn dynamic_tool_identifier_is_valid(identifier: &str) -> bool { + !identifier.is_empty() + && identifier.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') +} + +pub(crate) fn current_timestamp() -> String { + OffsetDateTime::now_utc().format(&Rfc3339).expect("timestamp formatting should succeed") +} + +fn resolve_review_handoff_github_token( + review_context: &ReviewHandoffContext, +) -> std::result::Result { + let Some(env_var) = review_context.github_token_env_var.as_deref() else { + return Err(String::from( + "`github.token_env_var` must be configured for PR-backed review handoff validation.", + )); + }; + let value = env::var(env_var).map_err(|error| { + format!( + "Failed to read environment variable `{env_var}` referenced by `github.token_env_var`: {error}" + ) + })?; + + if value.trim().is_empty() { + return Err(format!( + "Environment variable `{env_var}` referenced by `github.token_env_var` must not be blank." + )); + } + + Ok(value) +} + +fn run_command_for_stdout( + command: &str, + args: &[&str], + cwd: &std::path::Path, + purpose: &str, +) -> std::result::Result { + let output = Command::new(command) + .args(args) + .current_dir(cwd) + .output() + .map_err(|error| format!("Failed to {purpose} with `{command}`: {error}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + if detail.is_empty() { + return Err(format!("Failed to {purpose} with `{command}`.")); + } + + return Err(format!("Failed to {purpose} with `{command}`: {detail}")); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let value = stdout.trim(); + + if value.is_empty() { + return Err(format!("Failed to {purpose} with `{command}`: command returned no output.")); + } + + Ok(value.to_owned()) +} + +fn resolve_lane_default_branch(cwd: &std::path::Path) -> std::result::Result { + if let Some(default_branch) = resolve_lane_default_branch_from_local_cache(cwd)? { + return Ok(default_branch); + } + + let remote_default_branch = resolve_lane_default_branch_from_remote(cwd); + + if let Ok(Some(default_branch)) = remote_default_branch.as_ref() { + return Ok(default_branch.clone()); + } + + match remote_default_branch { + Err(error) => Err(error), + Ok(None) => Err(String::from( + "Failed to inspect lane default branch with `git`: neither remote `origin` nor local `origin/HEAD` exposed a default branch.", + )), + Ok(Some(_)) => unreachable!("handled authoritative remote branch above"), + } +} + +fn resolve_lane_default_branch_from_remote( + cwd: &std::path::Path, +) -> std::result::Result, String> { + let remote_probe = Command::new("git") + .args(["ls-remote", "--symref", "origin", "HEAD"]) + .current_dir(cwd) + .output() + .map_err(|error| format!("Failed to inspect lane default branch with `git`: {error}"))?; + + if !remote_probe.status.success() { + let stderr = String::from_utf8_lossy(&remote_probe.stderr); + let stdout = String::from_utf8_lossy(&remote_probe.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + if detail.is_empty() { + return Err(String::from("Failed to inspect lane default branch with `git`.")); + } + + return Err(format!("Failed to inspect lane default branch with `git`: {detail}")); + } + + Ok(parse_remote_head_symref_output(String::from_utf8_lossy(&remote_probe.stdout).as_ref())) +} + +fn resolve_lane_default_branch_from_local_cache( + cwd: &std::path::Path, +) -> std::result::Result, String> { + let symbolic_ref = Command::new("git") + .args(["symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD"]) + .current_dir(cwd) + .output() + .map_err(|error| format!("Failed to inspect lane default branch with `git`: {error}"))?; + + if !symbolic_ref.status.success() { + return Ok(None); + } + + let stdout = String::from_utf8_lossy(&symbolic_ref.stdout); + let default_branch = stdout.trim(); + + if default_branch.is_empty() { + return Ok(None); + } + + Ok(Some(default_branch.strip_prefix("origin/").unwrap_or(default_branch).to_owned())) +} + +fn parse_remote_head_symref_output(stdout: &str) -> Option { + stdout.lines().find_map(|line| { + let line = line.trim(); + + line.strip_prefix("ref: refs/heads/") + .and_then(|remainder| remainder.strip_suffix("\tHEAD")) + .map(str::to_owned) + }) +} + +fn parse_github_repository_identity( + remote_url: &str, +) -> std::result::Result { + let path = if let Some(path) = remote_url.strip_prefix("git@github.com:") { + path + } else { + parse_github_remote_with_authority(remote_url)? + }; + let path = path.strip_suffix(".git").unwrap_or(path); + let mut parts = path.split('/'); + let Some(owner) = parts.next() else { + return Err(format!("Unsupported GitHub remote URL `{remote_url}`.")); + }; + let Some(name) = parts.next() else { + return Err(format!("Unsupported GitHub remote URL `{remote_url}`.")); + }; + + if owner.is_empty() || name.is_empty() || parts.next().is_some() { + return Err(format!("Unsupported GitHub remote URL `{remote_url}`.")); + } + + Ok(RepositoryIdentity { name: name.to_owned(), owner: owner.to_owned() }) +} + +fn parse_github_remote_with_authority(remote_url: &str) -> std::result::Result<&str, String> { + let rest = remote_url + .strip_prefix("https://") + .or_else(|| remote_url.strip_prefix("http://")) + .or_else(|| remote_url.strip_prefix("ssh://")) + .ok_or_else(|| format!("Unsupported GitHub remote URL `{remote_url}`."))?; + let (authority, path) = rest + .split_once('/') + .ok_or_else(|| format!("Unsupported GitHub remote URL `{remote_url}`."))?; + let authority = authority.rsplit('@').next().unwrap_or(authority); + let host = authority.split_once(':').map(|(host, _)| host).unwrap_or(authority); + + if host != "github.com" { + return Err(format!("Unsupported GitHub remote URL `{remote_url}`.")); + } + + Ok(path) +} + +fn validate_public_comment_body(body: &str) -> Result<(), String> { + for line in body.lines() { + let Some((field_name, value)) = extract_structured_field(line) else { + continue; + }; + + if field_name == "worktree_path" { + validate_repo_relative_path(value, field_name)?; + + continue; + } + if field_name.ends_with("_path") { + return Err(format!( + "Unsupported structured field `{field_name}` in public issue comments." + )); + } + } + + Ok(()) +} + +fn extract_structured_field(line: &str) -> Option<(&str, &str)> { + let trimmed = line.trim(); + let trimmed = trimmed.strip_prefix("- ").unwrap_or(trimmed); + let (key, value) = trimmed.split_once(':')?; + + Some((key.trim(), value.trim().trim_matches('`'))) +} + +fn validate_repo_relative_path(path: &str, field_name: &str) -> Result<(), String> { + if path.is_empty() { + return Err(format!("`{field_name}` must not be empty.")); + } + if path.starts_with('/') || path.starts_with("~/") || has_drive_root_prefix(path) { + return Err(format!("`{field_name}` must be repository-relative, not `{path}`.")); + } + + let components = std::path::Path::new(path).components(); + + if components.into_iter().any(|component| matches!(component, Component::ParentDir)) { + return Err(format!("`{field_name}` must stay within the repository, not `{path}`.")); + } + + Ok(()) +} + +fn has_drive_root_prefix(path: &str) -> bool { + let bytes = path.as_bytes(); + + bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/') +} + +fn normalize_summary(summary: &str) -> String { + summary.split_whitespace().collect::>().join(" ") +} + +fn normalize_progress_list(items: Vec) -> Vec { + items.into_iter().map(|item| normalize_summary(&item)).filter(|item| !item.is_empty()).collect() +} + +fn normalize_optional_progress_field(value: Option) -> Option { + value.and_then(|value| { + let normalized = normalize_summary(&value); + + (!normalized.is_empty()).then_some(normalized) + }) +} + +fn format_review_handoff_comment( + review_context: &ReviewHandoffContext, + pending_review_handoff: &PendingReviewAction, +) -> String { + format!( + "decodex run completed and is ready for review\n\n- run_id: `{run_id}`\n- attempt: `{attempt}`\n- finished_at: `{finished_at}`\n- branch: `{branch}`\n- pr_url: `{pr_url}`\n- worktree_path: `{worktree_path}`\n- validation_result: `passed`\n- summary: {summary}", + run_id = review_context.run_id, + attempt = review_context.attempt_number, + finished_at = current_timestamp(), + branch = review_context.branch_name, + pr_url = pending_review_handoff.pr_url, + worktree_path = review_context.worktree_path, + summary = pending_review_handoff.summary, + ) +} + +fn format_review_repair_comment( + review_context: &ReviewHandoffContext, + pending_review_repair: &PendingReviewAction, +) -> String { + format!( + "decodex review repair completed and requested fresh review\n\n- run_id: `{run_id}`\n- attempt: `{attempt}`\n- finished_at: `{finished_at}`\n- branch: `{branch}`\n- pr_url: `{pr_url}`\n- worktree_path: `{worktree_path}`\n- validation_result: `passed`\n- summary: {summary}", + run_id = review_context.run_id, + attempt = review_context.attempt_number, + finished_at = current_timestamp(), + branch = review_context.branch_name, + pr_url = pending_review_repair.pr_url, + worktree_path = review_context.worktree_path, + summary = pending_review_repair.summary, + ) +} + +#[cfg(test)] mod tests; diff --git a/apps/decodex/src/agent/tracker_tool_bridge/review.rs b/apps/decodex/src/agent/tracker_tool_bridge/review.rs new file mode 100644 index 00000000..3c8e189b --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/review.rs @@ -0,0 +1,1024 @@ +use color_eyre::Report; + +use crate::{ + agent::tracker_tool_bridge::{ + self, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + ISSUE_TRANSITION_TOOL_NAME, LocalRepoDetails, PendingReviewCompletion, PullRequestDetails, + REVIEW_POLICY_CONVERGENCE_BUDGET, ReviewExecutionMode, ReviewHandoffContext, + ReviewHandoffWritebackFailed, ReviewPolicyPhase, ReviewPolicyState, ReviewPolicyStatus, + ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, ScopeArgs, + TrackerToolBridge, + }, + prelude::eyre, + state::{self, ReviewHandoffMarker, ReviewOrchestrationMarker}, + tracker::{ + self, TrackerIssue, + records::{self}, + }, +}; + +enum CloseoutIssueStateValidation { + RefreshRequired, + AlreadyVerified, +} + +impl<'a> TrackerToolBridge<'a> { + fn persist_linear_execution_event( + &self, + record: &records::LinearExecutionEventRecord, + ) -> crate::prelude::Result<()> { + if let Some(state_store) = self.state_store { + state_store.record_linear_execution_event(record)?; + } + + Ok(()) + } + + fn persist_review_handoff_marker( + &self, + review_context: &ReviewHandoffContext, + marker: &ReviewHandoffMarker, + ) -> crate::prelude::Result<()> { + let state_store = self.state_store.ok_or_else(|| { + eyre::eyre!( + "Runtime state store is required to persist review handoff for issue `{}`.", + self.issue.identifier + ) + })?; + + state_store.upsert_review_handoff_marker(&review_context.service_id, &self.issue.id, marker) + } + + fn persist_review_orchestration_marker( + &self, + review_context: &ReviewHandoffContext, + marker: &ReviewOrchestrationMarker, + ) -> crate::prelude::Result<()> { + let state_store = self.state_store.ok_or_else(|| { + eyre::eyre!( + "Runtime state store is required to persist review orchestration for issue `{}`.", + self.issue.identifier + ) + })?; + + state_store.upsert_review_orchestration_marker( + &review_context.service_id, + &self.issue.id, + marker, + ) + } + + pub(super) fn cache_review_policy_state_best_effort( + &self, + review_context: &ReviewHandoffContext, + review_policy_phase: ReviewPolicyPhase, + review_policy_status: ReviewPolicyStatus, + head_sha: &str, + nonclean_rounds: i64, + ) { + if let Err(error) = state::write_run_review_policy_state( + &review_context.cwd, + &review_context.run_id, + review_context.attempt_number, + review_policy_phase.as_str(), + review_policy_status.as_str(), + head_sha, + nonclean_rounds, + ) { + tracing::warn!( + ?error, + issue = self.issue.identifier, + run_id = review_context.run_id, + worktree_path = %review_context.cwd.display(), + "Review policy marker write failed; continuing with runtime state." + ); + } + } + + pub(super) fn ensure_issue_scope(&self, scope: &ScopeArgs) -> Result<(), String> { + if let Some(issue_id) = scope.issue_id.as_deref() + && issue_id != self.issue.id + { + return Err(format!( + "Tool call targeted issue id `{issue_id}`, but the leased issue id is `{}`.", + self.issue.id + )); + } + if let Some(issue_identifier) = scope.issue_identifier.as_deref() + && issue_identifier != self.issue.identifier + { + return Err(format!( + "Tool call targeted issue identifier `{issue_identifier}`, but the leased issue identifier is `{}`.", + self.issue.identifier + )); + } + + Ok(()) + } + + pub(super) fn allowed_transition_states(&self) -> Vec<&str> { + let tracker = self.workflow.frontmatter().tracker(); + + if matches!( + self.review_context.as_ref().map(|context| context.mode), + Some(ReviewExecutionMode::Closeout) + ) { + return vec![tracker.resolved_completed_state()]; + } + + let success_state = tracker.success_state(); + let mut states = tracker + .startable_states() + .iter() + .map(String::as_str) + .filter(|state| *state != success_state) + .collect::>(); + + for state in [tracker.in_progress_state(), tracker.failure_state()] { + if state != success_state && !states.iter().any(|existing| existing == &state) { + states.push(state); + } + } + + states + } + + pub(super) fn refreshed_issue_snapshot(&self) -> crate::prelude::Result> { + let issue_ids = [self.issue.id.clone()]; + let mut refreshed_issues = self.tracker.refresh_issues(&issue_ids)?; + + Ok(refreshed_issues.pop()) + } + + pub(super) fn validate_review_action_pr( + &self, + review_context: &ReviewHandoffContext, + pr_url: &str, + ) -> std::result::Result { + let github_token = + tracker_tool_bridge::resolve_review_handoff_github_token(review_context)?; + let pull_request = self.pull_request_inspector.inspect_pull_request( + &review_context.cwd, + pr_url, + github_token.as_str(), + )?; + let local_repo = self.local_repo_inspector.inspect_local_repo(&review_context.cwd)?; + + if pull_request.head_repository_owner != local_repo.repository_owner + || pull_request.head_repository_name != local_repo.repository_name + { + return Err(format!( + "Pull request `{}` belongs to repository `{}/{}`, but the current lane repository is `{}/{}`.", + pull_request.url, + pull_request.head_repository_owner, + pull_request.head_repository_name, + local_repo.repository_owner, + local_repo.repository_name + )); + } + if pull_request.base_ref_name != local_repo.default_branch { + return Err(format!( + "Pull request `{}` targets base branch `{}`, but retained review lanes must target the repository default branch `{}`.", + pull_request.url, pull_request.base_ref_name, local_repo.default_branch + )); + } + if pull_request.head_ref_name != review_context.branch_name { + return Err(format!( + "Pull request `{}` is for branch `{}`, but the current lane branch is `{}`.", + pull_request.url, pull_request.head_ref_name, review_context.branch_name + )); + } + if pull_request.head_ref_oid != local_repo.head_oid { + return Err(format!( + "Pull request `{}` points at commit `{}`, but the current lane HEAD is `{}`. Push the latest lane commit before review handoff.", + pull_request.url, pull_request.head_ref_oid, local_repo.head_oid + )); + } + if pull_request.state != "OPEN" { + return Err(format!( + "Pull request `{}` is `{}`; it must be open for review handoff.", + pull_request.url, pull_request.state + )); + } + if pull_request.is_draft { + return Err(format!( + "Pull request `{}` is still draft; mark it ready for review before handoff.", + pull_request.url + )); + } + + if let Some(recorded_pr_url) = review_context.recorded_pr_url.as_deref() + && pull_request.url != recorded_pr_url + { + return Err(format!( + "Pull request `{}` does not match the retained lane PR `{}`.", + pull_request.url, recorded_pr_url + )); + } + + Ok(pull_request) + } + + pub(super) fn validate_closeout_pr( + &self, + review_context: &ReviewHandoffContext, + pr_url: &str, + ) -> std::result::Result { + let github_token = + tracker_tool_bridge::resolve_review_handoff_github_token(review_context)?; + let pull_request = self.pull_request_inspector.inspect_pull_request( + &review_context.cwd, + pr_url, + github_token.as_str(), + )?; + let local_repo = self.local_repo_inspector.inspect_local_repo(&review_context.cwd)?; + + if pull_request.head_repository_owner != local_repo.repository_owner + || pull_request.head_repository_name != local_repo.repository_name + { + return Err(format!( + "Pull request `{}` belongs to repository `{}/{}`, but the current lane repository is `{}/{}`.", + pull_request.url, + pull_request.head_repository_owner, + pull_request.head_repository_name, + local_repo.repository_owner, + local_repo.repository_name + )); + } + if pull_request.base_ref_name != local_repo.default_branch { + return Err(format!( + "Pull request `{}` targets base branch `{}`, but retained closeout requires the repository default branch `{}`.", + pull_request.url, pull_request.base_ref_name, local_repo.default_branch + )); + } + if pull_request.head_ref_name != review_context.branch_name { + return Err(format!( + "Pull request `{}` is for branch `{}`, but the current lane branch is `{}`.", + pull_request.url, pull_request.head_ref_name, review_context.branch_name + )); + } + if pull_request.head_ref_oid != local_repo.head_oid { + return Err(format!( + "Pull request `{}` points at commit `{}`, but the current lane HEAD is `{}`. Finish closeout from the merged lane head.", + pull_request.url, pull_request.head_ref_oid, local_repo.head_oid + )); + } + if pull_request.state != "MERGED" { + return Err(format!( + "Pull request `{}` is `{}`; it must be merged before closeout completes.", + pull_request.url, pull_request.state + )); + } + if pull_request.is_draft { + return Err(format!( + "Pull request `{}` is still draft; closeout requires a merged non-draft PR lineage.", + pull_request.url + )); + } + + if let Some(recorded_pr_url) = review_context.recorded_pr_url.as_deref() + && pull_request.url != recorded_pr_url + { + return Err(format!( + "Pull request `{}` does not match the retained lane PR `{}`.", + pull_request.url, recorded_pr_url + )); + } + + Ok(pull_request) + } + + pub(super) fn record_continuation_blocking_transition(&self, state: &str) { + if state != self.workflow.frontmatter().tracker().in_progress_state() { + self.record_continuation_blocking_write(format!( + "`{ISSUE_TRANSITION_TOOL_NAME}` to state `{state}`" + )); + } + } + + pub(super) fn record_continuation_blocking_write(&self, reason: String) { + self.continuation_blocking_tracker_write.replace(Some(reason)); + } + + pub(super) fn local_issue_remains_active(&self) -> bool { + self.local_issue_state_name.borrow().as_str() + == self.workflow.frontmatter().tracker().in_progress_state() + && !*self.local_opt_out_requested.borrow() + && !*self.manual_attention_requested.borrow() + } + + pub(super) fn continuation_blocking_write_reason( + &self, + ) -> crate::prelude::Result> { + let Some(reason) = self.continuation_blocking_tracker_write.borrow().clone() else { + return Ok(None); + }; + let tracker_policy = self.workflow.frontmatter().tracker(); + let run_started_active = self.issue.state.name == tracker_policy.in_progress_state(); + + if run_started_active && !self.local_issue_remains_active() { + return Ok(Some(reason)); + } + + let issue = match self.refreshed_issue_snapshot()? { + Some(issue) => issue, + None => return Ok(Some(reason)), + }; + let issue_still_active = issue.state.name == tracker_policy.in_progress_state() + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()); + + if issue_still_active { + return Ok(None); + } + + Ok(Some(reason)) + } + + pub(crate) fn startup_transition_succeeded_locally(&self) -> bool { + self.local_issue_state_name.borrow().as_str() + == self.workflow.frontmatter().tracker().in_progress_state() + } + + pub(crate) fn completion_disposition( + &self, + ) -> crate::prelude::Result { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + let manual_attention_requested = *self.manual_attention_requested.borrow(); + let manual_attention_comment_recorded = *self.manual_attention_comment_recorded.borrow(); + let review_completion = self.pending_review_completion.borrow().clone(); + + match (manual_attention_requested, manual_attention_comment_recorded, review_completion) { + (false, false, Some(PendingReviewCompletion::Handoff(_))) => + Ok(RunCompletionDisposition::ReviewHandoff), + (false, false, Some(PendingReviewCompletion::Repair(_))) => + Ok(RunCompletionDisposition::ReviewRepair), + (false, false, Some(PendingReviewCompletion::Closeout(_))) => + Ok(RunCompletionDisposition::Closeout), + (true, true, None) => Ok(RunCompletionDisposition::ManualAttention), + (true, false, None) => eyre::bail!( + "Run `{}` requested human attention with label `{}`, but issue `{}` never recorded the required explanatory comment.", + review_context.run_id, + self.workflow.frontmatter().tracker().needs_attention_label(), + self.issue.identifier + ), + (true, _, Some(_)) => eyre::bail!( + "Run `{}` recorded both `{}` and label `{}`. Use exactly one final tracker exit path.", + review_context.run_id, + self.required_pr_completion_tool_name(), + self.workflow.frontmatter().tracker().needs_attention_label() + ), + (false, false, None) => eyre::bail!( + "Run `{}` completed, but issue `{}` recorded neither `{}` nor label `{}` for human attention.", + review_context.run_id, + self.issue.identifier, + self.required_pr_completion_tool_name(), + self.workflow.frontmatter().tracker().needs_attention_label() + ), + (false, true, None) | (false, true, Some(_)) => eyre::bail!( + "Run `{}` recorded a human-attention comment for issue `{}`, but never recorded label `{}`.", + review_context.run_id, + self.issue.identifier, + self.workflow.frontmatter().tracker().needs_attention_label() + ), + } + } + + pub(crate) fn apply_review_handoff(&self) -> crate::prelude::Result<()> { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + let pending_review_handoff = { + let pending_review_handoff = self.pending_review_completion.borrow(); + let Some(PendingReviewCompletion::Handoff(pending_review_handoff)) = + pending_review_handoff.as_ref() + else { + eyre::bail!( + "Run `{}` completed, but issue `{}` never recorded a PR-backed review handoff.", + review_context.run_id, + self.issue.identifier + ); + }; + + pending_review_handoff.clone() + }; + let pull_request = self + .validate_review_action_pr(review_context, &pending_review_handoff.pr_url) + .map_err(|error| eyre::eyre!(error))?; + let completion_comment = tracker_tool_bridge::format_review_handoff_comment( + review_context, + &pending_review_handoff, + ); + let handoff_record = linear_execution_review_event( + self.issue, + review_context, + &pull_request, + "review_handoff", + "review_handoff", + &pending_review_handoff.summary, + ); + + tracker_tool_bridge::validate_public_comment_body(&completion_comment) + .map_err(|error| eyre::eyre!(error))?; + + let success_state = self.workflow.frontmatter().tracker().success_state(); + let success_state_id = self.issue.state_id_for_name(success_state).ok_or_else(|| { + eyre::eyre!( + "State `{success_state}` does not exist on issue `{}`.", + self.issue.identifier + ) + })?; + let handoff_marker = ReviewHandoffMarker::new( + review_context.run_id.clone(), + review_context.attempt_number, + review_context.branch_name.clone(), + pull_request.url.clone(), + pull_request.base_ref_name.clone(), + pull_request.head_ref_name.clone(), + pull_request.head_ref_oid.clone(), + ); + let orchestration_marker = ReviewOrchestrationMarker::new( + review_context.run_id.clone(), + review_context.attempt_number, + review_context.branch_name.clone(), + pull_request.url.clone(), + pull_request.head_ref_oid.clone(), + "request_pending", + None, + None, + None, + 0, + 0, + None, + ); + + if let Err(error) = tracker::create_linear_execution_event_comment( + self.tracker, + &self.issue.id, + &completion_comment, + &handoff_record, + ) { + return Err(Report::new(ReviewHandoffWritebackFailed { + issue_identifier: self.issue.identifier.clone(), + run_id: review_context.run_id.clone(), + pr_url: pending_review_handoff.pr_url, + success_state: success_state.to_owned(), + source: format!("failed to persist the tracker review handoff record: {error}"), + })); + } + + self.persist_linear_execution_event(&handoff_record)?; + self.persist_review_handoff_marker(review_context, &handoff_marker)?; + self.persist_review_orchestration_marker(review_context, &orchestration_marker)?; + + if let Err(error) = self.tracker.update_issue_state(&self.issue.id, success_state_id) { + if let Some(state_store) = self.state_store { + state_store.clear_review_markers(&self.issue.id)?; + } + + return Err(Report::new(ReviewHandoffWritebackFailed { + issue_identifier: self.issue.identifier.clone(), + run_id: review_context.run_id.clone(), + pr_url: pull_request.url.clone(), + success_state: success_state.to_owned(), + source: format!("failed to move the tracker issue to `{success_state}`: {error}"), + })); + } + + self.pending_review_completion.borrow_mut().take(); + + Ok(()) + } + + pub(crate) fn apply_review_repair(&self) -> crate::prelude::Result<()> { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + let pending_review_repair = { + let pending_review_repair = self.pending_review_completion.borrow(); + let Some(PendingReviewCompletion::Repair(pending_review_repair)) = + pending_review_repair.as_ref() + else { + eyre::bail!( + "Run `{}` completed, but issue `{}` never recorded retained review repair completion.", + review_context.run_id, + self.issue.identifier + ); + }; + + pending_review_repair.clone() + }; + let pull_request = self + .validate_review_action_pr(review_context, &pending_review_repair.pr_url) + .map_err(|error| eyre::eyre!(error))?; + let completion_comment = tracker_tool_bridge::format_review_repair_comment( + review_context, + &pending_review_repair, + ); + let handoff_record = linear_execution_review_event( + self.issue, + review_context, + &pull_request, + "repair_handoff", + "review_repair", + &pending_review_repair.summary, + ); + let review_handoff = ReviewHandoffMarker::new( + review_context.run_id.clone(), + review_context.attempt_number, + review_context.branch_name.clone(), + pull_request.url.clone(), + pull_request.base_ref_name.clone(), + pull_request.head_ref_name.clone(), + pull_request.head_ref_oid.clone(), + ); + + tracker_tool_bridge::validate_public_comment_body(&completion_comment) + .map_err(|error| eyre::eyre!(error))?; + + let state_store = self.state_store.ok_or_else(|| { + eyre::eyre!( + "Runtime state store is required to read review orchestration for issue `{}`.", + self.issue.identifier + ) + })?; + let previous_review_handoff = state_store.review_handoff_marker( + &review_context.service_id, + &self.issue.id, + &review_context.branch_name, + )?; + let persisted_orchestration = previous_review_handoff + .as_ref() + .map(|marker| { + state_store.review_orchestration_marker( + &review_context.service_id, + &self.issue.id, + marker, + ) + }) + .transpose()? + .flatten(); + let external_round_count = persisted_orchestration.map_or(0, |marker| { + if marker.external_round_count() >= 4 { 0 } else { marker.external_round_count() } + }); + + tracker::create_linear_execution_event_comment( + self.tracker, + &self.issue.id, + &completion_comment, + &handoff_record, + )?; + + self.persist_linear_execution_event(&handoff_record)?; + self.persist_review_handoff_marker(review_context, &review_handoff)?; + self.persist_review_orchestration_marker( + review_context, + &ReviewOrchestrationMarker::new( + review_context.run_id.clone(), + review_context.attempt_number, + review_context.branch_name.clone(), + pull_request.url.clone(), + pull_request.head_ref_oid.clone(), + "request_pending", + None, + None, + None, + 0, + external_round_count, + None, + ), + )?; + self.pending_review_completion.borrow_mut().take(); + + Ok(()) + } + + pub(crate) fn apply_closeout(&self) -> crate::prelude::Result<()> { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + let pending_closeout = { + let pending_review_completion = self.pending_review_completion.borrow(); + let Some(PendingReviewCompletion::Closeout(pending_closeout)) = + pending_review_completion.as_ref() + else { + eyre::bail!( + "Run `{}` completed, but issue `{}` never recorded retained closeout completion.", + review_context.run_id, + self.issue.identifier + ); + }; + + pending_closeout.clone() + }; + + self.write_closeout_record( + review_context, + &pending_closeout.pr_url, + CloseoutIssueStateValidation::RefreshRequired, + &pending_closeout.summary, + )?; + self.pending_review_completion.borrow_mut().take(); + + Ok(()) + } + + pub(crate) fn validate_deterministic_closeout_pr( + &self, + pr_url: &str, + ) -> crate::prelude::Result { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + + self.validate_closeout_pr(review_context, pr_url).map_err(|error| eyre::eyre!(error)) + } + + pub(crate) fn apply_validated_deterministic_closeout( + &self, + pull_request: PullRequestDetails, + ) -> crate::prelude::Result<()> { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + + self.write_validated_closeout_record( + review_context, + pull_request, + CloseoutIssueStateValidation::AlreadyVerified, + "Validated merged PR lineage and completed retained closeout.", + ) + } + + fn write_closeout_record( + &self, + review_context: &ReviewHandoffContext, + pr_url: &str, + issue_state_validation: CloseoutIssueStateValidation, + summary: &str, + ) -> crate::prelude::Result<()> { + let pull_request = self + .validate_closeout_pr(review_context, pr_url) + .map_err(|error| eyre::eyre!(error))?; + + self.write_validated_closeout_record( + review_context, + pull_request, + issue_state_validation, + summary, + ) + } + + fn write_validated_closeout_record( + &self, + review_context: &ReviewHandoffContext, + pull_request: PullRequestDetails, + issue_state_validation: CloseoutIssueStateValidation, + summary: &str, + ) -> crate::prelude::Result<()> { + if matches!(issue_state_validation, CloseoutIssueStateValidation::RefreshRequired) { + self.validate_closeout_issue_completed_state().map_err(|error| eyre::eyre!(error))?; + } + + let closeout_record = + linear_execution_closeout_event(self.issue, review_context, &pull_request, summary); + let closeout_comment = format!( + "decodex closeout completed\n\n- run_id: `{}`\n- attempt: `{}`\n- finished_at: `{}`\n- branch: `{}`\n- pr_url: `{}`\n- worktree_path: `{}`\n- summary: {}", + review_context.run_id, + review_context.attempt_number, + tracker_tool_bridge::current_timestamp(), + review_context.branch_name, + pull_request.url, + review_context.worktree_path, + summary, + ); + + tracker_tool_bridge::validate_public_comment_body(&closeout_comment) + .map_err(|error| eyre::eyre!(error))?; + tracker::create_linear_execution_event_comment( + self.tracker, + &self.issue.id, + &closeout_comment, + &closeout_record, + )?; + + self.persist_linear_execution_event(&closeout_record)?; + + Ok(()) + } + + pub(crate) fn clear_closeout_issue_scope(&self) -> crate::prelude::Result<()> { + let Some(review_context) = self.review_context.as_ref() else { + eyre::bail!( + "Review handoff context is unavailable for issue `{}`.", + self.issue.identifier + ); + }; + + tracker::clear_automation_lane_labels(self.tracker, self.issue, &review_context.service_id) + } + + pub(super) fn canonicalize_current_lane_head_sha( + &self, + tool_name: &str, + head_sha: &str, + current_head_sha: &str, + ) -> std::result::Result { + let head_sha = head_sha.trim(); + + if head_sha.is_empty() { + return Err(format!("`{tool_name}` requires a non-empty `head_sha`.")); + } + if head_sha == current_head_sha { + return Ok(current_head_sha.to_owned()); + } + if head_sha.len() >= 7 && current_head_sha.starts_with(head_sha) { + return Ok(current_head_sha.to_owned()); + } + + Err(format!( + "`{tool_name}` head `{head_sha}` does not match the current lane HEAD `{current_head_sha}`." + )) + } + + pub(super) fn current_local_repo_details( + &self, + review_context: &ReviewHandoffContext, + ) -> std::result::Result { + self.local_repo_inspector.inspect_local_repo(&review_context.cwd) + } + + pub(super) fn resolve_progress_checkpoint_head_sha( + &self, + head_sha: Option, + ) -> std::result::Result, String> { + let normalized_head_sha = tracker_tool_bridge::normalize_optional_progress_field(head_sha); + let Some(review_context) = self.review_context.as_ref() else { + return Ok(normalized_head_sha); + }; + let local_repo = self.current_local_repo_details(review_context)?; + + match normalized_head_sha { + Some(head_sha) => self + .canonicalize_current_lane_head_sha( + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + &head_sha, + &local_repo.head_oid, + ) + .map(Some), + None => Ok(Some(local_repo.head_oid)), + } + } + + pub(super) fn validate_closeout_issue_completed_state( + &self, + ) -> std::result::Result<(), String> { + let completed_state = self.workflow.frontmatter().tracker().resolved_completed_state(); + let current_issue = self.refreshed_issue_snapshot().map_err(|error| error.to_string())? + .ok_or_else(|| { + format!( + "Failed to refresh issue `{}` during closeout validation: tracker returned no current snapshot.", + self.issue.identifier + ) + })?; + + if current_issue.state.name != completed_state { + return Err(format!( + "Closeout for issue `{}` requires tracker state `{}`, but the refreshed issue is still `{}`. Move the issue to `{}` with `{}` before calling `{}`.", + self.issue.identifier, + completed_state, + current_issue.state.name, + completed_state, + ISSUE_TRANSITION_TOOL_NAME, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME + )); + } + + Ok(()) + } + + pub(super) fn review_policy_state_for_current_phase( + &self, + review_context: &ReviewHandoffContext, + ) -> crate::prelude::Result> { + let Some(current_phase) = ReviewPolicyPhase::for_mode(review_context.mode) else { + return Ok(None); + }; + let Some(marker) = state::read_run_activity_marker_snapshot(&review_context.cwd)? else { + return Ok(None); + }; + let Some(phase) = marker.review_policy_phase() else { + return Ok(None); + }; + let Some(status) = marker.review_policy_status() else { + eyre::bail!( + "Run activity marker for issue `{}` is missing `review_policy_status`.", + self.issue.identifier + ); + }; + let Some(head_sha) = marker.review_policy_head_sha() else { + eyre::bail!( + "Run activity marker for issue `{}` is missing `review_policy_head_sha`.", + self.issue.identifier + ); + }; + let Some(nonclean_rounds) = marker.review_policy_nonclean_rounds() else { + eyre::bail!( + "Run activity marker for issue `{}` is missing `review_policy_nonclean_rounds`.", + self.issue.identifier + ); + }; + let phase = ReviewPolicyPhase::parse(phase).map_err(|error| eyre::eyre!(error))?; + let status = ReviewPolicyStatus::parse(status).map_err(|error| eyre::eyre!(error))?; + + if phase != current_phase { + return Ok(None); + } + + Ok(Some(ReviewPolicyState { + phase, + status, + head_sha: head_sha.to_owned(), + nonclean_rounds, + })) + } + + pub(super) fn review_policy_state_for_current_head( + &self, + review_context: &ReviewHandoffContext, + ) -> crate::prelude::Result> { + let Some(checkpoint) = self.review_policy_state_for_current_phase(review_context)? else { + return Ok(None); + }; + let local_repo = + self.current_local_repo_details(review_context).map_err(|error| eyre::eyre!(error))?; + + if checkpoint.head_sha != local_repo.head_oid { + return Ok(None); + } + + Ok(Some(checkpoint)) + } + + pub(super) fn require_clean_review_checkpoint( + &self, + review_context: &ReviewHandoffContext, + ) -> std::result::Result<(), String> { + if !review_context.internal_review_checkpoint_enabled() { + return Ok(()); + } + + let Some(checkpoint) = self + .review_policy_state_for_current_head(review_context) + .map_err(|error| error.to_string())? + else { + return Err(format!( + "`{}` requires a current `{}` review checkpoint with status `clean` for the current lane HEAD.", + self.required_pr_completion_tool_name(), + ReviewPolicyPhase::for_mode(review_context.mode) + .expect("review completion should only be available during review phases") + .as_str(), + )); + }; + + if checkpoint.status != ReviewPolicyStatus::Clean { + return Err(format!( + "`{}` requires the latest review checkpoint to be `clean`, not `{}`.", + self.required_pr_completion_tool_name(), + checkpoint.status.as_str(), + )); + } + + Ok(()) + } + + pub(super) fn review_policy_stop_requested( + &self, + review_context: &ReviewHandoffContext, + ) -> crate::prelude::Result> { + if !review_context.internal_review_checkpoint_enabled() { + return Ok(None); + } + + let Some(checkpoint) = self.review_policy_state_for_current_head(review_context)? else { + return Ok(None); + }; + let stop_reason = match checkpoint.status { + ReviewPolicyStatus::Clean => return Ok(None), + ReviewPolicyStatus::Findings + if checkpoint.nonclean_rounds < REVIEW_POLICY_CONVERGENCE_BUDGET => + { + return Ok(None); + }, + ReviewPolicyStatus::Findings => ReviewPolicyStopReason::Exhausted, + ReviewPolicyStatus::NeedsArchitectureReview => + ReviewPolicyStopReason::ArchitectureReviewRequired, + ReviewPolicyStatus::Blocked => ReviewPolicyStopReason::Blocked, + }; + + Ok(Some(ReviewPolicyStopRequested { + head_sha: checkpoint.head_sha, + issue_identifier: self.issue.identifier.clone(), + nonclean_rounds: Some(checkpoint.nonclean_rounds), + reason: stop_reason, + run_id: review_context.run_id.clone(), + })) + } + + pub(super) fn required_pr_completion_tool_name(&self) -> &'static str { + match self.review_context.as_ref().map(|context| context.mode) { + Some(ReviewExecutionMode::Handoff) => ISSUE_REVIEW_HANDOFF_TOOL_NAME, + Some(ReviewExecutionMode::Repair) => ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + Some(ReviewExecutionMode::Closeout) => ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + None => ISSUE_REVIEW_HANDOFF_TOOL_NAME, + } + } +} + +fn linear_execution_identity<'a>( + issue: &'a TrackerIssue, + review_context: &'a ReviewHandoffContext, +) -> records::LinearExecutionEventIdentity<'a> { + records::LinearExecutionEventIdentity { + service_id: &review_context.service_id, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + run_id: &review_context.run_id, + attempt_number: review_context.attempt_number, + } +} + +fn linear_execution_review_event( + issue: &TrackerIssue, + review_context: &ReviewHandoffContext, + pull_request: &PullRequestDetails, + event_type: &str, + terminal_path: &str, + summary: &str, +) -> records::LinearExecutionEventRecord { + let anchor = records::stable_event_anchor(&[&pull_request.url, &pull_request.head_ref_oid]); + let mut record = records::LinearExecutionEventRecord::new( + linear_execution_identity(issue, review_context), + event_type, + tracker_tool_bridge::current_timestamp(), + &anchor, + ); + + record.branch = Some(review_context.branch_name.clone()); + record.worktree_path = Some(review_context.worktree_path.clone()); + record.pr_url = Some(pull_request.url.clone()); + record.pr_head_sha = Some(pull_request.head_ref_oid.clone()); + record.pr_base_ref = Some(pull_request.base_ref_name.clone()); + record.commit_sha = Some(pull_request.head_ref_oid.clone()); + record.validation_result = Some(String::from("passed")); + record.summary = Some(summary.to_owned()); + record.terminal_path = Some(terminal_path.to_owned()); + record.verification = Some(vec![String::from("repo gate passed before tracker writeback")]); + + record +} + +fn linear_execution_closeout_event( + issue: &TrackerIssue, + review_context: &ReviewHandoffContext, + pull_request: &PullRequestDetails, + summary: &str, +) -> records::LinearExecutionEventRecord { + let anchor = records::stable_event_anchor(&[&pull_request.url, &pull_request.head_ref_oid]); + let mut record = records::LinearExecutionEventRecord::new( + linear_execution_identity(issue, review_context), + "closeout", + tracker_tool_bridge::current_timestamp(), + &anchor, + ); + + record.branch = Some(review_context.branch_name.clone()); + record.worktree_path = Some(review_context.worktree_path.clone()); + record.pr_url = Some(pull_request.url.clone()); + record.commit_sha = Some(pull_request.head_ref_oid.clone()); + record.summary = Some(summary.to_owned()); + record.validation_result = Some(String::from("passed")); + + record +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests.rs new file mode 100644 index 00000000..47c62a79 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests.rs @@ -0,0 +1,576 @@ +use std::{ + cell::RefCell, + collections::HashMap, + env, + ffi::OsString, + fs, + path::{Path, PathBuf}, + process::{self, Command}, +}; + +use tempfile::TempDir; + +use crate::{ + agent::tracker_tool_bridge::{ + DynamicToolContentItem, DynamicToolHandler, ISSUE_COMMENT_TOOL_NAME, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, LocalRepoDetails, + LocalRepoInspector, PullRequestDetails, PullRequestInspector, ReviewExecutionMode, + ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, + ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, + TurnCompletionStatus, + }, + config::InternalReviewMode, + prelude::eyre, + state::{self, ReviewHandoffMarker, ReviewOrchestrationMarker, StateStore}, + tracker::{ + IssueTracker, TrackerComment, TrackerIssue, TrackerLabel, TrackerState, TrackerTeam, + records, + }, + workflow::WorkflowDocument, +}; + +// Tracker mutation policy for active execution turns. +include!("tests/mutation/dispatch.rs"); +include!("tests/mutation/continuation.rs"); +include!("tests/mutation/progress.rs"); + +// Review handoff, repair, closeout, and internal-review policy. +include!("tests/review/policy.rs"); +include!("tests/review/handoff.rs"); + +const TEST_SERVICE_ID: &str = "pubfi"; + +struct FakeTracker { + state_updates: RefCell>, + label_updates: RefCell>>, + label_additions: RefCell>>, + label_removals: RefCell>>, + comments: RefCell>, + issue_comments: RefCell>>, + refresh_snapshots: RefCell>>, + issues_by_label: RefCell>>, + team_label_ids_by_name: RefCell>, + fail_state_update: RefCell>, + fail_label_update: RefCell>, + fail_comment: RefCell>, +} +impl FakeTracker { + fn new() -> Self { + Self { + state_updates: RefCell::new(Vec::new()), + label_updates: RefCell::new(Vec::new()), + label_additions: RefCell::new(Vec::new()), + label_removals: RefCell::new(Vec::new()), + comments: RefCell::new(Vec::new()), + issue_comments: RefCell::new(HashMap::new()), + refresh_snapshots: RefCell::new(Vec::new()), + issues_by_label: RefCell::new(HashMap::new()), + team_label_ids_by_name: RefCell::new(HashMap::new()), + fail_state_update: RefCell::new(None), + fail_label_update: RefCell::new(None), + fail_comment: RefCell::new(None), + } + } + + fn with_refresh_snapshots(refresh_snapshots: Vec>) -> Self { + let tracker = Self::new(); + + tracker.refresh_snapshots.replace(refresh_snapshots); + + tracker + } + + fn with_state_update_error(message: &str) -> Self { + let tracker = Self::new(); + + tracker.fail_state_update.replace(Some(message.to_owned())); + + tracker + } + + fn with_label_update_error(message: &str) -> Self { + let tracker = Self::new(); + + tracker.fail_label_update.replace(Some(message.to_owned())); + + tracker + } + + fn with_comment_error(message: &str) -> Self { + let tracker = Self::new(); + + tracker.fail_comment.replace(Some(message.to_owned())); + + tracker + } + + fn with_label_lookup_issues(self, label_name: &str, issues: Vec) -> Self { + self.issues_by_label.borrow_mut().insert(label_name.to_owned(), issues); + + self + } + + fn with_team_label_lookup_id(self, team_id: &str, label_name: &str, label_id: &str) -> Self { + self.team_label_ids_by_name + .borrow_mut() + .insert((team_id.to_owned(), label_name.to_owned()), label_id.to_owned()); + + self + } +} + +impl IssueTracker for FakeTracker { + fn list_issues_with_label( + &self, + label_name: &str, + ) -> crate::prelude::Result> { + Ok(self.issues_by_label.borrow().get(label_name).cloned().unwrap_or_default()) + } + + fn find_team_label_id( + &self, + team_id: &str, + label_name: &str, + ) -> crate::prelude::Result> { + Ok(self + .team_label_ids_by_name + .borrow() + .get(&(team_id.to_owned(), label_name.to_owned())) + .cloned()) + } + + fn get_issue_by_identifier( + &self, + _issue_identifier: &str, + ) -> crate::prelude::Result> { + Ok(None) + } + + fn refresh_issues(&self, _issue_ids: &[String]) -> crate::prelude::Result> { + if self.refresh_snapshots.borrow().is_empty() { + return Ok(Vec::new()); + } + + Ok(self.refresh_snapshots.borrow_mut().remove(0)) + } + + fn list_comments(&self, issue_id: &str) -> crate::prelude::Result> { + Ok(self.issue_comments.borrow().get(issue_id).cloned().unwrap_or_default()) + } + + fn update_issue_state(&self, _issue_id: &str, state_id: &str) -> crate::prelude::Result<()> { + if let Some(message) = self.fail_state_update.borrow().as_ref() { + return Err(eyre::eyre!(message.clone())); + } + + self.state_updates.borrow_mut().push(state_id.to_owned()); + + Ok(()) + } + + fn add_issue_labels( + &self, + _issue_id: &str, + label_ids: &[String], + ) -> crate::prelude::Result<()> { + if let Some(message) = self.fail_label_update.borrow().as_ref() { + return Err(eyre::eyre!(message.clone())); + } + + self.label_additions.borrow_mut().push(label_ids.to_vec()); + + Ok(()) + } + + fn remove_issue_labels( + &self, + _issue_id: &str, + label_ids: &[String], + ) -> crate::prelude::Result<()> { + if let Some(message) = self.fail_label_update.borrow().as_ref() { + return Err(eyre::eyre!(message.clone())); + } + + self.label_removals.borrow_mut().push(label_ids.to_vec()); + + Ok(()) + } + + fn create_comment(&self, _issue_id: &str, body: &str) -> crate::prelude::Result<()> { + if let Some(message) = self.fail_comment.borrow().as_ref() { + return Err(eyre::eyre!(message.clone())); + } + + self.comments.borrow_mut().push(body.to_owned()); + self.issue_comments.borrow_mut().entry(_issue_id.to_owned()).or_default().push( + TrackerComment { + body: body.to_owned(), + created_at: String::from("2026-04-12T00:00:00Z"), + }, + ); + + Ok(()) + } +} + +struct FakePullRequestInspector { + responses: RefCell>>, +} +impl FakePullRequestInspector { + fn new(responses: Vec>) -> Self { + Self { responses: RefCell::new(responses) } + } +} + +impl PullRequestInspector for FakePullRequestInspector { + fn inspect_pull_request( + &self, + _cwd: &Path, + _pr_url: &str, + _github_token: &str, + ) -> std::result::Result { + self.responses.borrow_mut().remove(0) + } +} + +struct GitHubTokenAssertingPullRequestInspector { + expected_token: String, + response: PullRequestDetails, +} +impl PullRequestInspector for GitHubTokenAssertingPullRequestInspector { + fn inspect_pull_request( + &self, + _cwd: &Path, + _pr_url: &str, + github_token: &str, + ) -> std::result::Result { + assert_eq!(github_token, self.expected_token.as_str()); + + Ok(self.response.clone()) + } +} + +struct FakeLocalRepoInspector { + responses: RefCell>>, +} +impl FakeLocalRepoInspector { + fn new(responses: Vec>) -> Self { + Self { responses: RefCell::new(responses) } + } +} + +impl LocalRepoInspector for FakeLocalRepoInspector { + fn inspect_local_repo(&self, _cwd: &Path) -> std::result::Result { + let mut responses = self.responses.borrow_mut(); + + match responses.len() { + 0 => panic!("fake local repo inspector ran out of responses"), + 1 => responses[0].clone(), + _ => responses.remove(0), + } + } +} + +struct TestEnvVarGuard { + key: String, + previous: Option, +} +impl TestEnvVarGuard { + fn set(key: impl Into, value: &str) -> Self { + let key = key.into(); + let previous = env::var_os(&key); + + unsafe { env::set_var(&key, value) }; + + Self { key, previous } + } +} + +impl Drop for TestEnvVarGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(previous) => unsafe { env::set_var(&self.key, previous) }, + None => unsafe { env::remove_var(&self.key) }, + } + } +} + +fn sample_issue() -> TrackerIssue { + TrackerIssue { + id: String::from("issue-1"), + identifier: String::from("DEC-1"), + #[cfg(test)] + project_slug: Some(String::from("decodex")), + title: String::from("Sample"), + description: String::from("Body"), + priority: Some(3), + created_at: String::from("2026-03-13T04:16:17.133Z"), + updated_at: String::from("2026-03-13T04:16:17.133Z"), + state: TrackerState { id: String::from("state-todo"), name: String::from("Todo") }, + team: TrackerTeam { + id: String::from("team-1"), + name: String::from("Decodex"), + states: vec![ + TrackerState { id: String::from("state-todo"), name: String::from("Todo") }, + TrackerState { + id: String::from("state-progress"), + name: String::from("In Progress"), + }, + TrackerState { id: String::from("state-review"), name: String::from("In Review") }, + ], + labels: vec![ + TrackerLabel { + id: String::from("label-queued"), + name: crate::tracker::automation_queue_label(TEST_SERVICE_ID), + }, + TrackerLabel { + id: String::from("label-active"), + name: crate::tracker::automation_active_label(TEST_SERVICE_ID), + }, + TrackerLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }, + TrackerLabel { + id: String::from("label-needs"), + name: String::from("decodex:needs-attention"), + }, + ], + }, + labels_complete: true, + labels: Vec::new(), + blockers: Vec::new(), + } +} + +fn sample_in_progress_issue() -> TrackerIssue { + let mut issue = sample_issue(); + + issue.state = + TrackerState { id: String::from("state-progress"), name: String::from("In Progress") }; + + issue +} + +fn sample_review_issue() -> TrackerIssue { + let mut issue = sample_issue(); + + issue.state = + TrackerState { id: String::from("state-review"), name: String::from("In Review") }; + + issue +} + +fn tracker_with_current_issue_snapshot(issue: &TrackerIssue) -> FakeTracker { + FakeTracker::with_refresh_snapshots(vec![vec![issue.clone()]]) +} + +fn sample_workflow() -> WorkflowDocument { + sample_workflow_with_tracker_states(&["Todo"], "In Progress", "In Review", "Todo") +} + +fn sample_workflow_with_startable_states(startable_states: &[&str]) -> WorkflowDocument { + sample_workflow_with_tracker_states(startable_states, "In Progress", "In Review", "Todo") +} + +fn sample_workflow_with_tracker_states( + startable_states: &[&str], + in_progress_state: &str, + success_state: &str, + failure_state: &str, +) -> WorkflowDocument { + let startable_states = + startable_states.iter().map(|state| format!("\"{state}\"")).collect::>().join(", "); + + WorkflowDocument::parse_markdown(&format!( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = [{startable_states}] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "{in_progress_state}" +success_state = "{success_state}" +completed_state = "Done" +failure_state = "{failure_state}" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {{}} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Use the tracker tools. +"#, + )) + .expect("workflow should parse") +} + +fn sample_review_context() -> ReviewHandoffContext { + ReviewHandoffContext { + attempt_number: 2, + branch_name: String::from("x/decodex-pub-618"), + run_id: String::from("pub-618-attempt-2-123"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: String::from(".worktrees/PUB-618"), + cwd: PathBuf::from("/tmp/PUB-618"), + github_token_env_var: Some(String::from("HOME")), + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Handoff, + recorded_pr_url: None, + } +} + +fn sample_review_context_in(cwd: &Path) -> ReviewHandoffContext { + ReviewHandoffContext { + attempt_number: 2, + branch_name: String::from("x/decodex-pub-618"), + run_id: String::from("pub-618-attempt-2-123"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: String::from(".worktrees/PUB-618"), + cwd: cwd.to_path_buf(), + github_token_env_var: Some(String::from("HOME")), + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Handoff, + recorded_pr_url: None, + } +} + +fn sample_review_repair_context_in(cwd: &Path, pr_url: &str) -> ReviewHandoffContext { + ReviewHandoffContext { + attempt_number: 3, + branch_name: String::from("x/decodex-pub-618"), + run_id: String::from("pub-618-attempt-3-123"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: String::from(".worktrees/PUB-618"), + cwd: cwd.to_path_buf(), + github_token_env_var: Some(String::from("HOME")), + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Repair, + recorded_pr_url: Some(pr_url.to_owned()), + } +} + +fn sample_closeout_context_in(cwd: &Path, pr_url: &str) -> ReviewHandoffContext { + ReviewHandoffContext { + attempt_number: 4, + branch_name: String::from("x/decodex-pub-618"), + run_id: String::from("pub-618-attempt-4-123"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: String::from(".worktrees/PUB-618"), + cwd: cwd.to_path_buf(), + github_token_env_var: Some(String::from("HOME")), + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Closeout, + recorded_pr_url: Some(pr_url.to_owned()), + } +} + +fn sample_pull_request() -> PullRequestDetails { + PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/48"), + } +} + +fn sample_local_repo() -> LocalRepoDetails { + LocalRepoDetails { + default_branch: String::from("main"), + head_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + repository_name: String::from("decodex"), + repository_owner: String::from("hack-ink"), + } +} + +fn write_review_policy_checkpoint( + review_context: &ReviewHandoffContext, + phase: &str, + status: &str, + head_sha: &str, + nonclean_rounds: i64, +) { + fs::create_dir_all(&review_context.cwd).expect("review policy worktree should exist"); + state::write_run_review_policy_state( + &review_context.cwd, + &review_context.run_id, + review_context.attempt_number, + phase, + status, + head_sha, + nonclean_rounds, + ) + .expect("review policy state should write"); +} + +fn write_clean_review_checkpoint(review_context: &ReviewHandoffContext) { + let phase = match review_context.mode { + ReviewExecutionMode::Handoff => "handoff", + ReviewExecutionMode::Repair => "repair", + ReviewExecutionMode::Closeout => { + panic!("closeout does not support review checkpoints") + }, + }; + + write_review_policy_checkpoint( + review_context, + phase, + "clean", + &sample_local_repo().head_oid, + 0, + ); +} + +fn bridge_state_store<'a>(bridge: &TrackerToolBridge<'a>) -> &'a StateStore { + bridge.state_store.expect("test bridge should have a runtime state store") +} + +fn persisted_review_handoff_marker( + bridge: &TrackerToolBridge<'_>, + issue: &TrackerIssue, + review_context: &ReviewHandoffContext, +) -> ReviewHandoffMarker { + bridge_state_store(bridge) + .review_handoff_marker(&review_context.service_id, &issue.id, &review_context.branch_name) + .expect("review handoff marker should read") + .expect("review handoff marker should exist") +} + +fn persisted_review_orchestration_marker( + bridge: &TrackerToolBridge<'_>, + issue: &TrackerIssue, + review_context: &ReviewHandoffContext, + review_handoff: &ReviewHandoffMarker, +) -> ReviewOrchestrationMarker { + bridge_state_store(bridge) + .review_orchestration_marker(&review_context.service_id, &issue.id, review_handoff) + .expect("review orchestration marker should read") + .expect("review orchestration marker should exist") +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/continuation.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/continuation.rs new file mode 100644 index 00000000..fea824d8 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/continuation.rs @@ -0,0 +1,444 @@ +#[test] +fn completion_disposition_allows_manual_attention_exit_without_review_handoff() { + let issue = sample_issue(); + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + let comment_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_COMMENT_TOOL_NAME, + serde_json::json!({ "body": "Blocked on missing tracker permission; handing off for manual repair." }), + ); + + assert!(response.success); + assert!(comment_response.success); + assert_eq!( + bridge.completion_disposition().expect("manual attention should be accepted"), + RunCompletionDisposition::ManualAttention + ); +} + +#[test] +fn turn_completion_rejects_xy_156_shape_without_terminal_tracker_action() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let error = DynamicToolHandler::validate_turn_completion( + &bridge, + "Implementation and tests are done, but commit, push, PR, and tracker handoff remain.", + ) + .expect_err("turn completion should reject missing terminal tracker actions"); + + assert!(error.to_string().contains("recorded neither `issue_review_handoff`")); +} + +#[test] +fn turn_classification_allows_continuation_without_terminal_tracker_action() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + + assert_eq!( + DynamicToolHandler::classify_turn_completion( + &bridge, + "Still implementing; no terminal tracker action has been recorded yet." + ) + .expect("missing terminal action should request continuation"), + TurnCompletionStatus::Continue + ); +} + +#[test] +fn turn_classification_rejects_clean_closeout_continuation() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let merged_pull_request = { + let mut pull_request = sample_pull_request(); + + pull_request.url = String::from(pr_url); + pull_request.state = String::from("MERGED"); + + pull_request + }; + let inspector = FakePullRequestInspector::new(vec![Ok(merged_pull_request)]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + &inspector, + &local_repo_inspector, + ); + let error = DynamicToolHandler::classify_turn_completion( + &bridge, + "Still re-reading merged closeout context; no terminal tracker action has been recorded yet.", + ) + .expect_err("closeout should not yield another clean continuation boundary"); + + assert!(error.to_string().contains("deterministic tail")); + assert!(error.to_string().contains(ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME)); + assert!(error.to_string().contains(ISSUE_TERMINAL_FINALIZE_TOOL_NAME)); +} + +#[test] +fn turn_classification_rejects_continuation_blocking_writes_without_terminal_path() { + for (tool_name, args) in [ + (ISSUE_LABEL_ADD_TOOL_NAME, serde_json::json!({ "label": "decodex:manual-only" })), + (ISSUE_TRANSITION_TOOL_NAME, serde_json::json!({ "state": "Todo" })), + ] { + let mut refreshed_issue = sample_issue(); + + if tool_name == ISSUE_LABEL_ADD_TOOL_NAME { + refreshed_issue.labels.push(TrackerLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }); + } else { + refreshed_issue.state = + TrackerState { id: String::from("state-todo"), name: String::from("Todo") }; + } + + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![refreshed_issue]]); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call(&bridge, tool_name, args); + + assert!(response.success); + + let error = DynamicToolHandler::classify_turn_completion( + &bridge, + "The lane recorded a continuation-blocking tracker write without a terminal path.", + ) + .expect_err("continuation-blocking writes must not exit via a clean boundary"); + + assert!(error.to_string().contains("without recording a terminal path")); + assert!(error.to_string().contains(tool_name)); + } +} + +#[test] +fn turn_classification_rejects_continuation_blocking_writes_for_stale_active_refresh() { + for (tool_name, args) in [ + (ISSUE_LABEL_ADD_TOOL_NAME, serde_json::json!({ "label": "decodex:manual-only" })), + (ISSUE_TRANSITION_TOOL_NAME, serde_json::json!({ "state": "Todo" })), + ] { + let active_issue = sample_in_progress_issue(); + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![active_issue]]); + let issue = sample_in_progress_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call(&bridge, tool_name, args); + + assert!(response.success); + + let error = DynamicToolHandler::classify_turn_completion( + &bridge, + "The run started active, so a stale active reread must not clear a local stop write.", + ) + .expect_err("active-start lanes must keep local stop writes blocking"); + + assert!(error.to_string().contains("without recording a terminal path")); + assert!(error.to_string().contains(tool_name)); + } +} + +#[test] +fn turn_classification_allows_continuation_blocking_writes_after_reactivation() { + for (tool_name, args) in [ + (ISSUE_LABEL_ADD_TOOL_NAME, serde_json::json!({ "label": "decodex:manual-only" })), + (ISSUE_TRANSITION_TOOL_NAME, serde_json::json!({ "state": "Todo" })), + ] { + let mut reactivated_issue = sample_issue(); + + reactivated_issue.state = + TrackerState { id: String::from("state-progress"), name: String::from("In Progress") }; + + let tracker = FakeTracker::with_refresh_snapshots(vec![ + vec![reactivated_issue.clone()], + vec![reactivated_issue], + ]); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call(&bridge, tool_name, args); + + assert!(response.success); + assert_eq!( + DynamicToolHandler::classify_turn_completion( + &bridge, + "The issue was reactivated before turn completion, so the stale stop write must not block continuation." + ) + .expect("startable-start lanes should allow continuation after reactivation"), + TurnCompletionStatus::Continue + ); + } +} + +#[test] +fn manual_attention_requires_explanatory_comment() { + let issue = sample_issue(); + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(response.success); + + let error = bridge + .completion_disposition() + .expect_err("manual attention must require an explanatory comment"); + + assert!(error.to_string().contains("never recorded the required explanatory comment")); +} + +#[test] +fn failed_needs_attention_label_update_does_not_record_manual_attention() { + let tracker = FakeTracker::with_label_update_error("tracker labels unavailable"); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(!response.success); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + + let error = bridge + .completion_disposition() + .expect_err("failed label writes must not count as manual attention"); + + assert!(error.to_string().contains("recorded neither")); +} + +#[test] +fn label_add_refreshes_issue_snapshot_before_merging_label_ids() { + let initial_issue = sample_issue(); + let mut refreshed_issue = initial_issue.clone(); + + refreshed_issue.labels.push(TrackerLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }); + + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![refreshed_issue]]); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &initial_issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(response.success); + assert_eq!(tracker.label_additions.borrow().as_slice(), [vec![String::from("label-needs")]]); +} + +#[test] +fn label_add_fails_when_refresh_returns_no_snapshot() { + let tracker = FakeTracker::with_refresh_snapshots(vec![Vec::new()]); + let workflow = sample_workflow(); + let issue = sample_issue(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(!response.success); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: format!( + "Failed to refresh issue `{}` before updating labels: tracker returned no current snapshot.", + issue.identifier + ), + }] + ); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); +} + +#[test] +fn turn_classification_rejects_continuation_blocking_write_when_refresh_returns_no_snapshot() { + let mut opted_out_issue = sample_issue(); + + opted_out_issue.labels.push(TrackerLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }); + + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![opted_out_issue], Vec::new()]); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:manual-only" }), + ); + + assert!(response.success); + + let error = DynamicToolHandler::classify_turn_completion( + &bridge, + "The lane recorded a continuation-blocking tracker write without a terminal path.", + ) + .expect_err("missing refresh snapshots must not allow a clean continuation boundary"); + + assert!(error.to_string().contains("without recording a terminal path")); + assert!(error.to_string().contains(ISSUE_LABEL_ADD_TOOL_NAME)); +} + +#[test] +fn completion_disposition_rejects_conflicting_review_handoff_and_manual_attention() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let issue = sample_issue(); + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/48"), + })]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &inspector, + &local_repo_inspector, + ); + + write_clean_review_checkpoint(&sample_review_context_in(temp_dir.path())); + + let review_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + let label_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(review_response.success); + assert!(label_response.success); + + let error = bridge + .completion_disposition() + .expect_err("conflicting completion signals should be rejected"); + + assert!(error.to_string().contains("Use exactly one final tracker exit path.")); +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/dispatch.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/dispatch.rs new file mode 100644 index 00000000..fd3805b9 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/dispatch.rs @@ -0,0 +1,858 @@ +use records::CLOSEOUT_RECORD_TYPE; +use records::CloseoutRecord; + +use crate::tracker; + +#[test] +fn closeout_apply_validates_merged_pr_and_completed_issue_state() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + let tracker = FakeTracker::with_refresh_snapshots(vec![ + vec![completed_issue.clone()], + vec![completed_issue], + ]); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let mut merged_pull_request = sample_pull_request(); + + merged_pull_request.url = String::from(pr_url); + merged_pull_request.state = String::from("MERGED"); + + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Merged the approved lane and finished closeout." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "closeout" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect("closeout completion should allow the turn to complete"); + + bridge.apply_closeout().expect("closeout should validate cleanly"); + + let comments = tracker.comments.borrow(); + + assert_eq!(comments.len(), 1); + assert!(comments[0].contains("decodex closeout completed")); + assert!(comments[0].contains("\"record_type\": \"decodex.linear_execution_event\"")); + assert!(comments[0].contains("\"event_type\": \"closeout\"")); + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn closeout_apply_writes_coarse_comment_without_replaying_existing_records() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + completed_issue.labels.push(TrackerLabel { + id: String::from("label-active"), + name: tracker::automation_active_label(TEST_SERVICE_ID), + }); + + let tracker = FakeTracker::with_refresh_snapshots(vec![ + vec![completed_issue.clone()], + vec![completed_issue], + ]); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let mut merged_pull_request = sample_pull_request(); + + merged_pull_request.url = String::from(pr_url); + merged_pull_request.state = String::from("MERGED"); + + let existing_record = records::append_structured_comment_record( + "decodex closeout completed", + &CloseoutRecord { + record_type: String::from(CLOSEOUT_RECORD_TYPE), + completed_at: String::from("2026-04-12T00:00:00Z"), + run_id: String::from("pub-618-attempt-4-123"), + attempt_number: 4, + branch_name: String::from("x/decodex-pub-618"), + pr_url: String::from(pr_url), + }, + ) + .expect("closeout record should serialize"); + + tracker.issue_comments.borrow_mut().insert( + issue.id.clone(), + vec![TrackerComment { + body: existing_record, + created_at: String::from("2026-04-12T00:00:00Z"), + }], + ); + + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Merged the approved lane and finished closeout." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "closeout" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + bridge.apply_closeout().expect("closeout should persist a coarse tracker summary"); + + assert_eq!(tracker.comments.borrow().len(), 1); + assert!( + tracker.label_removals.borrow().is_empty(), + "closeout writeback should keep ownership until cleanup succeeds" + ); + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn closeout_clear_uses_server_team_label_lookup_for_active_label_removal() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + completed_issue + .labels + .push(TrackerLabel { id: String::from("label-active"), name: active_label.clone() }); + completed_issue + .labels + .push(TrackerLabel { id: String::from("label-queued"), name: queue_label.clone() }); + completed_issue.team.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::with_refresh_snapshots(vec![ + vec![completed_issue.clone()], + vec![completed_issue.clone()], + ]) + .with_team_label_lookup_id(&completed_issue.team.id, &active_label, "label-active"); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let mut merged_pull_request = sample_pull_request(); + + merged_pull_request.url = String::from(pr_url); + merged_pull_request.state = String::from("MERGED"); + + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Merged the approved lane and finished closeout." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "closeout" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + bridge.clear_closeout_issue_scope().expect( + "closeout cleanup should resolve the active label id server-side when team labels paginate", + ); + + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [vec![String::from("label-active")], vec![String::from("label-queued")],] + ); + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn closeout_apply_keeps_active_label_until_cleanup() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + completed_issue.labels_complete = false; + + completed_issue.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::with_refresh_snapshots(vec![ + vec![completed_issue.clone()], + vec![completed_issue.clone()], + ]) + .with_label_lookup_issues(&active_label, vec![completed_issue.clone()]); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let mut merged_pull_request = sample_pull_request(); + + merged_pull_request.url = String::from(pr_url); + merged_pull_request.state = String::from("MERGED"); + + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Merged the approved lane and finished closeout." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "closeout" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + bridge + .apply_closeout() + .expect("closeout writeback should succeed without clearing the active label"); + + assert!( + tracker.label_removals.borrow().is_empty(), + "closeout writeback should not clear ownership before cleanup" + ); + assert!(tracker.label_updates.borrow().is_empty()); +} + +#[test] +fn closeout_clear_clears_active_label_when_issue_labels_paginate() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + completed_issue.labels_complete = false; + + completed_issue.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![completed_issue.clone()]]) + .with_label_lookup_issues(&active_label, vec![completed_issue.clone()]) + .with_label_lookup_issues(&queue_label, vec![completed_issue.clone()]); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let merged_pull_request = { + let mut pull_request = sample_pull_request(); + + pull_request.url = String::from(pr_url); + pull_request.state = String::from("MERGED"); + + pull_request + }; + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + + bridge + .clear_closeout_issue_scope() + .expect("closeout cleanup should clear the active and queue labels incrementally when issue labels paginate"); + + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [vec![String::from("label-active")], vec![String::from("label-queued")],] + ); +} + +#[test] +fn closeout_clear_treats_missing_lane_label_removal_as_idempotent() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + completed_issue + .labels + .push(TrackerLabel { id: String::from("label-active"), name: active_label }); + completed_issue + .labels + .push(TrackerLabel { id: String::from("label-queued"), name: queue_label }); + + let tracker = + FakeTracker::with_label_update_error("Linear GraphQL request failed: Label not on issue"); + + tracker.refresh_snapshots.replace(vec![vec![completed_issue.clone()]]); + + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let merged_pull_request = { + let mut pull_request = sample_pull_request(); + + pull_request.url = String::from(pr_url); + pull_request.state = String::from("MERGED"); + + pull_request + }; + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + + bridge + .clear_closeout_issue_scope() + .expect("closeout cleanup should ignore already-absent Linear lane labels"); +} + +#[test] +fn closeout_clear_skips_lane_labels_when_server_confirms_absent() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut completed_issue = sample_review_issue(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + completed_issue + .labels + .retain(|label| label.name != active_label.as_str() && label.name != queue_label.as_str()); + + let tracker = FakeTracker::with_refresh_snapshots(vec![vec![completed_issue.clone()]]); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let merged_pull_request = { + let mut pull_request = sample_pull_request(); + + pull_request.url = String::from(pr_url); + pull_request.state = String::from("MERGED"); + + pull_request + }; + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + + bridge + .clear_closeout_issue_scope() + .expect("closeout cleanup should be idempotent after lane labels are already gone"); + + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn closeout_complete_rejects_issue_that_is_not_yet_completed() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = tracker_with_current_issue_snapshot(&sample_review_issue()); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/261"; + let mut merged_pull_request = sample_pull_request(); + + merged_pull_request.url = String::from(pr_url); + merged_pull_request.state = String::from("MERGED"); + + let inspector = FakePullRequestInspector::new(vec![ + Ok(merged_pull_request.clone()), + Ok(merged_pull_request), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_inspectors( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + Some(TrackerToolBridge::leaked_test_state_store()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Merged the approved lane and attempted closeout." + }), + ); + + assert!(!response.success); + assert!(matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText { text }] + if text.contains("requires tracker state `Done`") + && text.contains("Move the issue to `Done` with `issue_transition` before calling `issue_closeout_complete`") + )); +} + +#[test] +fn review_handoff_inspection_uses_configured_github_token() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = GitHubTokenAssertingPullRequestInspector { + expected_token: env::var("HOME").expect("HOME should exist"), + response: sample_pull_request(), + }; + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(response.success); +} + +#[test] +fn review_handoff_inspection_rejects_missing_or_blank_github_token() { + { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let mut review_context = sample_review_context_in(temp_dir.path()); + + review_context.github_token_env_var = None; + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(!response.success); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: String::from( + "`github.token_env_var` must be configured for PR-backed review handoff validation.", + ), + }] + ); + } + { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let env_var = + format!("DECODEX_TEST_BLANK_REVIEW_HANDOFF_GITHUB_TOKEN_ENV_{}", process::id()); + let _env_guard = TestEnvVarGuard::set(&env_var, ""); + let mut review_context = sample_review_context_in(temp_dir.path()); + + review_context.github_token_env_var = Some(env_var.clone()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(!response.success); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: format!( + "Environment variable `{env_var}` referenced by `github.token_env_var` must not be blank." + ), + }] + ); + } +} + +#[test] +fn transitions_current_issue_with_allowed_state() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "issue_identifier": "DEC-1", "state": "In Progress" }), + ); + + assert!(response.success); + assert_eq!(tracker.state_updates.borrow().as_slice(), ["state-progress"]); +} + +#[test] +fn rejects_success_transition_without_review_handoff() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "In Review" }), + ); + + assert!(!response.success); + assert!(tracker.state_updates.borrow().is_empty()); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: String::from( + "State `In Review` requires `issue_review_handoff` after the branch is pushed and a reviewable PR exists." + ), + }] + ); +} + +#[test] +fn rejects_invalid_transition_argument_shapes() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let cases = [ + ( + serde_json::json!({ "unexpected_state_field": "In Progress" }), + "Invalid `issue.transition` arguments: missing field `state`", + ), + ( + serde_json::json!({ + "state": "In Progress", + "unexpected_state_field": "In Progress", + }), + "Invalid `issue.transition` arguments: unknown field `unexpected_state_field`", + ), + ]; + + for (arguments, message) in cases { + let response = + DynamicToolHandler::handle_call(&bridge, ISSUE_TRANSITION_TOOL_NAME, arguments); + + assert!(!response.success); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { text: String::from(message) }] + ); + } + + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn rejects_success_transition_regardless_of_other_workflow_state_membership() { + for workflow in [ + sample_workflow_with_startable_states(&["Todo", "In Review"]), + sample_workflow_with_tracker_states(&["Todo"], "In Progress", "In Review", "In Review"), + sample_workflow_with_tracker_states(&["Todo"], "In Review", "In Review", "Todo"), + ] { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + + assert_success_transition_requires_review_handoff(workflow, &tracker, &issue); + } +} + +fn assert_success_transition_requires_review_handoff( + workflow: WorkflowDocument, + tracker: &FakeTracker, + issue: &TrackerIssue, +) { + let bridge = TrackerToolBridge::new(tracker, issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "In Review" }), + ); + + assert!(!response.success); + assert!(tracker.state_updates.borrow().is_empty()); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { + text: String::from( + "State `In Review` requires `issue_review_handoff` after the branch is pushed and a reviewable PR exists." + ), + }] + ); +} + +#[test] +fn rejects_tool_calls_for_another_issue() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_COMMENT_TOOL_NAME, + serde_json::json!({ "issue_identifier": "DEC-999", "body": "hello" }), + ); + + assert!(!response.success); + assert!(tracker.comments.borrow().is_empty()); +} + +#[test] +fn accepts_public_comments_without_sensitive_paths() { + for body in [ + "Started work and running validation now.", + "decodex run failed and will retry\n\n- worktree_path: `.worktrees/DEC-1`", + ] { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_COMMENT_TOOL_NAME, + serde_json::json!({ "body": body }), + ); + + assert!(response.success, "comment should be accepted: {body}"); + assert_eq!(tracker.comments.borrow().as_slice(), [body]); + } +} + +#[test] +fn rejects_public_comments_with_sensitive_or_unknown_paths() { + for (body, expected_error) in [ + ( + "decodex run failed and will retry\n\n- worktree_path: `/absolute/path/to/repo/.worktrees/DEC-1`", + "`worktree_path` must be repository-relative, not `/absolute/path/to/repo/.worktrees/DEC-1`.", + ), + ( + "decodex run failed and will retry\n\n- unexpected_path: `/absolute/path/to/repo/.worktrees/DEC-1`", + "Unsupported structured field `unexpected_path` in public issue comments.", + ), + ( + "decodex run failed and will retry\n\n- worktree_path: `C:/absolute/path/to/repo/.worktrees/DEC-1`", + "`worktree_path` must be repository-relative, not `C:/absolute/path/to/repo/.worktrees/DEC-1`.", + ), + ] { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_COMMENT_TOOL_NAME, + serde_json::json!({ "body": body }), + ); + + assert!(!response.success, "comment should be rejected: {body}"); + assert!(tracker.comments.borrow().is_empty()); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { text: String::from(expected_error) }] + ); + } +} + +#[test] +fn adds_allowed_workflow_label() { + let issue = sample_issue(); + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + + assert!(response.success); + assert_eq!(tracker.label_additions.borrow().as_slice(), [vec![String::from("label-needs")]]); +} + +#[test] +fn rejects_invalid_label_add_argument_shapes() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let cases = [ + ( + serde_json::json!({ "unexpected_label_field": "decodex:needs-attention" }), + "Invalid `issue.label.add` arguments: missing field `label`", + ), + ( + serde_json::json!({ + "label": "decodex:needs-attention", + "unexpected_label_field": "decodex:needs-attention", + }), + "Invalid `issue.label.add` arguments: unknown field `unexpected_label_field`", + ), + ]; + + for (arguments, message) in cases { + let response = + DynamicToolHandler::handle_call(&bridge, ISSUE_LABEL_ADD_TOOL_NAME, arguments); + + assert!(!response.success); + assert_eq!( + response.content_items, + vec![DynamicToolContentItem::InputText { text: String::from(message) }] + ); + } + + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/progress.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/progress.rs new file mode 100644 index 00000000..d2cf581e --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/progress.rs @@ -0,0 +1,197 @@ +#[test] +fn progress_checkpoint_writes_structured_issue_comment() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "phase": "implementing", + "focus": "Wire the new execution-state skill into tracker-driven flows.", + "next_action": "Add the issue_progress_checkpoint runtime tool.", + "blockers": [], + "evidence": ["Research decision favors Linear-backed execution snapshots."], + "verification": ["Local inventory of active execution-state boundary references completed."], + "head_sha": sample_local_repo().head_oid, + "branch": "x/decodex-1", + "pr_url": "https://github.com/hack-ink/decodex/pull/12" + }), + ); + + assert!(response.success); + assert_eq!(tracker.comments.borrow().len(), 1); + + let comment = tracker.comments.borrow(); + let body = &comment[0]; + let record = records::parse_linear_execution_event_record(body) + .expect("progress checkpoint should be a Linear execution event"); + + assert!(body.starts_with("```json\n{")); + assert_eq!(record.record_type, records::LINEAR_EXECUTION_EVENT_RECORD_TYPE); + assert_eq!(record.event_type, "progress_checkpoint"); + assert_eq!(record.phase.as_deref(), Some("implementing")); + assert_eq!( + record.focus.as_deref(), + Some("Wire the new execution-state skill into tracker-driven flows.") + ); + assert_eq!( + record.next_action.as_deref(), + Some("Add the issue_progress_checkpoint runtime tool.") + ); + assert_eq!(record.commit_sha.as_deref(), Some(sample_local_repo().head_oid.as_str())); + assert_eq!(record.branch.as_deref(), Some("x/decodex-1")); +} + +#[test] +fn blocked_progress_checkpoint_requires_concrete_blocker() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "phase": "blocked", + "focus": "Unblock closeout.", + "next_action": "Wait for a blocker to be clarified.", + "blockers": [], + "evidence": [] + }), + ); + + assert!(!response.success); + assert!( + response + .content_items + .iter() + .any(|item| matches!(item, DynamicToolContentItem::InputText { text } if text.contains("requires at least one blocker"))) + ); + assert!(tracker.comments.borrow().is_empty()); +} + +#[test] +fn progress_checkpoint_rejects_stale_head_sha() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "phase": "implementing", + "focus": "Keep execution state tied to the current lane head.", + "next_action": "Reject stale checkpoint writes.", + "blockers": [], + "evidence": [], + "head_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + }), + ); + + assert!(!response.success); + assert!( + response + .content_items + .iter() + .any(|item| matches!(item, DynamicToolContentItem::InputText { text } if text.contains("does not match the current lane HEAD"))) + ); + assert!(tracker.comments.borrow().is_empty()); +} + +#[test] +fn progress_checkpoint_normalizes_matching_short_head_sha_to_full_head() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "phase": "closeout", + "focus": "Finish retained closeout bookkeeping.", + "next_action": "Record the closeout checkpoint with the live lane head.", + "blockers": [], + "evidence": [], + "head_sha": &sample_local_repo().head_oid[..7] + }), + ); + + assert!(response.success); + assert_eq!(tracker.comments.borrow().len(), 1); + + let record = records::parse_linear_execution_event_record(&tracker.comments.borrow()[0]) + .expect("progress checkpoint should be a Linear execution event"); + + assert_eq!(record.commit_sha.as_deref(), Some(sample_local_repo().head_oid.as_str())); +} + +#[test] +fn progress_checkpoint_retries_do_not_duplicate_same_ledger_event() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &pull_request_inspector, + &local_repo_inspector, + ); + let arguments = serde_json::json!({ + "phase": "implementing", + "focus": "Keep duplicate checkpoint writes idempotent.", + "next_action": "Retry the same tracker write.", + "blockers": [], + "evidence": ["The same logical checkpoint is being retried."], + "head_sha": sample_local_repo().head_oid + }); + let first = DynamicToolHandler::handle_call( + &bridge, + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + arguments.clone(), + ); + let second = + DynamicToolHandler::handle_call(&bridge, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, arguments); + + assert!(first.success); + assert!(second.success); + assert_eq!(tracker.comments.borrow().len(), 1); +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests/review/handoff.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests/review/handoff.rs new file mode 100644 index 00000000..3347e665 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests/review/handoff.rs @@ -0,0 +1,983 @@ +use records::REVIEW_HANDOFF_RECORD_TYPE; +use records::ReviewHandoffRecord; +use state::RUN_ACTIVITY_MARKER_FILE; + +#[test] +fn turn_completion_requires_explicit_terminal_finalize_after_review_handoff() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/52"), + })]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/52", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + let error = DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect_err("review handoff should still require explicit finalization"); + + assert!(error.to_string().contains(ISSUE_TERMINAL_FINALIZE_TOOL_NAME)); + assert!(error.to_string().contains("review_handoff")); +} + +#[test] +fn terminal_finalize_accepts_matching_review_handoff_path() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/53"), + })]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let review_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/53", + "summary": "Ready for review." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "review_handoff" }), + ); + + assert!(review_response.success); + assert!(finalize_response.success); + + DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect("matching finalization should allow the turn to complete"); +} + +#[test] +fn terminal_finalize_rejects_mismatched_path() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/54"), + })]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let review_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/54", + "summary": "Ready for review." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "manual_attention" }), + ); + + assert!(review_response.success); + assert!(!finalize_response.success); + assert!(matches!( + finalize_response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains( + "requested path `manual_attention`, but the recorded terminal path is `review_handoff`" + ) + )); +} + +#[test] +fn terminal_finalize_accepts_matching_manual_attention_path() { + let issue = sample_issue(); + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let label_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_LABEL_ADD_TOOL_NAME, + serde_json::json!({ "label": "decodex:needs-attention" }), + ); + let comment_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_COMMENT_TOOL_NAME, + serde_json::json!({ + "body": "Blocked on missing tracker permission; handing off for manual repair." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "manual_attention" }), + ); + + assert!(label_response.success); + assert!(comment_response.success); + assert!(finalize_response.success); + + DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect("matching manual-attention finalization should allow the turn to complete"); +} + +#[test] +fn rejects_review_handoff_apply_when_lane_head_changes_after_recording() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/47"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/47"), + }), + ]); + let mut updated_local_repo = sample_local_repo(); + + updated_local_repo.head_oid = String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + + let local_repo_inspector = FakeLocalRepoInspector::new(vec![ + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(updated_local_repo), + ]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/47", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + let error = bridge + .apply_review_handoff() + .expect_err("writeback should revalidate the current lane head"); + + assert!(error.to_string().contains("Push the latest lane commit before review handoff.")); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn review_handoff_apply_writes_coarse_comment_without_replaying_existing_records() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let mut pull_request = sample_pull_request(); + + pull_request.url = String::from("https://github.com/hack-ink/decodex/pull/47"); + + let inspector = + FakePullRequestInspector::new(vec![Ok(pull_request.clone()), Ok(pull_request.clone())]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let existing_record = records::append_structured_comment_record( + "Review handoff already persisted.", + &ReviewHandoffRecord { + record_type: String::from(REVIEW_HANDOFF_RECORD_TYPE), + completed_at: String::from("2026-04-12T00:00:00Z"), + run_id: review_context.run_id.clone(), + attempt_number: review_context.attempt_number, + branch_name: review_context.branch_name.clone(), + pr_url: pull_request.url.clone(), + target_base_ref_name: pull_request.base_ref_name.clone(), + pr_head_ref_name: pull_request.head_ref_name.clone(), + pr_head_oid: pull_request.head_ref_oid.clone(), + summary: String::from("Ready for review."), + }, + ) + .expect("review handoff record should serialize"); + + tracker.issue_comments.borrow_mut().insert( + issue.id.clone(), + vec![TrackerComment { + body: existing_record, + created_at: String::from("2026-04-12T00:00:00Z"), + }], + ); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": pull_request.url, + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + bridge + .apply_review_handoff() + .expect("writeback should persist runtime state and coarse tracker summary"); + + assert_eq!(tracker.comments.borrow().len(), 1); + assert_eq!(tracker.state_updates.borrow().as_slice(), [String::from("state-review")]); +} + +#[test] +fn review_handoff_apply_does_not_duplicate_existing_ledger_event() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let pull_request = sample_pull_request(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(pull_request.clone()), + Ok(pull_request.clone()), + Ok(pull_request.clone()), + Ok(pull_request.clone()), + ]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![ + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + ]); + let review_context = sample_review_context_in(temp_dir.path()); + + for _ in 0..2 { + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": pull_request.url.clone(), + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + bridge.apply_review_handoff().expect("review handoff should apply"); + } + + assert_eq!(tracker.comments.borrow().len(), 1); + + let record = records::parse_linear_execution_event_record(&tracker.comments.borrow()[0]) + .expect("review handoff should write a Linear execution event"); + + assert_eq!(record.event_type, "review_handoff"); + assert_eq!(record.idempotency_key.matches("review_handoff").count(), 1); +} + +#[test] +fn reports_partial_review_handoff_when_state_transition_fails_after_tracker_record_write() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::with_state_update_error("tracker state write failed"); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/49"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/49"), + }), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &inspector, + &local_repo_inspector, + ); + + write_clean_review_checkpoint(&sample_review_context_in(temp_dir.path())); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/49", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + let error = bridge.apply_review_handoff().expect_err( + "state transition failures must surface after the tracker handoff record write", + ); + + assert!(error.to_string().contains("tracker state write failed")); + assert_eq!(tracker.comments.borrow().len(), 1); + assert!(tracker.state_updates.borrow().is_empty()); + assert!( + bridge_state_store(&bridge) + .review_handoff_marker(TEST_SERVICE_ID, &issue.id, "x/decodex-pub-618") + .expect("runtime handoff marker read should succeed") + .is_none() + ); +} + +#[test] +fn reports_review_handoff_writeback_failure_when_tracker_comment_write_fails() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::with_comment_error("tracker comment write failed"); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/50"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/50"), + }), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &inspector, + &local_repo_inspector, + ); + + write_clean_review_checkpoint(&sample_review_context_in(temp_dir.path())); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/50", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + let error = bridge.apply_review_handoff().expect_err( + "comment write failures after state transition must surface as partial writeback", + ); + let writeback_error = error + .downcast_ref::() + .expect("partial writeback should use dedicated error type"); + + assert_eq!(writeback_error.issue_identifier, "DEC-1"); + assert_eq!(writeback_error.run_id, "pub-618-attempt-2-123"); + assert_eq!(writeback_error.success_state, "In Review"); + assert!(writeback_error.source.contains("failed to persist the tracker review handoff record")); + assert!(writeback_error.source.contains("tracker comment write failed")); + assert!(tracker.state_updates.borrow().is_empty()); + assert!(tracker.comments.borrow().is_empty()); + assert!( + bridge_state_store(&bridge) + .review_handoff_marker(TEST_SERVICE_ID, &issue.id, "x/decodex-pub-618") + .expect("runtime handoff marker read should succeed") + .is_none() + ); +} + +#[test] +fn review_handoff_persists_runtime_state_without_local_marker_cache() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/150"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/150"), + }), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &inspector, + &local_repo_inspector, + ); + + write_clean_review_checkpoint(&sample_review_context_in(temp_dir.path())); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/150", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + bridge + .apply_review_handoff() + .expect("runtime state persistence should not depend on local marker files"); + + assert_eq!(tracker.state_updates.borrow().as_slice(), ["state-review"]); + assert_eq!(tracker.comments.borrow().len(), 1); + assert_eq!( + persisted_review_handoff_marker( + &bridge, + &issue, + &sample_review_context_in(temp_dir.path()) + ) + .pr_url(), + "https://github.com/hack-ink/decodex/pull/150" + ); +} + +#[test] +fn review_handoff_does_not_record_completion_when_review_policy_clear_fails() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let marker_path = temp_dir.path().join(RUN_ACTIVITY_MARKER_FILE); + + fs::remove_file(&marker_path).expect("existing run activity marker should be removable"); + fs::create_dir_all(&marker_path).expect("run activity marker path directory should write"); + + let inspector = FakePullRequestInspector::new(vec![Ok(sample_pull_request())]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(!response.success); + assert!( + bridge.completion_disposition().is_err(), + "a failed review-policy clear must not leave a pending review completion behind" + ); +} + +fn review_handoff_pr_details( + url: &str, + head_ref_name: &str, + head_ref_oid: &str, + owner: &str, + repository: &str, + base_ref_name: &str, + is_draft: bool, +) -> PullRequestDetails { + PullRequestDetails { + head_ref_name: String::from(head_ref_name), + head_ref_oid: String::from(head_ref_oid), + head_repository_name: String::from(repository), + head_repository_owner: String::from(owner), + is_draft, + state: String::from("OPEN"), + base_ref_name: String::from(base_ref_name), + url: String::from(url), + } +} + +#[test] +fn rejects_invalid_pull_requests_for_review_handoff() { + for (case_name, pull_request, expected_error) in [ + ( + "another branch", + review_handoff_pr_details( + "https://github.com/hack-ink/decodex/pull/43", + "x/decodex-pub-999", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "hack-ink", + "decodex", + "main", + false, + ), + None, + ), + ( + "draft pull request", + review_handoff_pr_details( + "https://github.com/hack-ink/decodex/pull/44", + "x/decodex-pub-618", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "hack-ink", + "decodex", + "main", + true, + ), + None, + ), + ( + "stale PR head", + review_handoff_pr_details( + "https://github.com/hack-ink/decodex/pull/45", + "x/decodex-pub-618", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "hack-ink", + "decodex", + "main", + false, + ), + None, + ), + ( + "another repository", + review_handoff_pr_details( + "https://github.com/someone-else/decodex-fork/pull/46", + "x/decodex-pub-618", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "someone-else", + "decodex-fork", + "main", + false, + ), + None, + ), + ( + "non-default target branch", + review_handoff_pr_details( + "https://github.com/hack-ink/decodex/pull/47", + "x/decodex-pub-618", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "hack-ink", + "decodex", + "release/1.x", + false, + ), + Some("retained review lanes must target the repository default branch `main`"), + ), + ] { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let pr_url = pull_request.url.clone(); + let inspector = FakePullRequestInspector::new(vec![Ok(pull_request)]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Ready for review." + }), + ); + + assert!(!response.success, "{case_name}"); + assert!(tracker.comments.borrow().is_empty(), "{case_name}"); + assert!(tracker.state_updates.borrow().is_empty(), "{case_name}"); + + if let Some(expected_error) = expected_error { + assert!( + matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] if text.contains(expected_error) + ), + "{case_name}" + ); + } + + assert!(bridge.apply_review_handoff().is_err(), "{case_name}"); + } +} + +#[test] +fn parses_credentialed_https_github_remote() { + let repository = super::parse_github_repository_identity( + "https://x-access-token@github.com/hack-ink/decodex.git", + ) + .expect("credentialed GitHub remote should parse"); + + assert_eq!( + repository, + super::RepositoryIdentity { + owner: String::from("hack-ink"), + name: String::from("decodex"), + } + ); +} + +#[test] +fn parses_default_branch_from_ls_remote_symref_output() { + let parsed = super::parse_remote_head_symref_output( + "ref: refs/heads/main\tHEAD\n9c0ffee\tHEAD\n9c0ffee\trefs/heads/main\n", + ); + + assert_eq!(parsed.as_deref(), Some("main")); +} + +#[test] +fn ignores_non_head_lines_when_parsing_default_branch_from_ls_remote_output() { + let parsed = super::parse_remote_head_symref_output( + "9c0ffee\trefs/heads/main\n9c0ffee\trefs/heads/release/1.x\n", + ); + + assert_eq!(parsed, None); +} + +#[test] +fn resolve_lane_default_branch_prefers_cached_origin_head() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let remote_root = temp_dir.path().join("origin.git"); + let repo_root = temp_dir.path().join("repo"); + + run_git_for_handoff( + temp_dir.path(), + &[ + "init", + "--bare", + "--initial-branch", + "main", + remote_root.to_str().expect("remote path utf-8"), + ], + ); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + + run_git_for_handoff(&repo_root, &["init", "--initial-branch", "main"]); + run_git_for_handoff(&repo_root, &["config", "user.name", "Decodex Tests"]); + run_git_for_handoff(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git_for_handoff( + &repo_root, + &["remote", "add", "origin", remote_root.to_str().expect("remote path utf-8")], + ); + + fs::write(repo_root.join("README.md"), "seed\n").expect("seed file should write"); + + run_git_for_handoff(&repo_root, &["add", "README.md"]); + run_git_for_handoff(&repo_root, &["commit", "-m", "seed"]); + run_git_for_handoff(&repo_root, &["push", "-u", "origin", "main"]); + run_git_for_handoff(&repo_root, &["checkout", "-b", "trunk"]); + run_git_for_handoff(&repo_root, &["push", "origin", "trunk"]); + run_git_for_handoff(&remote_root, &["symbolic-ref", "HEAD", "refs/heads/trunk"]); + run_git_for_handoff( + &repo_root, + &["symbolic-ref", "refs/remotes/origin/HEAD", "refs/remotes/origin/main"], + ); + run_git_for_handoff(&repo_root, &["checkout", "main"]); + + let resolved = + super::resolve_lane_default_branch(&repo_root).expect("default branch should resolve"); + + assert_eq!(resolved, "main"); +} + +#[test] +fn resolve_lane_default_branch_uses_remote_head_when_local_cache_is_missing() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let remote_root = temp_dir.path().join("origin.git"); + let repo_root = temp_dir.path().join("repo"); + + run_git_for_handoff( + temp_dir.path(), + &[ + "init", + "--bare", + "--initial-branch", + "main", + remote_root.to_str().expect("remote path utf-8"), + ], + ); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + + run_git_for_handoff(&repo_root, &["init", "--initial-branch", "main"]); + run_git_for_handoff(&repo_root, &["config", "user.name", "Decodex Tests"]); + run_git_for_handoff(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git_for_handoff( + &repo_root, + &["remote", "add", "origin", remote_root.to_str().expect("remote path utf-8")], + ); + + fs::write(repo_root.join("README.md"), "seed\n").expect("seed file should write"); + + run_git_for_handoff(&repo_root, &["add", "README.md"]); + run_git_for_handoff(&repo_root, &["commit", "-m", "seed"]); + run_git_for_handoff(&repo_root, &["push", "-u", "origin", "main"]); + run_git_for_handoff(&repo_root, &["checkout", "-b", "trunk"]); + run_git_for_handoff(&repo_root, &["push", "origin", "trunk"]); + run_git_for_handoff(&remote_root, &["symbolic-ref", "HEAD", "refs/heads/trunk"]); + run_git_for_handoff(&repo_root, &["checkout", "main"]); + + let resolved = + super::resolve_lane_default_branch(&repo_root).expect("default branch should resolve"); + + assert_eq!(resolved, "trunk"); +} + +#[test] +fn resolve_lane_default_branch_uses_cached_origin_head_without_reachable_remote() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let remote_root = temp_dir.path().join("origin.git"); + let repo_root = temp_dir.path().join("repo"); + + run_git_for_handoff( + temp_dir.path(), + &[ + "init", + "--bare", + "--initial-branch", + "main", + remote_root.to_str().expect("remote path utf-8"), + ], + ); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + + run_git_for_handoff(&repo_root, &["init", "--initial-branch", "main"]); + run_git_for_handoff(&repo_root, &["config", "user.name", "Decodex Tests"]); + run_git_for_handoff(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git_for_handoff( + &repo_root, + &["remote", "add", "origin", remote_root.to_str().expect("remote path utf-8")], + ); + + fs::write(repo_root.join("README.md"), "seed\n").expect("seed file should write"); + + run_git_for_handoff(&repo_root, &["add", "README.md"]); + run_git_for_handoff(&repo_root, &["commit", "-m", "seed"]); + run_git_for_handoff(&repo_root, &["push", "-u", "origin", "main"]); + run_git_for_handoff( + &repo_root, + &["symbolic-ref", "refs/remotes/origin/HEAD", "refs/remotes/origin/main"], + ); + run_git_for_handoff( + &repo_root, + &[ + "remote", + "set-url", + "origin", + temp_dir.path().join("missing-origin.git").to_str().expect("missing remote path utf-8"), + ], + ); + + let resolved = + super::resolve_lane_default_branch(&repo_root).expect("default branch should resolve"); + + assert_eq!(resolved, "main"); +} + +fn run_git_for_handoff(cwd: &Path, args: &[&str]) { + let status = Command::new("git") + .args(["-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"]) + .arg("-C") + .arg(cwd) + .args(args) + .status() + .expect("git should run"); + + assert!(status.success(), "git {:?} should succeed in `{}`", args, cwd.display()); +} + +#[test] +fn publishes_protocol_safe_tool_names() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context(), + &inspector, + &local_repo_inspector, + ); + let tool_specs = DynamicToolHandler::tool_specs(&bridge); + + assert!(!tool_specs.is_empty()); + assert!(tool_specs.into_iter().all(|tool| { + tool.name.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + })); +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tests/review/policy.rs b/apps/decodex/src/agent/tracker_tool_bridge/tests/review/policy.rs new file mode 100644 index 00000000..750bb9d3 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tests/review/policy.rs @@ -0,0 +1,1258 @@ +fn sample_review_repair_apply_inspectors( + pr_url: &str, +) -> (FakePullRequestInspector, FakeLocalRepoInspector) { + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from(pr_url), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from(pr_url), + }), + ]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![ + Ok(LocalRepoDetails { + default_branch: String::from("main"), + head_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + repository_name: String::from("decodex"), + repository_owner: String::from("hack-ink"), + }), + Ok(LocalRepoDetails { + default_branch: String::from("main"), + head_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + repository_name: String::from("decodex"), + repository_owner: String::from("hack-ink"), + }), + ]); + + (inspector, local_repo_inspector) +} + +fn seed_review_repair_apply_state( + state_store: &StateStore, + review_context: &ReviewHandoffContext, + issue_id: &str, + pr_url: &str, + external_round_count: i64, +) { + let review_handoff = ReviewHandoffMarker::new( + String::from("pub-618-attempt-2-100"), + 2, + review_context.branch_name.clone(), + String::from(pr_url), + String::from("main"), + review_context.branch_name.clone(), + String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + ); + + state_store + .upsert_review_handoff_marker(&review_context.service_id, issue_id, &review_handoff) + .expect("original review handoff marker should persist"); + + write_review_policy_checkpoint( + review_context, + "repair", + "clean", + "18a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 0, + ); + + state_store + .upsert_review_orchestration_marker( + &review_context.service_id, + issue_id, + &ReviewOrchestrationMarker::new( + review_handoff.run_id().to_owned(), + review_handoff.attempt_number(), + review_handoff.branch_name().to_owned(), + pr_url.to_owned(), + String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + "repair_required", + Some(91), + Some(1_763_600_000), + Some(0), + 0, + external_round_count, + None, + ), + ) + .expect("review orchestration marker should persist"); +} + +#[test] +fn records_review_handoff_and_applies_it_after_validation() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/42"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/42"), + }), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + + write_clean_review_checkpoint(&review_context); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/42", + "summary": "Implemented the PR-backed review handoff." + }), + ); + + assert!(response.success); + + bridge.apply_review_handoff().expect("review handoff should apply"); + + assert_eq!(tracker.state_updates.borrow().as_slice(), ["state-review"]); + + let comments = tracker.comments.borrow(); + + assert_eq!(comments.len(), 1); + assert!(comments[0].contains("- pr_url: `https://github.com/hack-ink/decodex/pull/42`")); + assert!(comments[0].contains("- validation_result: `passed`")); + assert!(comments[0].contains("- worktree_path: `.worktrees/PUB-618`")); +} + +#[test] +fn review_handoff_apply_persists_runtime_handoff_marker() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![ + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/142"), + }), + Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from("https://github.com/hack-ink/decodex/pull/142"), + }), + ]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let review_context = sample_review_context_in(temp_dir.path()); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + + write_clean_review_checkpoint(&review_context); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/142", + "summary": "Ready for review." + }), + ); + + assert!(response.success); + + bridge.apply_review_handoff().expect("review handoff should apply"); + + let marker = persisted_review_handoff_marker(&bridge, &issue, &review_context); + + assert_eq!(marker.branch_name(), review_context.branch_name); + assert_eq!(marker.pr_url(), "https://github.com/hack-ink/decodex/pull/142"); + assert_eq!(marker.pr_head_oid(), "08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); +} + +#[test] +fn review_repair_tool_surface_excludes_issue_transition() { + let tracker = FakeTracker::new(); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let temp_dir = TempDir::new().expect("tempdir should create"); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + sample_review_repair_context_in(temp_dir.path(), pr_url), + &inspector, + &local_repo_inspector, + ); + let tool_names = DynamicToolHandler::tool_specs(&bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + + assert!(!tool_names.contains(&String::from(ISSUE_TRANSITION_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_COMMENT_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_LABEL_ADD_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_TERMINAL_FINALIZE_TOOL_NAME))); +} + +#[test] +fn review_checkpoint_tool_surface_excludes_closeout() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let review_issue = sample_review_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let handoff_pr_inspector = FakePullRequestInspector::new(Vec::new()); + let handoff_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let handoff_bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &handoff_pr_inspector, + &handoff_repo_inspector, + ); + let repair_pr_inspector = FakePullRequestInspector::new(Vec::new()); + let repair_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let repair_bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &review_issue, + &workflow, + sample_review_repair_context_in( + temp_dir.path(), + "https://github.com/hack-ink/decodex/pull/242", + ), + &repair_pr_inspector, + &repair_repo_inspector, + ); + let closeout_bridge = TrackerToolBridge::with_run_context( + &tracker, + &review_issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), "https://github.com/hack-ink/decodex/pull/260"), + ); + let handoff_tools = DynamicToolHandler::tool_specs(&handoff_bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let repair_tools = DynamicToolHandler::tool_specs(&repair_bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let closeout_tools = DynamicToolHandler::tool_specs(&closeout_bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + + assert!(handoff_tools.contains(&String::from(ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME))); + assert!(repair_tools.contains(&String::from(ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME))); + assert!(closeout_tools.contains(&String::from(ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME))); + assert!(handoff_tools.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(repair_tools.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(!closeout_tools.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); +} + +#[test] +fn review_checkpoint_tool_surface_respects_internal_review_config() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let review_issue = sample_review_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let mut review_context = sample_review_context_in(temp_dir.path()); + let mut repair_context = sample_review_repair_context_in( + temp_dir.path(), + "https://github.com/hack-ink/decodex/pull/242", + ); + + review_context.internal_review_mode = InternalReviewMode::Off; + repair_context.internal_review_mode = InternalReviewMode::Off; + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + let repair_bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &review_issue, + &workflow, + repair_context, + &inspector, + &local_repo_inspector, + ); + let tool_names = DynamicToolHandler::tool_specs(&bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let repair_tool_names = DynamicToolHandler::tool_specs(&repair_bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let checkpoint_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "clean", + "head_sha": "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "evidence": [] + }), + ); + + assert!(!tool_names.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_REVIEW_HANDOFF_TOOL_NAME))); + assert!(!repair_tool_names.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(repair_tool_names.contains(&String::from(ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME))); + assert!(!checkpoint_response.success); + assert!(matches!( + checkpoint_response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains("codex.internal_review_mode = \"off\"") + )); +} + +#[test] +fn prompt_only_internal_review_mode_does_not_expose_checkpoint_tool() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let mut review_context = sample_review_context_in(temp_dir.path()); + + review_context.internal_review_mode = InternalReviewMode::Prompt; + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let tool_names = DynamicToolHandler::tool_specs(&bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let checkpoint_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "clean", + "head_sha": "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "evidence": [] + }), + ); + + assert!(!tool_names.contains(&String::from(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME))); + assert!(tool_names.contains(&String::from(ISSUE_REVIEW_HANDOFF_TOOL_NAME))); + assert!(!checkpoint_response.success); + assert!(matches!( + checkpoint_response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains("codex.internal_review_mode = \"prompt\"") + )); +} + +#[test] +fn review_checkpoint_normalizes_matching_short_head_sha_to_full_head() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let review_context = sample_review_context_in(temp_dir.path()); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "clean", + "head_sha": &sample_local_repo().head_oid[..7], + "evidence": ["Closeout and review policy both point at the current lane head."] + }), + ); + + assert!(response.success); + assert!(tracker.comments.borrow().is_empty()); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.review_policy_head_sha(), Some(sample_local_repo().head_oid.as_str())); +} + +#[test] +fn review_checkpoint_findings_continue_until_budget_then_stop() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let review_context = sample_review_context_in(temp_dir.path()); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![ + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + ]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &pull_request_inspector, + &local_repo_inspector, + ); + + for expected_round in [1_i64, 2_i64] { + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "findings", + "head_sha": sample_local_repo().head_oid, + "evidence": ["owned fix still pending"] + }), + ); + + assert!(response.success); + assert_eq!( + DynamicToolHandler::classify_turn_completion(&bridge, "continue") + .expect("findings below the convergence budget should continue"), + TurnCompletionStatus::Continue + ); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.review_policy_phase(), Some("handoff")); + assert_eq!(marker.review_policy_status(), Some("findings")); + assert_eq!(marker.review_policy_nonclean_rounds(), Some(expected_round)); + } + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "findings", + "head_sha": sample_local_repo().head_oid, + "evidence": ["still not converged"] + }), + ); + + assert!(response.success); + + let error = DynamicToolHandler::classify_turn_completion(&bridge, "stop") + .expect_err("third consecutive findings checkpoint should stop the lane"); + let stop = error + .downcast_ref::() + .expect("stop boundary should expose a typed review policy error"); + + assert_eq!(stop.reason, ReviewPolicyStopReason::Exhausted); + assert_eq!(stop.nonclean_rounds, Some(3)); +} + +#[test] +fn review_checkpoint_clean_resets_nonclean_rounds_before_next_findings() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let review_context = sample_review_context_in(temp_dir.path()); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![ + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + Ok(sample_local_repo()), + ]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + + for status in ["findings", "findings", "clean", "findings"] { + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": status, + "head_sha": sample_local_repo().head_oid, + "evidence": ["review evidence"] + }), + ); + + assert!(response.success); + } + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.review_policy_status(), Some("findings")); + assert_eq!(marker.review_policy_nonclean_rounds(), Some(1)); + assert_eq!( + DynamicToolHandler::classify_turn_completion(&bridge, "continue") + .expect("findings after a clean checkpoint should continue"), + TurnCompletionStatus::Continue + ); +} + +#[test] +fn review_checkpoint_does_not_depend_on_tracker_comment_write() { + let tracker = FakeTracker::with_comment_error("tracker write failed"); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let review_context = sample_review_context_in(temp_dir.path()); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(sample_local_repo()), Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "findings", + "head_sha": sample_local_repo().head_oid, + "evidence": ["tracker write failed before checkpoint persisted"] + }), + ); + + assert!(response.success); + assert!(tracker.comments.borrow().is_empty()); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.review_policy_nonclean_rounds(), Some(1)); +} + +#[test] +fn review_checkpoint_architecture_and_blocked_statuses_stop_immediately() { + for (status, expected_reason) in [ + ("needs_architecture_review", ReviewPolicyStopReason::ArchitectureReviewRequired), + ("blocked", ReviewPolicyStopReason::Blocked), + ] { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": status, + "head_sha": sample_local_repo().head_oid, + "evidence": ["requires human follow-up"] + }), + ); + + assert!(response.success); + + let error = DynamicToolHandler::classify_turn_completion(&bridge, "stop") + .expect_err("stop statuses should fail immediately"); + let stop = error + .downcast_ref::() + .expect("stop boundary should expose a typed review policy error"); + + assert_eq!(stop.reason, expected_reason); + } +} + +#[test] +fn review_checkpoint_phase_switch_resets_nonclean_rounds() { + let tracker = FakeTracker::new(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let repair_context = sample_review_repair_context_in( + temp_dir.path(), + "https://github.com/hack-ink/decodex/pull/242", + ); + let issue = sample_review_issue(); + + write_review_policy_checkpoint( + &ReviewHandoffContext { mode: ReviewExecutionMode::Handoff, ..repair_context.clone() }, + "handoff", + "findings", + &sample_local_repo().head_oid, + 2, + ); + + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + repair_context, + &pull_request_inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + serde_json::json!({ + "status": "findings", + "head_sha": sample_local_repo().head_oid, + "evidence": [] + }), + ); + + assert!(response.success); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.review_policy_phase(), Some("repair")); + assert_eq!(marker.review_policy_nonclean_rounds(), Some(1)); +} + +#[test] +fn stale_review_checkpoint_for_old_head_does_not_stop_new_head() { + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let temp_dir = TempDir::new().expect("tempdir should create"); + let review_context = sample_review_context_in(temp_dir.path()); + let mut updated_local_repo = sample_local_repo(); + + updated_local_repo.head_oid = String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + + write_review_policy_checkpoint( + &review_context, + "handoff", + "blocked", + &sample_local_repo().head_oid, + 0, + ); + + let pull_request_inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(updated_local_repo)]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &pull_request_inspector, + &local_repo_inspector, + ); + + assert_eq!( + DynamicToolHandler::classify_turn_completion(&bridge, "continue") + .expect("a stale checkpoint from an older head should be ignored"), + TurnCompletionStatus::Continue + ); +} + +#[test] +fn review_handoff_requires_a_clean_checkpoint() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(vec![Ok(sample_pull_request())]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + sample_review_context_in(temp_dir.path()), + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(!response.success); + assert!(matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains("requires a current `handoff` review checkpoint with status `clean`") + )); +} + +#[test] +fn review_completion_skips_clean_checkpoint_when_internal_review_disabled() { + for completion_path in ["handoff", "repair"] { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let workflow = sample_workflow(); + + if completion_path == "handoff" { + let issue = sample_issue(); + let inspector = FakePullRequestInspector::new(vec![Ok(sample_pull_request())]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(sample_local_repo())]); + let mut review_context = sample_review_context_in(temp_dir.path()); + + review_context.internal_review_mode = InternalReviewMode::Off; + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/48", + "summary": "Ready for review." + }), + ); + + assert!(response.success, "{completion_path} should not require a clean checkpoint"); + } else { + let review_issue = sample_review_issue(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let (repair_inspector, repair_local_repo_inspector) = + sample_review_repair_apply_inspectors(pr_url); + let mut review_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + + review_context.internal_review_mode = InternalReviewMode::Off; + + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &review_issue, + &workflow, + review_context, + &repair_inspector, + &repair_local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Addressed the requested review changes." + }), + ); + + assert!(response.success, "{completion_path} should not require a clean checkpoint"); + } + } +} + +#[test] +fn disabled_internal_review_ignores_stale_review_policy_stop_state() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let inspector = FakePullRequestInspector::new(Vec::new()); + let local_repo_inspector = FakeLocalRepoInspector::new(Vec::new()); + let mut review_context = sample_review_context_in(temp_dir.path()); + + review_context.internal_review_mode = InternalReviewMode::Off; + + write_review_policy_checkpoint( + &review_context, + "handoff", + "findings", + &sample_local_repo().head_oid, + 3, + ); + + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let completion_status = DynamicToolHandler::classify_turn_completion(&bridge, "done") + .expect("disabled internal review should ignore stale review stop state"); + + assert_eq!(completion_status, TurnCompletionStatus::Continue); +} + +#[test] +fn review_handoff_rejects_stale_clean_checkpoint_for_previous_head() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_issue(); + let workflow = sample_workflow(); + let mut updated_local_repo = sample_local_repo(); + let mut updated_pull_request = sample_pull_request(); + + updated_local_repo.head_oid = String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + updated_pull_request.head_ref_oid = updated_local_repo.head_oid.clone(); + updated_pull_request.url = String::from("https://github.com/hack-ink/decodex/pull/149"); + + let review_context = sample_review_context_in(temp_dir.path()); + + write_review_policy_checkpoint( + &review_context, + "handoff", + "clean", + &sample_local_repo().head_oid, + 0, + ); + + let inspector = FakePullRequestInspector::new(vec![Ok(updated_pull_request)]); + let local_repo_inspector = + FakeLocalRepoInspector::new(vec![Ok(updated_local_repo.clone()), Ok(updated_local_repo)]); + let bridge = TrackerToolBridge::with_review_handoff_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + serde_json::json!({ + "pr_url": "https://github.com/hack-ink/decodex/pull/149", + "summary": "Ready for review." + }), + ); + + assert!(!response.success); + assert!(matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains("requires a current `handoff` review checkpoint with status `clean` for the current lane HEAD") + )); +} + +#[test] +fn review_repair_complete_requires_a_clean_checkpoint() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let inspector = FakePullRequestInspector::new(vec![Ok(PullRequestDetails { + head_ref_name: String::from("x/decodex-pub-618"), + head_ref_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + head_repository_name: String::from("decodex"), + head_repository_owner: String::from("hack-ink"), + is_draft: false, + state: String::from("OPEN"), + base_ref_name: String::from("main"), + url: String::from(pr_url), + })]); + let local_repo_inspector = FakeLocalRepoInspector::new(vec![Ok(LocalRepoDetails { + default_branch: String::from("main"), + head_oid: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + repository_name: String::from("decodex"), + repository_owner: String::from("hack-ink"), + })]); + let review_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + + bridge_state_store(&bridge) + .upsert_review_handoff_marker( + TEST_SERVICE_ID, + &issue.id, + &ReviewHandoffMarker::new( + String::from("pub-618-attempt-2-100"), + 2, + review_context.branch_name.clone(), + String::from(pr_url), + String::from("main"), + review_context.branch_name.clone(), + String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + ), + ) + .expect("original review handoff marker should persist"); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Ready for fresh review." + }), + ); + + assert!(!response.success); + assert!(matches!( + response.content_items.as_slice(), + [DynamicToolContentItem::InputText{ text }] + if text.contains("requires a current `repair` review checkpoint with status `clean`") + )); +} + +#[test] +fn closeout_tool_surface_includes_issue_transition_for_completed_state() { + let mut issue = sample_review_issue(); + + issue + .team + .states + .push(TrackerState { id: String::from("state-done"), name: String::from("Done") }); + + let tracker = tracker_with_current_issue_snapshot(&issue); + let workflow = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Use the tracker tools. +"#, + ) + .expect("workflow should parse"); + let pr_url = "https://github.com/hack-ink/decodex/pull/260"; + let temp_dir = TempDir::new().expect("tempdir should create"); + let bridge = TrackerToolBridge::with_run_context( + &tracker, + &issue, + &workflow, + sample_closeout_context_in(temp_dir.path(), pr_url), + ); + let tool_names = DynamicToolHandler::tool_specs(&bridge) + .into_iter() + .map(|tool| tool.name) + .collect::>(); + let transition_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "Done" }), + ); + let invalid_transition_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "In Progress" }), + ); + + assert!(tool_names.contains(&String::from(ISSUE_TRANSITION_TOOL_NAME))); + assert!(transition_response.success); + assert!(!invalid_transition_response.success); + assert_eq!(tracker.state_updates.borrow().as_slice(), [String::from("state-done")]); +} + +#[test] +fn review_repair_apply_persists_updated_handoff_marker_without_tracker_transition() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let (inspector, local_repo_inspector) = sample_review_repair_apply_inspectors(pr_url); + let review_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + + seed_review_repair_apply_state( + bridge_state_store(&bridge), + &review_context, + &issue.id, + pr_url, + 2, + ); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Addressed the requested review changes." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "review_repair" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect("review repair completion should allow the turn to complete"); + + bridge.apply_review_repair().expect("review repair should apply"); + + assert!(tracker.state_updates.borrow().is_empty()); + + let comments = tracker.comments.borrow(); + + assert_eq!(comments.len(), 1); + assert!(comments[0].contains("fresh review")); + assert!(comments[0].contains("- pr_url: `https://github.com/hack-ink/decodex/pull/242`")); + + let marker = persisted_review_handoff_marker(&bridge, &issue, &review_context); + + assert_eq!(marker.pr_url(), pr_url); + assert_eq!(marker.pr_head_oid(), "18a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + + let orchestration_marker = + persisted_review_orchestration_marker(&bridge, &issue, &review_context, &marker); + + assert_eq!(orchestration_marker.phase(), "request_pending"); + assert_eq!(orchestration_marker.head_sha(), "18a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + assert_eq!(orchestration_marker.external_round_count(), 2); +} + +#[test] +fn review_repair_apply_resets_external_round_budget_after_fourth_round() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::new(); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let (inspector, local_repo_inspector) = sample_review_repair_apply_inspectors(pr_url); + let review_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + review_context.clone(), + &inspector, + &local_repo_inspector, + ); + + seed_review_repair_apply_state( + bridge_state_store(&bridge), + &review_context, + &issue.id, + pr_url, + 4, + ); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Addressed the requested review changes." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "review_repair" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + DynamicToolHandler::validate_turn_completion(&bridge, "done") + .expect("review repair completion should allow the turn to complete"); + + bridge.apply_review_repair().expect("review repair should apply"); + + let marker = persisted_review_handoff_marker(&bridge, &issue, &review_context); + let orchestration_marker = + persisted_review_orchestration_marker(&bridge, &issue, &review_context, &marker); + + assert_eq!(orchestration_marker.phase(), "request_pending"); + assert_eq!(orchestration_marker.external_round_count(), 0); +} + +#[test] +fn review_repair_apply_preserves_existing_markers_when_comment_write_fails() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let tracker = FakeTracker::with_comment_error("tracker comment write failed"); + let issue = sample_review_issue(); + let workflow = sample_workflow(); + let pr_url = "https://github.com/hack-ink/decodex/pull/242"; + let (inspector, local_repo_inspector) = sample_review_repair_apply_inspectors(pr_url); + let review_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + let bridge = TrackerToolBridge::with_review_repair_for_test( + &tracker, + &issue, + &workflow, + review_context, + &inspector, + &local_repo_inspector, + ); + let seed_context = sample_review_repair_context_in(temp_dir.path(), pr_url); + + seed_review_repair_apply_state( + bridge_state_store(&bridge), + &seed_context, + &issue.id, + pr_url, + 2, + ); + + let response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + serde_json::json!({ + "pr_url": pr_url, + "summary": "Addressed the requested review changes." + }), + ); + let finalize_response = DynamicToolHandler::handle_call( + &bridge, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + serde_json::json!({ "path": "review_repair" }), + ); + + assert!(response.success); + assert!(finalize_response.success); + + let error = bridge + .apply_review_repair() + .expect_err("comment write failures must preserve the original handoff marker"); + + assert!(error.to_string().contains("tracker comment write failed")); + assert!(tracker.comments.borrow().is_empty()); + + let marker = persisted_review_handoff_marker(&bridge, &issue, &seed_context); + + assert_eq!(marker.pr_url(), pr_url); + assert_eq!(marker.pr_head_oid(), "08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + + let orchestration_marker = + persisted_review_orchestration_marker(&bridge, &issue, &seed_context, &marker); + + assert_eq!(orchestration_marker.phase(), "repair_required"); + assert_eq!(orchestration_marker.head_sha(), "08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + assert_eq!(orchestration_marker.external_round_count(), 2); +} diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tools.rs b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs new file mode 100644 index 00000000..0c90d2c1 --- /dev/null +++ b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs @@ -0,0 +1,1064 @@ +use serde_json::{self, Value}; + +use crate::{ + agent::tracker_tool_bridge::{ + self, CommentArgs, DynamicToolCallResponse, DynamicToolSpec, ExecutionProgressPhase, + ISSUE_COMMENT_TOOL_NAME, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + ISSUE_TRANSITION_TOOL_NAME, LabelArgs, PendingReviewAction, PendingReviewCompletion, + ProgressCheckpointArgs, ReviewCheckpointArgs, ReviewExecutionMode, ReviewHandoffArgs, + ReviewHandoffContext, ReviewPolicyPhase, ReviewPolicyStatus, RunCompletionDisposition, + TerminalFinalizeArgs, TrackerToolBridge, TransitionArgs, + }, + state, + tracker::{ + self, records, + records::{LinearExecutionEventIdentity, LinearExecutionEventRecord}, + }, +}; + +impl<'a> TrackerToolBridge<'a> { + pub(super) fn build_tool_specs(&self) -> Vec { + let mut tool_specs = match self.review_context.as_ref().map(|context| context.mode) { + Some(ReviewExecutionMode::Repair) => { + let mut tool_specs = self.comment_tool_specs(); + + tool_specs.extend(self.progress_checkpoint_tool_specs()); + + if self + .review_context + .as_ref() + .is_some_and(ReviewHandoffContext::internal_review_checkpoint_enabled) + { + tool_specs.extend(self.review_checkpoint_tool_specs()); + } + + tool_specs + }, + Some(ReviewExecutionMode::Closeout) => self.closeout_base_tool_specs(), + Some(ReviewExecutionMode::Handoff) => { + let mut tool_specs = self.base_tool_specs(); + + if self + .review_context + .as_ref() + .is_some_and(ReviewHandoffContext::internal_review_checkpoint_enabled) + { + tool_specs.extend(self.review_checkpoint_tool_specs()); + } + + tool_specs.extend(self.review_handoff_tool_specs()); + + tool_specs + }, + None => self.base_tool_specs(), + }; + + if matches!( + self.review_context.as_ref().map(|context| context.mode), + Some(ReviewExecutionMode::Repair) + ) { + tool_specs.extend(self.review_repair_tool_specs()); + } + if matches!( + self.review_context.as_ref().map(|context| context.mode), + Some(ReviewExecutionMode::Closeout) + ) { + tool_specs.extend(self.closeout_tool_specs()); + } + + tool_specs.push(self.label_add_tool_spec()); + + tool_specs + } + + pub(super) fn base_tool_specs(&self) -> Vec { + let mut tool_specs = vec![self.transition_tool_spec()]; + + tool_specs.extend(self.comment_tool_specs()); + tool_specs.extend(self.progress_checkpoint_tool_specs()); + + tool_specs + } + + pub(super) fn closeout_base_tool_specs(&self) -> Vec { + let mut tool_specs = vec![self.transition_tool_spec()]; + + tool_specs.extend(self.comment_tool_specs()); + tool_specs.extend(self.progress_checkpoint_tool_specs()); + + tool_specs + } + + pub(super) fn comment_tool_specs(&self) -> Vec { + vec![DynamicToolSpec::new( + ISSUE_COMMENT_TOOL_NAME, + "Add a comment to the currently leased issue.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["body"], + "additionalProperties": false + }), + )] + } + + pub(super) fn progress_checkpoint_tool_specs(&self) -> [DynamicToolSpec; 1] { + [DynamicToolSpec::new( + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + "Record the current execution-state snapshot for the leased issue as a durable Linear-backed progress checkpoint without changing lifecycle authority. On retained lanes, omit `head_sha` to capture the exact current lane HEAD automatically, or pass a matching current-lane HEAD SHA.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "phase": { + "type": "string", + "enum": [ + "probing", + "implementing", + "verifying", + "blocked", + "ready_for_review", + "review_repair", + "ready_to_land", + "closeout" + ] + }, + "focus": { "type": "string" }, + "next_action": { "type": "string" }, + "blockers": { + "type": "array", + "items": { "type": "string" } + }, + "evidence": { + "type": "array", + "items": { "type": "string" } + }, + "verification": { + "type": "array", + "items": { "type": "string" } + }, + "head_sha": { "type": "string" }, + "branch": { "type": "string" }, + "pr_url": { "type": "string" } + }, + "required": ["phase", "focus", "next_action", "blockers", "evidence"], + "additionalProperties": false + }), + )] + } + + pub(super) fn transition_tool_spec(&self) -> DynamicToolSpec { + DynamicToolSpec::new( + ISSUE_TRANSITION_TOOL_NAME, + "Move the currently leased issue to another allowed workflow state.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "state": { "type": "string" } + }, + "required": ["state"], + "additionalProperties": false + }), + ) + } + + pub(super) fn review_handoff_tool_specs(&self) -> [DynamicToolSpec; 2] { + [ + DynamicToolSpec::new( + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + "Record a PR-backed review handoff for the currently leased issue after the branch is pushed and a non-draft PR is ready for review.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "pr_url": { "type": "string" }, + "summary": { "type": "string" } + }, + "required": ["pr_url", "summary"], + "additionalProperties": false + }), + ), + DynamicToolSpec::new( + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + "Finalize the current run's terminal tracker path after either PR-backed review handoff or the manual-attention exit has been fully recorded.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "path": { + "type": "string", + "enum": ["review_handoff", "manual_attention"] + } + }, + "required": ["path"], + "additionalProperties": false + }), + ), + ] + } + + pub(super) fn review_checkpoint_tool_specs(&self) -> [DynamicToolSpec; 1] { + [DynamicToolSpec::new( + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + "Record the current repo-native bounded-review result for the leased issue so Decodex can decide whether the lane may continue or must stop for human intervention. `head_sha` must resolve to the current lane HEAD.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "status": { + "type": "string", + "enum": ["clean", "findings", "needs_architecture_review", "blocked"] + }, + "head_sha": { "type": "string" }, + "evidence": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["status", "head_sha", "evidence"], + "additionalProperties": false + }), + )] + } + + pub(super) fn review_repair_tool_specs(&self) -> [DynamicToolSpec; 2] { + [ + DynamicToolSpec::new( + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + "Record that the retained in-review lane repaired the current PR head, pushed it, and requested fresh review on the same PR lineage.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "pr_url": { "type": "string" }, + "summary": { "type": "string" } + }, + "required": ["pr_url", "summary"], + "additionalProperties": false + }), + ), + DynamicToolSpec::new( + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + "Finalize the current run's terminal tracker path after either retained review repair or the manual-attention exit has been fully recorded.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "path": { + "type": "string", + "enum": ["review_repair", "manual_attention"] + } + }, + "required": ["path"], + "additionalProperties": false + }), + ), + ] + } + + pub(super) fn closeout_tool_specs(&self) -> [DynamicToolSpec; 2] { + [ + DynamicToolSpec::new( + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + "Record that the retained post-review lane finished merge plus closeout for the same owned PR lineage.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "pr_url": { "type": "string" }, + "summary": { "type": "string" } + }, + "required": ["pr_url", "summary"], + "additionalProperties": false + }), + ), + DynamicToolSpec::new( + ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + "Finalize the current run's terminal tracker path after either post-review closeout or the manual-attention exit has been fully recorded.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "path": { + "type": "string", + "enum": ["closeout", "manual_attention"] + } + }, + "required": ["path"], + "additionalProperties": false + }), + ), + ] + } + + pub(super) fn label_add_tool_spec(&self) -> DynamicToolSpec { + DynamicToolSpec::new( + ISSUE_LABEL_ADD_TOOL_NAME, + "Add an allowed workflow label to the currently leased issue.", + serde_json::json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "issue_identifier": { "type": "string" }, + "label": { "type": "string" } + }, + "required": ["label"], + "additionalProperties": false + }), + ) + } + + pub(super) fn handle_call_inner( + &self, + tool_name: &str, + arguments: Value, + ) -> DynamicToolCallResponse { + match tool_name { + ISSUE_TRANSITION_TOOL_NAME => self.handle_transition(arguments), + ISSUE_COMMENT_TOOL_NAME => self.handle_comment(arguments), + ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME => self.handle_progress_checkpoint(arguments), + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME => self.handle_review_checkpoint(arguments), + ISSUE_REVIEW_HANDOFF_TOOL_NAME => self.handle_review_handoff(arguments), + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME => self.handle_review_repair_complete(arguments), + ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME => self.handle_closeout_complete(arguments), + ISSUE_LABEL_ADD_TOOL_NAME => self.handle_add_label(arguments), + ISSUE_TERMINAL_FINALIZE_TOOL_NAME => self.handle_terminal_finalize(arguments), + _ => + DynamicToolCallResponse::failure(format!("Unsupported tracker tool `{tool_name}`.")), + } + } + + pub(super) fn handle_progress_checkpoint(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.progress_checkpoint` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let phase = match ExecutionProgressPhase::parse(&parsed.phase) { + Ok(phase) => phase, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + let focus = tracker_tool_bridge::normalize_summary(&parsed.focus); + let next_action = tracker_tool_bridge::normalize_summary(&parsed.next_action); + let blockers = tracker_tool_bridge::normalize_progress_list(parsed.blockers); + let evidence = tracker_tool_bridge::normalize_progress_list(parsed.evidence); + let verification = tracker_tool_bridge::normalize_progress_list(parsed.verification); + let head_sha = match self.resolve_progress_checkpoint_head_sha(parsed.head_sha) { + Ok(head_sha) => head_sha, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + let branch = tracker_tool_bridge::normalize_optional_progress_field(parsed.branch); + let pr_url = tracker_tool_bridge::normalize_optional_progress_field(parsed.pr_url); + + if focus.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_progress_checkpoint` requires a non-empty `focus`.", + )); + } + if next_action.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_progress_checkpoint` requires a non-empty `next_action`.", + )); + } + if phase == ExecutionProgressPhase::Blocked && blockers.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_progress_checkpoint` phase `blocked` requires at least one blocker.", + )); + } + + let Some(review_context) = self.review_context.as_ref() else { + return DynamicToolCallResponse::failure(String::from( + "`issue_progress_checkpoint` requires an active Decodex run context.", + )); + }; + let branch = branch.or_else(|| Some(review_context.branch_name.clone())); + let anchor = records::stable_event_anchor(&[ + phase.as_str(), + focus.as_str(), + next_action.as_str(), + head_sha.as_deref().unwrap_or_default(), + branch.as_deref().unwrap_or_default(), + pr_url.as_deref().unwrap_or_default(), + &blockers.join("\n"), + &evidence.join("\n"), + &verification.join("\n"), + ]); + let mut checkpoint = LinearExecutionEventRecord::new( + LinearExecutionEventIdentity { + service_id: &review_context.service_id, + issue_id: &self.issue.id, + issue_identifier: &self.issue.identifier, + run_id: &review_context.run_id, + attempt_number: review_context.attempt_number, + }, + "progress_checkpoint", + tracker_tool_bridge::current_timestamp(), + &anchor, + ); + + checkpoint.phase = Some(phase.as_str().to_owned()); + checkpoint.focus = Some(focus); + checkpoint.next_action = Some(next_action); + checkpoint.blockers = Some(blockers); + checkpoint.evidence = Some(evidence); + checkpoint.verification = Some(verification); + checkpoint.commit_sha = head_sha; + checkpoint.branch = branch; + checkpoint.worktree_path = Some(review_context.worktree_path.clone()); + checkpoint.pr_url = pr_url; + + match tracker::create_linear_execution_event_comment( + self.tracker, + &self.issue.id, + "", + &checkpoint, + ) { + Ok(_) => { + if let Some(state_store) = self.state_store + && let Err(error) = state_store.record_linear_execution_event(&checkpoint) + { + return DynamicToolCallResponse::failure(format!( + "Failed to persist the local execution-state checkpoint for issue `{}`: {error}", + self.issue.identifier + )); + } + + DynamicToolCallResponse::success(format!( + "Recorded `{}` execution state for issue `{}`.", + phase.as_str(), + self.issue.identifier + )) + }, + Err(error) => DynamicToolCallResponse::failure(format!( + "Failed to record an execution-state checkpoint for issue `{}`: {error}", + self.issue.identifier + )), + } + } + + pub(super) fn handle_transition(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.transition` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let allowed_states = self.allowed_transition_states(); + + if !allowed_states.iter().any(|state| state == &parsed.state) { + let success_state = self.workflow.frontmatter().tracker().success_state(); + + if parsed.state == success_state { + return DynamicToolCallResponse::failure(format!( + "State `{}` requires `{}` after the branch is pushed and a reviewable PR exists.", + parsed.state, ISSUE_REVIEW_HANDOFF_TOOL_NAME + )); + } + + return DynamicToolCallResponse::failure(format!( + "State `{}` is outside the allowed tracker tool policy.", + parsed.state + )); + } + + let Some(state_id) = self.issue.state_id_for_name(&parsed.state) else { + return DynamicToolCallResponse::failure(format!( + "State `{}` does not exist on issue `{}`.", + parsed.state, self.issue.identifier + )); + }; + + match self.tracker.update_issue_state(&self.issue.id, state_id) { + Ok(()) => { + self.local_issue_state_name.replace(parsed.state.clone()); + self.record_continuation_blocking_transition(&parsed.state); + + DynamicToolCallResponse::success(format!( + "Issue `{}` moved to `{}`.", + self.issue.identifier, parsed.state + )) + }, + Err(error) => DynamicToolCallResponse::failure(format!( + "Failed to move issue `{}` to `{}`: {error}", + self.issue.identifier, parsed.state + )), + } + } + + pub(super) fn handle_comment(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.comment` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + if parsed.body.trim().is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue.comment` requires a non-empty `body`.", + )); + } + + if let Err(error) = tracker_tool_bridge::validate_public_comment_body(&parsed.body) { + return DynamicToolCallResponse::failure(error); + } + + match self.tracker.create_comment(&self.issue.id, &parsed.body) { + Ok(()) => { + if *self.manual_attention_requested.borrow() { + self.manual_attention_comment_recorded.replace(true); + } + + DynamicToolCallResponse::success(format!( + "Comment added to issue `{}`.", + self.issue.identifier + )) + }, + Err(error) => DynamicToolCallResponse::failure(format!( + "Failed to add a comment to issue `{}`: {error}", + self.issue.identifier + )), + } + } + + pub(super) fn handle_review_checkpoint(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.review_checkpoint` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let Some(review_context) = self.review_context.as_ref() else { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_checkpoint` is unavailable for this run.", + )); + }; + + if !review_context.internal_review_checkpoint_enabled() { + return DynamicToolCallResponse::failure(format!( + "`issue_review_checkpoint` is disabled because `codex.internal_review_mode = \"{}\"` for this run.", + review_context.internal_review_mode.as_str() + )); + } + + let Some(review_policy_phase) = ReviewPolicyPhase::for_mode(review_context.mode) else { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_checkpoint` is unavailable for retained closeout runs.", + )); + }; + let review_policy_status = match ReviewPolicyStatus::parse(&parsed.status) { + Ok(status) => status, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + let local_repo = match self.current_local_repo_details(review_context) { + Ok(local_repo) => local_repo, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + let head_sha = match self.canonicalize_current_lane_head_sha( + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + parsed.head_sha.as_str(), + &local_repo.head_oid, + ) { + Ok(head_sha) => head_sha, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + let evidence = parsed + .evidence + .into_iter() + .map(|item| item.trim().to_owned()) + .filter(|item| !item.is_empty()) + .collect::>(); + let nonclean_rounds = match self.review_checkpoint_nonclean_rounds( + review_context, + review_policy_phase, + review_policy_status, + ) { + Ok(nonclean_rounds) => nonclean_rounds, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + + self.cache_review_policy_state_best_effort( + review_context, + review_policy_phase, + review_policy_status, + &head_sha, + nonclean_rounds, + ); + + let message = self.review_checkpoint_success_message( + review_policy_phase, + review_policy_status, + &head_sha, + nonclean_rounds, + evidence.len(), + ); + + DynamicToolCallResponse::success(message) + } + + fn review_checkpoint_nonclean_rounds( + &self, + review_context: &ReviewHandoffContext, + review_policy_phase: ReviewPolicyPhase, + review_policy_status: ReviewPolicyStatus, + ) -> Result { + let previous_state = self + .review_policy_state_for_current_phase(review_context) + .map_err(|error| error.to_string())?; + let phase_changed = previous_state + .as_ref() + .is_some_and(|previous_state| previous_state.phase != review_policy_phase); + let previous_nonclean_rounds = if phase_changed { + 0 + } else { + previous_state.as_ref().map_or(0, |previous_state| previous_state.nonclean_rounds) + }; + + Ok(match review_policy_status { + ReviewPolicyStatus::Findings => previous_nonclean_rounds.saturating_add(1), + ReviewPolicyStatus::Clean + | ReviewPolicyStatus::NeedsArchitectureReview + | ReviewPolicyStatus::Blocked => 0, + }) + } + + fn review_checkpoint_success_message( + &self, + review_policy_phase: ReviewPolicyPhase, + review_policy_status: ReviewPolicyStatus, + head_sha: &str, + nonclean_rounds: i64, + evidence_count: usize, + ) -> String { + let evidence_suffix = if evidence_count == 0 { + String::from("no evidence items recorded") + } else { + format!("{evidence_count} evidence item(s) recorded") + }; + + match review_policy_status { + ReviewPolicyStatus::Clean => format!( + "Recorded a clean `{}` review checkpoint for issue `{}` at HEAD `{head_sha}`; {evidence_suffix}.", + review_policy_phase.as_str(), + self.issue.identifier, + ), + ReviewPolicyStatus::Findings => format!( + "Recorded `{}` review findings for issue `{}` at HEAD `{head_sha}`; consecutive non-clean rounds now `{nonclean_rounds}`; {evidence_suffix}.", + review_policy_phase.as_str(), + self.issue.identifier, + ), + ReviewPolicyStatus::NeedsArchitectureReview => format!( + "Recorded `needs_architecture_review` for issue `{}` at HEAD `{head_sha}`; Decodex will require human architecture review if the turn ends on this checkpoint.", + self.issue.identifier, + ), + ReviewPolicyStatus::Blocked => format!( + "Recorded `blocked` for issue `{}` at HEAD `{head_sha}`; Decodex will require human intervention if the turn ends on this checkpoint.", + self.issue.identifier, + ), + } + } + + fn clear_review_policy_state_after_completion( + &self, + review_context: &ReviewHandoffContext, + tool_name: &str, + ) -> Result<(), String> { + match state::clear_run_review_policy_state(&review_context.cwd) { + Ok(()) => Ok(()), + Err(error) if review_context.internal_review_checkpoint_enabled() => Err(format!( + "Failed to clear review policy state for issue `{}` after recording `{tool_name}`: {error}", + self.issue.identifier + )), + Err(error) => { + tracing::warn!( + ?error, + issue = self.issue.identifier, + run_id = review_context.run_id, + tool_name, + worktree_path = %review_context.cwd.display(), + "Review policy state clear failed while internal review is disabled; continuing." + ); + + Ok(()) + }, + } + } + + pub(super) fn handle_review_handoff(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.review_handoff` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let Some(review_context) = self.review_context.as_ref() else { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_handoff` is unavailable for this run.", + )); + }; + + if review_context.mode != ReviewExecutionMode::Handoff { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_handoff` is unavailable for retained review-repair runs.", + )); + } + + let pr_url = parsed.pr_url.trim(); + + if pr_url.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_handoff` requires a non-empty `pr_url`.", + )); + } + + let summary = tracker_tool_bridge::normalize_summary(&parsed.summary); + + if summary.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_handoff` requires a non-empty `summary`.", + )); + } + + let pull_request = match self.validate_review_action_pr(review_context, pr_url) { + Ok(pull_request) => pull_request, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + + if let Err(error) = self.require_clean_review_checkpoint(review_context) { + return DynamicToolCallResponse::failure(error); + } + if let Err(error) = self.clear_review_policy_state_after_completion( + review_context, + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + ) { + return DynamicToolCallResponse::failure(error); + } + + self.pending_review_completion.borrow_mut().replace(PendingReviewCompletion::Handoff( + PendingReviewAction { pr_url: pull_request.url.clone(), summary }, + )); + + DynamicToolCallResponse::success(format!( + "Recorded review handoff for issue `{}` with PR `{}`. Decodex will apply the completion comment and move the issue to `{}` after service validation passes.", + self.issue.identifier, + pull_request.url, + self.workflow.frontmatter().tracker().success_state() + )) + } + + pub(super) fn handle_review_repair_complete( + &self, + arguments: Value, + ) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.review_repair_complete` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let Some(review_context) = self.review_context.as_ref() else { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_repair_complete` is unavailable for this run.", + )); + }; + + if review_context.mode != ReviewExecutionMode::Repair { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_repair_complete` is unavailable before a retained in-review repair run starts.", + )); + } + + let pr_url = parsed.pr_url.trim(); + + if pr_url.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_repair_complete` requires a non-empty `pr_url`.", + )); + } + + let summary = tracker_tool_bridge::normalize_summary(&parsed.summary); + + if summary.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_review_repair_complete` requires a non-empty `summary`.", + )); + } + + let pull_request = match self.validate_review_action_pr(review_context, pr_url) { + Ok(pull_request) => pull_request, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + + if let Err(error) = self.require_clean_review_checkpoint(review_context) { + return DynamicToolCallResponse::failure(error); + } + if let Err(error) = self.clear_review_policy_state_after_completion( + review_context, + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + ) { + return DynamicToolCallResponse::failure(error); + } + + self.pending_review_completion.borrow_mut().replace(PendingReviewCompletion::Repair( + PendingReviewAction { pr_url: pull_request.url.clone(), summary }, + )); + + DynamicToolCallResponse::success(format!( + "Recorded retained review repair completion for issue `{}` on PR `{}`. Decodex will persist the updated review lineage after service validation passes.", + self.issue.identifier, pull_request.url + )) + } + + pub(super) fn handle_closeout_complete(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.closeout_complete` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let Some(review_context) = self.review_context.as_ref() else { + return DynamicToolCallResponse::failure(String::from( + "`issue_closeout_complete` is unavailable for this run.", + )); + }; + + if review_context.mode != ReviewExecutionMode::Closeout { + return DynamicToolCallResponse::failure(String::from( + "`issue_closeout_complete` is unavailable before a retained post-review closeout run starts.", + )); + } + + let pr_url = parsed.pr_url.trim(); + + if pr_url.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_closeout_complete` requires a non-empty `pr_url`.", + )); + } + + let summary = tracker_tool_bridge::normalize_summary(&parsed.summary); + + if summary.is_empty() { + return DynamicToolCallResponse::failure(String::from( + "`issue_closeout_complete` requires a non-empty `summary`.", + )); + } + + let pull_request = match self.validate_closeout_pr(review_context, pr_url) { + Ok(pull_request) => pull_request, + Err(error) => return DynamicToolCallResponse::failure(error), + }; + + if let Err(error) = self.validate_closeout_issue_completed_state() { + return DynamicToolCallResponse::failure(error); + } + + self.pending_review_completion.borrow_mut().replace(PendingReviewCompletion::Closeout( + PendingReviewAction { pr_url: pull_request.url.clone(), summary }, + )); + + DynamicToolCallResponse::success(format!( + "Recorded retained closeout completion for issue `{}` on merged PR `{}`. Decodex will validate the merged lineage and terminal tracker state before cleaning up the lane.", + self.issue.identifier, pull_request.url + )) + } + + pub(super) fn handle_add_label(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.label.add` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let allowed_labels = [ + self.workflow.frontmatter().tracker().opt_out_label(), + self.workflow.frontmatter().tracker().needs_attention_label(), + ]; + + if !allowed_labels.iter().any(|label| label == &parsed.label) { + return DynamicToolCallResponse::failure(format!( + "Label `{}` is outside the allowed tracker tool policy.", + parsed.label + )); + } + + let current_issue = match self.refreshed_issue_snapshot() { + Ok(Some(issue)) => issue, + Ok(None) => { + return DynamicToolCallResponse::failure(format!( + "Failed to refresh issue `{}` before updating labels: tracker returned no current snapshot.", + self.issue.identifier + )); + }, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Failed to refresh issue `{}` before updating labels: {error}", + self.issue.identifier + )); + }, + }; + let manual_attention_label = + parsed.label == self.workflow.frontmatter().tracker().needs_attention_label(); + let label_added = match tracker::set_issue_label_presence( + self.tracker, + ¤t_issue, + &parsed.label, + true, + ) { + Ok(label_added) => label_added, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Failed to add label `{}` to issue `{}`: {error}", + parsed.label, self.issue.identifier + )); + }, + }; + + if !label_added { + self.record_label_add_local_effects(&parsed.label, manual_attention_label); + + return DynamicToolCallResponse::success(format!( + "Issue `{}` already has label `{}`.", + self.issue.identifier, parsed.label + )); + } + + self.record_label_add_local_effects(&parsed.label, manual_attention_label); + + DynamicToolCallResponse::success(format!( + "Label `{}` added to issue `{}`.", + parsed.label, self.issue.identifier + )) + } + + fn record_label_add_local_effects(&self, label: &str, manual_attention_label: bool) { + if manual_attention_label { + self.manual_attention_requested.replace(true); + } else if label == self.workflow.frontmatter().tracker().opt_out_label() { + self.local_opt_out_requested.replace(true); + self.record_continuation_blocking_write(format!( + "`{ISSUE_LABEL_ADD_TOOL_NAME}` with label `{label}`", + )); + } + } + + pub(super) fn handle_terminal_finalize(&self, arguments: Value) -> DynamicToolCallResponse { + let parsed = match serde_json::from_value::(arguments) { + Ok(parsed) => parsed, + Err(error) => { + return DynamicToolCallResponse::failure(format!( + "Invalid `issue.terminal_finalize` arguments: {error}" + )); + }, + }; + + if let Err(error) = self.ensure_issue_scope(&parsed.scope) { + return DynamicToolCallResponse::failure(error); + } + + let requested_path = match parsed.path.as_str() { + "review_handoff" => RunCompletionDisposition::ReviewHandoff, + "review_repair" => RunCompletionDisposition::ReviewRepair, + "closeout" => RunCompletionDisposition::Closeout, + "manual_attention" => RunCompletionDisposition::ManualAttention, + other => { + return DynamicToolCallResponse::failure(format!( + "`{ISSUE_TERMINAL_FINALIZE_TOOL_NAME}` path must be `review_handoff`, `review_repair`, `closeout`, or `manual_attention`, not `{other}`." + )); + }, + }; + let actual_path = match self.completion_disposition() { + Ok(actual_path) => actual_path, + Err(error) => return DynamicToolCallResponse::failure(error.to_string()), + }; + + if requested_path != actual_path { + return DynamicToolCallResponse::failure(format!( + "`{ISSUE_TERMINAL_FINALIZE_TOOL_NAME}` requested path `{}`, but the recorded terminal path is `{}`.", + requested_path.as_str(), + actual_path.as_str() + )); + } + + self.finalized_completion_path.replace(Some(actual_path)); + + DynamicToolCallResponse::success(format!( + "Finalized terminal path `{}` for issue `{}`. You can only finish the turn after this succeeds.", + actual_path.as_str(), + self.issue.identifier + )) + } +} diff --git a/apps/decodex/src/archive_hygiene.rs b/apps/decodex/src/archive_hygiene.rs new file mode 100644 index 00000000..46e9631f --- /dev/null +++ b/apps/decodex/src/archive_hygiene.rs @@ -0,0 +1,553 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + path::{Path, PathBuf}, +}; + +use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + config::ServiceConfig, + prelude::{Result, eyre}, + runtime, + state::StateStore, + tracker::{self, IssueTracker, TrackerIssue, linear::LinearClient}, + workflow::WorkflowDocument, +}; + +pub(crate) struct ArchiveHygieneRequest { + pub(crate) repo_labels: Vec, + pub(crate) older_than_days: u32, + pub(crate) execute: bool, +} + +#[derive(Debug, Eq, PartialEq)] +struct ArchivePlan { + candidates: Vec, + skipped: Vec, +} + +#[derive(Debug, Eq, PartialEq)] +struct ArchiveCandidate { + id: String, + identifier: String, + title: String, + state: String, + updated_at: String, + repo_labels: Vec, +} + +#[derive(Debug, Eq, PartialEq)] +struct ArchiveSkip { + identifier: String, + reason: String, +} + +pub(crate) fn run(config_path: Option<&Path>, request: &ArchiveHygieneRequest) -> Result<()> { + let state_store = runtime::open_runtime_store()?; + let Some(config_path) = resolve_config_path(config_path, &state_store)? else { + eyre::bail!( + "No Decodex project config found. Pass --config or register one with `decodex project add `." + ); + }; + let config = ServiceConfig::from_path(&config_path)?; + let workflow = WorkflowDocument::from_path(config.workflow_path())?; + let tracker = LinearClient::new(config.tracker().resolve_api_key()?)?; + let repo_labels = normalize_repo_labels(&request.repo_labels)?; + let updated_before = updated_before_timestamp(request.older_than_days)?; + let plan = build_archive_plan(&tracker, &config, &workflow, &repo_labels, &updated_before)?; + + print_archive_plan(&plan, &repo_labels, &updated_before, request.execute); + + if request.execute { + for candidate in &plan.candidates { + tracker.archive_issue(&candidate.id)?; + } + + println!("Archived {} Linear issue(s).", plan.candidates.len()); + } + + Ok(()) +} + +fn resolve_config_path( + explicit_path: Option<&Path>, + state_store: &StateStore, +) -> Result> { + if let Some(path) = explicit_path { + return Ok(Some(path.to_path_buf())); + } + + runtime::registered_config_path_for_cwd(state_store, &env::current_dir()?) +} + +fn normalize_repo_labels(repo_labels: &[String]) -> Result> { + let mut labels = BTreeSet::new(); + + for label in repo_labels { + if label.trim() != label || label.is_empty() { + eyre::bail!( + "`--repo-label` values must be non-empty labels without surrounding whitespace." + ); + } + if !label.starts_with("repo:") { + eyre::bail!("`--repo-label` must name a repo label such as `repo:decodex`."); + } + + labels.insert(label.clone()); + } + + if labels.is_empty() { + eyre::bail!("At least one `--repo-label` is required."); + } + + Ok(labels.into_iter().collect()) +} + +fn updated_before_timestamp(older_than_days: u32) -> Result { + if older_than_days == 0 { + eyre::bail!("`--older-than-days` must be greater than zero."); + } + + Ok((OffsetDateTime::now_utc() - Duration::days(i64::from(older_than_days))).format(&Rfc3339)?) +} + +fn build_archive_plan( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + repo_labels: &[String], + updated_before: &str, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let issues = collect_repo_labeled_issues(tracker, repo_labels)?; + let mut candidates = Vec::new(); + let mut skipped = Vec::new(); + + for (issue, matched_repo_labels) in issues { + match archive_skip_reason(tracker, project, workflow, &issue, updated_before)? { + Some(reason) => skipped.push(ArchiveSkip { identifier: issue.identifier, reason }), + None => candidates.push(ArchiveCandidate { + id: issue.id, + identifier: issue.identifier, + title: issue.title, + state: issue.state.name, + updated_at: issue.updated_at, + repo_labels: matched_repo_labels, + }), + } + } + + candidates.sort_by(|left, right| left.identifier.cmp(&right.identifier)); + skipped.sort_by(|left, right| left.identifier.cmp(&right.identifier)); + + Ok(ArchivePlan { candidates, skipped }) +} + +fn collect_repo_labeled_issues( + tracker: &T, + repo_labels: &[String], +) -> Result)>> +where + T: IssueTracker + ?Sized, +{ + let mut issues_by_id: BTreeMap)> = BTreeMap::new(); + + for repo_label in repo_labels { + for issue in tracker.list_issues_with_label(repo_label)? { + let entry = issues_by_id.entry(issue.id.clone()).or_insert((issue, BTreeSet::new())); + + entry.1.insert(repo_label.clone()); + } + } + + Ok(issues_by_id + .into_values() + .map(|(issue, labels)| (issue, labels.into_iter().collect())) + .collect()) +} + +fn archive_skip_reason( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + issue: &TrackerIssue, + updated_before: &str, +) -> Result> +where + T: IssueTracker + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + + if !tracker_policy.terminal_states().iter().any(|state| state == &issue.state.name) { + return Ok(Some(format!( + "state `{}` is not a configured terminal state", + issue.state.name + ))); + } + if issue_updated_at_is_not_older_than_cutoff(issue, updated_before)? { + return Ok(Some(format!( + "updated at `{}` is not older than cutoff `{updated_before}`", + issue.updated_at + ))); + } + + for label in protected_labels(project.service_id(), workflow) { + if tracker::issue_has_label_with_server_confirmation(tracker, issue, &label)? { + return Ok(Some(format!("protected label `{label}` is present"))); + } + } + + Ok(None) +} + +fn issue_updated_at_is_not_older_than_cutoff( + issue: &TrackerIssue, + updated_before: &str, +) -> Result { + let issue_updated_at = OffsetDateTime::parse(&issue.updated_at, &Rfc3339).map_err(|error| { + eyre::eyre!( + "Failed to parse Linear updatedAt `{}` for issue `{}`: {error}", + issue.updated_at, + issue.identifier + ) + })?; + let cutoff = OffsetDateTime::parse(updated_before, &Rfc3339).map_err(|error| { + eyre::eyre!("Failed to parse archive cutoff `{updated_before}`: {error}") + })?; + + Ok(issue_updated_at >= cutoff) +} + +fn protected_labels(service_id: &str, workflow: &WorkflowDocument) -> Vec { + let tracker_policy = workflow.frontmatter().tracker(); + + vec![ + tracker::automation_active_label(service_id), + tracker::automation_queue_label(service_id), + tracker_policy.needs_attention_label().to_owned(), + tracker_policy.opt_out_label().to_owned(), + ] +} + +fn print_archive_plan( + plan: &ArchivePlan, + repo_labels: &[String], + updated_before: &str, + execute: bool, +) { + let mode = if execute { "execute" } else { "dry run" }; + + println!("Linear tracker archive hygiene ({mode})"); + println!("Repo labels: {}", repo_labels.join(", ")); + println!("Updated before: {updated_before}"); + println!("Archive candidates: {}", plan.candidates.len()); + + for candidate in &plan.candidates { + println!( + "- {} [{}] updated={} labels={} title={}", + candidate.identifier, + candidate.state, + candidate.updated_at, + candidate.repo_labels.join(","), + candidate.title + ); + } + + if !plan.skipped.is_empty() { + println!("Skipped: {}", plan.skipped.len()); + + for skipped in &plan.skipped { + println!("- {}: {}", skipped.identifier, skipped.reason); + } + } +} + +#[cfg(test)] +mod tests { + use std::{cell::RefCell, collections::HashMap}; + + use crate::{ + archive_hygiene::{ + self, IssueTracker, Result, ServiceConfig, TrackerIssue, WorkflowDocument, + }, + tracker::{self, TrackerIssueBlocker, TrackerLabel, TrackerState, TrackerTeam}, + }; + + const WORKFLOW: &str = r#"+++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 3 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 3 +gate_profiles = {} +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ +"#; + + #[derive(Default)] + struct FakeArchiveTracker { + issues_by_label: HashMap>, + archived_issue_ids: RefCell>, + } + + impl FakeArchiveTracker { + fn with_label(mut self, label: &str, issues: Vec) -> Self { + self.issues_by_label.insert(label.to_owned(), issues); + + self + } + + fn archive_issue(&self, issue_id: &str) { + self.archived_issue_ids.borrow_mut().push(issue_id.to_owned()); + } + } + + impl IssueTracker for FakeArchiveTracker { + fn list_issues_with_label(&self, label_name: &str) -> Result> { + Ok(self.issues_by_label.get(label_name).cloned().unwrap_or_default()) + } + + fn find_team_label_id(&self, _team_id: &str, _label_name: &str) -> Result> { + Ok(None) + } + + fn get_issue_by_identifier(&self, _issue_identifier: &str) -> Result> { + Ok(None) + } + + fn refresh_issues(&self, _issue_ids: &[String]) -> Result> { + Ok(Vec::new()) + } + + fn list_comments(&self, _issue_id: &str) -> Result> { + Ok(Vec::new()) + } + + fn update_issue_state(&self, _issue_id: &str, _state_id: &str) -> Result<()> { + Ok(()) + } + + fn add_issue_labels(&self, _issue_id: &str, _label_ids: &[String]) -> Result<()> { + Ok(()) + } + + fn remove_issue_labels(&self, _issue_id: &str, _label_ids: &[String]) -> Result<()> { + Ok(()) + } + + fn create_comment(&self, _issue_id: &str, _body: &str) -> Result<()> { + Ok(()) + } + } + + #[test] + fn archive_plan_includes_only_old_terminal_repo_labeled_issues() { + let config = service_config(); + let workflow = workflow(); + let active = tracker::automation_active_label(config.service_id()); + let queued = tracker::automation_queue_label(config.service_id()); + let tracker = FakeArchiveTracker::default().with_label( + "repo:decodex", + vec![ + issue("issue-old", "XY-1", "Done", &["repo:decodex"], "2026-03-01T00:00:00Z"), + issue( + "issue-active", + "XY-2", + "Done", + &["repo:decodex", &active], + "2026-03-01T00:00:00Z", + ), + issue( + "issue-queued", + "XY-3", + "Canceled", + &["repo:decodex", &queued], + "2026-03-01T00:00:00Z", + ), + issue( + "issue-needs", + "XY-4", + "Duplicate", + &["repo:decodex", "decodex:needs-attention"], + "2026-03-01T00:00:00Z", + ), + issue( + "issue-manual", + "XY-5", + "Done", + &["repo:decodex", "decodex:manual-only"], + "2026-03-01T00:00:00Z", + ), + issue("issue-todo", "XY-6", "Todo", &["repo:decodex"], "2026-03-01T00:00:00Z"), + issue("issue-new", "XY-7", "Done", &["repo:decodex"], "2026-04-20T00:00:00Z"), + issue("issue-equal", "XY-8", "Done", &["repo:decodex"], "2026-04-01T00:00:00Z"), + ], + ); + let plan = archive_hygiene::build_archive_plan( + &tracker, + &config, + &workflow, + &[String::from("repo:decodex")], + "2026-04-01T00:00:00Z", + ) + .expect("archive plan should build"); + + assert_eq!( + plan.candidates + .iter() + .map(|candidate| candidate.identifier.as_str()) + .collect::>(), + vec!["XY-1"] + ); + assert_eq!(plan.skipped.len(), 7); + assert!( + plan.skipped + .iter() + .any(|skipped| skipped.reason.contains("protected label `decodex:active:decodex`")) + ); + assert!( + plan.skipped + .iter() + .any(|skipped| skipped.reason.contains("protected label `decodex:queued:decodex`")) + ); + assert!( + plan.skipped.iter().any(|skipped| skipped + .reason + .contains("protected label `decodex:needs-attention`")) + ); + assert!( + plan.skipped + .iter() + .any(|skipped| skipped.reason.contains("protected label `decodex:manual-only`")) + ); + } + + #[test] + fn archive_execution_uses_archive_mutation_only_for_candidates() { + let config = service_config(); + let workflow = workflow(); + let tracker = FakeArchiveTracker::default().with_label( + "repo:decodex", + vec![ + issue("issue-old", "XY-1", "Done", &["repo:decodex"], "2026-03-01T00:00:00Z"), + issue("issue-new", "XY-2", "Done", &["repo:decodex"], "2026-04-20T00:00:00Z"), + ], + ); + let plan = archive_hygiene::build_archive_plan( + &tracker, + &config, + &workflow, + &[String::from("repo:decodex")], + "2026-04-01T00:00:00Z", + ) + .expect("archive plan should build"); + + for candidate in &plan.candidates { + tracker.archive_issue(&candidate.id); + } + + assert_eq!(tracker.archived_issue_ids.borrow().as_slice(), ["issue-old"]); + } + + fn service_config() -> ServiceConfig { + ServiceConfig::parse_toml( + r#" +service_id = "decodex" + +[tracker] +api_key_env_var = "LINEAR_API_KEY_TEST" + +[github] +token_env_var = "GITHUB_TOKEN_TEST" + +[paths] +repo_root = "." +"#, + ) + .expect("config should parse") + } + + fn workflow() -> WorkflowDocument { + WorkflowDocument::parse_markdown(WORKFLOW).expect("workflow should parse") + } + + fn issue( + id: &str, + identifier: &str, + state: &str, + labels: &[&str], + updated_at: &str, + ) -> TrackerIssue { + let team_labels = [ + "repo:decodex", + "decodex:active:decodex", + "decodex:queued:decodex", + "decodex:needs-attention", + "decodex:manual-only", + ]; + + TrackerIssue { + id: id.to_owned(), + identifier: identifier.to_owned(), + #[cfg(test)] + project_slug: Some(String::from("decodex")), + title: format!("Issue {identifier}"), + description: String::new(), + priority: None, + created_at: String::from("2026-02-01T00:00:00Z"), + updated_at: updated_at.to_owned(), + state: TrackerState { id: format!("state-{state}"), name: state.to_owned() }, + team: TrackerTeam { + id: String::from("team-1"), + name: String::from("Decodex"), + states: Vec::new(), + labels: team_labels + .iter() + .enumerate() + .map(|(index, label)| TrackerLabel { + id: format!("team-label-{index}"), + name: (*label).to_owned(), + }) + .collect(), + }, + labels_complete: true, + labels: labels + .iter() + .enumerate() + .map(|(index, label)| TrackerLabel { + id: format!("issue-label-{index}"), + name: (*label).to_owned(), + }) + .collect(), + blockers: Vec::::new(), + } + } +} diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs new file mode 100644 index 00000000..6d7f664a --- /dev/null +++ b/apps/decodex/src/cli.rs @@ -0,0 +1,672 @@ +use std::{ + fs, + io::{self, Read as _}, + path::{Path, PathBuf}, + time::Duration, +}; + +use clap::{ + Args, Parser, Subcommand, + builder::{ + Styles, + styling::{AnsiColor, Effects}, + }, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + agent, + archive_hygiene::{self, ArchiveHygieneRequest}, + manual::{self, ManualCommitRequest, ManualLandRequest}, + orchestrator::{self, IssueDispatchMode, RunOnceRequest, ServeRequest}, + prelude::eyre, + runtime, +}; + +/// Root CLI parser for the Decodex control plane. +#[derive(Debug, Parser)] +#[command( + about = "Repo-native orchestration for autonomous coding agents.", + version = concat!( + env!("CARGO_PKG_VERSION"), + "-", + env!("VERGEN_GIT_SHA"), + "-", + env!("VERGEN_CARGO_TARGET_TRIPLE"), + ), + arg_required_else_help = true, + rename_all = "kebab", + subcommand_required = true, + styles = styles(), +)] +pub(crate) struct Cli { + /// Override the Decodex project directory or its `project.toml` path. + #[arg(short = 'c', long, global = true, value_name = "PROJECT_DIR")] + config: Option, + #[command(subcommand)] + command: Command, +} +impl Cli { + pub(crate) fn run(&self) -> crate::prelude::Result<()> { + let config_path = self.config.as_deref(); + + match &self.command { + Command::Commit(args) => args.run(config_path), + Command::Land(args) => args.run(config_path), + Command::Run(args) => args.run(config_path), + Command::Serve(args) => args.run(config_path), + Command::Project(args) => args.run(), + Command::Status(args) => args.run(config_path), + Command::ArchiveLinear(args) => args.run(config_path), + Command::Probe(args) => args.run(), + Command::Attempt(args) => args.run(config_path), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum AttemptDispatchMode { + Normal, + Retry, + ReviewRepair, + Closeout, +} +impl From for AttemptDispatchMode { + fn from(value: IssueDispatchMode) -> Self { + match value { + IssueDispatchMode::Normal => Self::Normal, + IssueDispatchMode::Retry => Self::Retry, + IssueDispatchMode::ReviewRepair => Self::ReviewRepair, + IssueDispatchMode::Closeout => Self::Closeout, + } + } +} + +impl From for IssueDispatchMode { + fn from(value: AttemptDispatchMode) -> Self { + match value { + AttemptDispatchMode::Normal => Self::Normal, + AttemptDispatchMode::Retry => Self::Retry, + AttemptDispatchMode::ReviewRepair => Self::ReviewRepair, + AttemptDispatchMode::Closeout => Self::Closeout, + } + } +} +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +pub(crate) struct AttemptRequest { + #[serde(default)] + pub(crate) dry_run: bool, + pub(crate) issue_id: String, + pub(crate) issue_state: String, + pub(crate) initial_issue_state: Option, + #[serde(default)] + pub(crate) lease_preacquired: bool, + pub(crate) issue_claim_fd: Option, + pub(crate) dispatch_slot_fd: Option, + pub(crate) dispatch_slot_index: Option, + pub(crate) dispatch_mode: AttemptDispatchMode, + pub(crate) run_id: String, + pub(crate) attempt_number: i64, + pub(crate) retry_budget_base: i64, + pub(crate) workflow_snapshot: String, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Create a signed local commit with a `decodex/commit/1` message. + Commit(CommitCommand), + /// Land the current reviewed lane with a GitHub admin merge commit. + Land(LandCommand), + /// Run one orchestration pass. + Run(RunCommand), + /// Run the local multi-project Decodex control plane. + Serve(ServeCommand), + /// Manage the local Decodex project registry. + Project(ProjectCommand), + /// Inspect the current local runtime state for one configured project. + Status(StatusCommand), + /// Dry-run or archive old terminal Linear issues by repo label. + ArchiveLinear(ArchiveLinearCommand), + /// Validate the local app-server integration boundary. + Probe(ProbeCommand), + /// Run one daemon-planned attempt from a structured request. + #[command(name = "_attempt", hide = true)] + Attempt(AttemptCommand), +} + +#[derive(Debug, Args)] +struct CommitCommand { + /// Tree-change summary for the new commit message. + #[arg(value_name = "SUMMARY")] + summary: String, + /// Primary issue that authorizes the change. Defaults to the current issue worktree name. + #[arg(long, value_name = "ISSUE", conflicts_with = "manual_authority")] + authority: Option, + /// Use reserved authority `manual` instead of a Linear issue. + #[arg(long, conflicts_with = "authority")] + manual_authority: bool, + /// Additional related issues for the commit message. + #[arg(long, value_name = "ISSUE")] + related: Vec, + /// Mark the change as breaking. + #[arg(long)] + breaking: bool, +} +impl CommitCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + manual::run_commit( + config_path, + &ManualCommitRequest { + summary: self.summary.clone(), + authority: self.authority.clone(), + manual_authority: self.manual_authority, + related: self.related.clone(), + breaking: self.breaking, + }, + ) + } +} + +#[derive(Debug, Args)] +struct LandCommand { + /// Tree-change summary for the landed change record. + #[arg(value_name = "SUMMARY")] + summary: String, + /// Primary issue that authorizes the merged change. Defaults to the current issue worktree + /// name. + #[arg(long, value_name = "ISSUE", conflicts_with = "manual_authority")] + authority: Option, + /// Use reserved authority `manual` instead of a Linear issue. + #[arg(long, conflicts_with = "authority")] + manual_authority: bool, + /// Pull request URL to land. Defaults to the current review handoff marker. + #[arg(long, value_name = "URL")] + pr: Option, + /// Additional related issues for the landed change record. + #[arg(long, value_name = "ISSUE")] + related: Vec, + /// Mark the landed change record as breaking. + #[arg(long)] + breaking: bool, +} +impl LandCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + manual::run_land( + config_path, + &ManualLandRequest { + summary: self.summary.clone(), + authority: self.authority.clone(), + manual_authority: self.manual_authority, + pr_url: self.pr.clone(), + related: self.related.clone(), + breaking: self.breaking, + }, + ) + } +} + +#[derive(Debug, Args)] +struct RunCommand { + /// Run a specific leased or queued issue by Linear identifier or tracker issue id. + #[arg(value_name = "ISSUE")] + issue: Option, + /// Skip external side effects where the later implementation allows it. + #[arg(long)] + dry_run: bool, +} +impl RunCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + orchestrator::run_once(RunOnceRequest { + config_path, + dry_run: self.dry_run, + preferred_issue_id: self.issue.as_deref(), + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_lease_acquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + preferred_dispatch_mode: None, + preferred_run_id: None, + preferred_attempt_number: None, + preferred_retry_budget_base: None, + preferred_workflow_snapshot: None, + }) + } +} + +#[derive(Debug, Args)] +struct ServeCommand { + /// Poll interval between control-plane ticks, for example `60s` or `5m`. + #[arg(long, value_name = "INTERVAL", default_value = "60s", value_parser = parse_duration_arg)] + interval: Duration, + /// Operator UI listen address. + #[arg(long, value_name = "ADDR", default_value = "127.0.0.1:8912")] + listen_address: String, +} +impl ServeCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + orchestrator::run_control_plane(ServeRequest { + config_path, + poll_interval: self.interval, + listen_address: &self.listen_address, + }) + } +} + +#[derive(Debug, Args)] +struct ProjectCommand { + #[command(subcommand)] + command: ProjectSubcommand, +} +impl ProjectCommand { + fn run(&self) -> crate::prelude::Result<()> { + let state_store = runtime::open_runtime_store()?; + + match &self.command { + ProjectSubcommand::Add(args) => { + let registration = + runtime::register_project_config(&state_store, &args.config, true)?; + + println!( + "registered project {} at {}", + registration.service_id(), + registration.config_path().display() + ); + }, + ProjectSubcommand::List => { + let projects = state_store.list_projects()?; + + if projects.is_empty() { + println!("No registered projects."); + } else { + for project in projects { + let status = if project.enabled() { "enabled" } else { "disabled" }; + + println!( + "{}\t{}\t{}", + project.service_id(), + status, + project.config_path().display() + ); + } + } + }, + ProjectSubcommand::Enable(args) => { + state_store.set_project_enabled(&args.service_id, true)?; + + println!("enabled project {}", args.service_id); + }, + ProjectSubcommand::Disable(args) => { + state_store.set_project_enabled(&args.service_id, false)?; + + println!("disabled project {}", args.service_id); + }, + } + + Ok(()) + } +} + +#[derive(Debug, Subcommand)] +enum ProjectSubcommand { + /// Register or refresh one Decodex project config. + Add(ProjectAddCommand), + /// List registered local projects. + List, + /// Enable one registered project for `decodex serve`. + Enable(ProjectToggleCommand), + /// Disable one registered project for `decodex serve`. + Disable(ProjectToggleCommand), +} + +#[derive(Debug, Args)] +struct ProjectAddCommand { + /// Path to a Decodex project directory containing `project.toml` and `WORKFLOW.md`. + #[arg(value_name = "PROJECT_DIR")] + config: PathBuf, +} + +#[derive(Debug, Args)] +struct ProjectToggleCommand { + /// Project service id from the registered Decodex config. + #[arg(value_name = "SERVICE_ID")] + service_id: String, +} + +#[derive(Debug, Args)] +struct StatusCommand { + /// Emit structured JSON instead of human-readable text. + #[arg(long)] + json: bool, + /// Maximum number of recent runs to display. + #[arg(long, value_name = "COUNT", default_value_t = orchestrator::DEFAULT_STATUS_RUN_LIMIT)] + limit: usize, +} +impl StatusCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + orchestrator::print_status(config_path, self.json, self.limit) + } +} + +#[derive(Debug, Args)] +struct ArchiveLinearCommand { + /// Repo label scope to inspect, for example `repo:decodex`. + #[arg(long = "repo-label", value_name = "LABEL", required = true)] + repo_labels: Vec, + /// Archive only issues last updated more than this many days ago. + #[arg(long, value_name = "DAYS", default_value_t = 30)] + older_than_days: u32, + /// Perform the archive mutation. Omit this flag for the dry-run candidate report. + #[arg(long)] + execute: bool, +} +impl ArchiveLinearCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + archive_hygiene::run( + config_path, + &ArchiveHygieneRequest { + repo_labels: self.repo_labels.clone(), + older_than_days: self.older_than_days, + execute: self.execute, + }, + ) + } +} + +#[derive(Debug, Args)] +struct ProbeCommand { + /// Override the expected app-server transport during probing. + #[arg(value_name = "TRANSPORT", default_value = "stdio://")] + transport: String, +} +impl ProbeCommand { + fn run(&self) -> crate::prelude::Result<()> { + let report = agent::probe_app_server(&self.transport)?; + + println!( + "probe ok: thread={} turn={} events={} output={}", + report.thread_id, report.turn_id, report.event_count, report.final_output + ); + + tracing::info!( + user_agent = %report.user_agent, + thread_id = %report.thread_id, + turn_id = %report.turn_id, + event_count = report.event_count, + "Completed probe." + ); + + Ok(()) + } +} + +#[derive(Debug, Args)] +struct AttemptCommand { + /// Structured request file path, or `-` to read the request from stdin. + #[arg(value_name = "REQUEST", default_value = "-")] + request: String, +} +impl AttemptCommand { + fn run(&self, config_path: Option<&Path>) -> crate::prelude::Result<()> { + let request = read_attempt_request(&self.request)?; + + orchestrator::run_once(RunOnceRequest { + config_path, + dry_run: request.dry_run, + preferred_issue_id: Some(request.issue_id.as_str()), + preferred_issue_state: Some(request.issue_state.as_str()), + preferred_initial_issue_state: request.initial_issue_state.as_deref(), + preferred_lease_acquired: request.lease_preacquired, + preferred_issue_claim_fd: request.issue_claim_fd, + preferred_dispatch_slot_fd: request.dispatch_slot_fd, + preferred_dispatch_slot_index: request.dispatch_slot_index, + preferred_dispatch_mode: Some(request.dispatch_mode.into()), + preferred_run_id: Some(request.run_id.as_str()), + preferred_attempt_number: Some(request.attempt_number), + preferred_retry_budget_base: Some(request.retry_budget_base), + preferred_workflow_snapshot: Some(request.workflow_snapshot.as_str()), + }) + } +} + +fn parse_duration_arg(raw: &str) -> std::result::Result { + let (number, unit) = raw + .strip_suffix('s') + .map(|value| (value, "s")) + .or_else(|| raw.strip_suffix('m').map(|value| (value, "m"))) + .or_else(|| raw.strip_suffix('h').map(|value| (value, "h"))) + .unwrap_or((raw, "s")); + let value = number.parse::().map_err(|_| { + format!("invalid duration `{raw}`; expected ``, `s`, `m`, or `h`") + })?; + + if value == 0 { + return Err(String::from("duration must be greater than zero")); + } + + match unit { + "s" => Ok(Duration::from_secs(value)), + "m" => value + .checked_mul(60) + .map(Duration::from_secs) + .ok_or_else(|| format!("duration `{raw}` is too large")), + "h" => value + .checked_mul(60) + .and_then(|minutes| minutes.checked_mul(60)) + .map(Duration::from_secs) + .ok_or_else(|| format!("duration `{raw}` is too large")), + _ => Err(format!("unsupported duration unit in `{raw}`")), + } +} + +fn read_attempt_request(request: &str) -> crate::prelude::Result { + let raw = if request == "-" { + let mut raw = String::new(); + + io::stdin().read_to_string(&mut raw)?; + + raw + } else { + fs::read_to_string(request)? + }; + + serde_json::from_str(&raw).map_err(|error| { + eyre::eyre!("Failed to parse `_attempt` request from `{}`: {error}", request) + }) +} + +fn styles() -> Styles { + Styles::styled() + .header(AnsiColor::Red.on_default() | Effects::BOLD) + .usage(AnsiColor::Red.on_default() | Effects::BOLD) + .literal(AnsiColor::Blue.on_default() | Effects::BOLD) + .placeholder(AnsiColor::Green.on_default()) +} + +#[cfg(test)] +mod tests { + use std::{path::PathBuf, time::Duration}; + + use clap::Parser; + + use crate::cli::{ + AttemptCommand, Cli, Command, CommitCommand, LandCommand, ProbeCommand, ProjectCommand, + ProjectSubcommand, RunCommand, ServeCommand, StatusCommand, + }; + + #[test] + fn parses_commit_with_authority_related_and_breaking() { + let cli = Cli::parse_from([ + "decodex", + "commit", + "redesign decodex cli", + "--authority", + "XY-225", + "--related", + "XY-201", + "--related", + "XY-202", + "--breaking", + ]); + + assert!(matches!( + cli.command, + Command::Commit(CommitCommand { + authority: Some(_), + manual_authority: false, + breaking: true, + .. + }) + )); + } + + #[test] + fn parses_land_with_pr_override() { + let cli = Cli::parse_from([ + "decodex", + "land", + "redesign decodex cli", + "--pr", + "https://github.com/hack-ink/decodex/pull/64", + ]); + + assert!(matches!(cli.command, Command::Land(LandCommand { pr: Some(_), .. }))); + } + + #[test] + fn parses_commit_with_manual_authority() { + let cli = Cli::parse_from(["decodex", "commit", "ship hotfix", "--manual-authority"]); + + assert!(matches!( + cli.command, + Command::Commit(CommitCommand { authority: None, manual_authority: true, .. }) + )); + } + + #[test] + fn parses_land_with_manual_authority() { + let cli = Cli::parse_from([ + "decodex", + "land", + "ship hotfix", + "--manual-authority", + "--pr", + "https://github.com/hack-ink/decodex/pull/64", + ]); + + assert!(matches!( + cli.command, + Command::Land(LandCommand { authority: None, manual_authority: true, pr: Some(_), .. }) + )); + } + + #[test] + fn commit_rejects_authority_and_manual_authority_together() { + let error = Cli::try_parse_from([ + "decodex", + "commit", + "ship hotfix", + "--authority", + "XY-225", + "--manual-authority", + ]) + .expect_err("authority and manual-authority should conflict"); + + assert!(error.to_string().contains("--authority")); + assert!(error.to_string().contains("--manual-authority")); + } + + #[test] + fn parses_run_with_positional_issue_and_dry_run() { + let cli = Cli::parse_from(["decodex", "run", "issue-1", "--dry-run"]); + + assert!(matches!(cli.command, Command::Run(RunCommand { issue: Some(_), dry_run: true }))); + } + + #[test] + fn parses_run_without_issue() { + let cli = Cli::parse_from(["decodex", "run"]); + + assert!(matches!(cli.command, Command::Run(RunCommand { issue: None, dry_run: false }))); + } + + #[test] + fn parses_serve_with_interval_listen_address_and_global_config() { + let cli = Cli::parse_from([ + "decodex", + "--config", + "./project.toml", + "serve", + "--interval", + "30s", + "--listen-address", + "127.0.0.1:9000", + ]); + + assert_eq!(cli.config, Some(PathBuf::from("./project.toml"))); + assert!(matches!( + cli.command, + Command::Serve(ServeCommand { interval, listen_address }) + if interval == Duration::from_secs(30) && listen_address == "127.0.0.1:9000" + )); + } + + #[test] + fn parses_project_add() { + let cli = Cli::parse_from(["decodex", "project", "add", "./project.toml"]); + + assert!(matches!( + cli.command, + Command::Project(ProjectCommand { command: ProjectSubcommand::Add(_) }) + )); + } + + #[test] + fn parses_project_enable() { + let cli = Cli::parse_from(["decodex", "project", "enable", "pubfi"]); + + assert!(matches!( + cli.command, + Command::Project(ProjectCommand { command: ProjectSubcommand::Enable(_) }) + )); + } + + #[test] + fn parses_hidden_attempt_with_stdin_request() { + let cli = Cli::parse_from(["decodex", "--config", "./project.toml", "_attempt", "-"]); + + assert_eq!(cli.config, Some(PathBuf::from("./project.toml"))); + assert!(matches!( + cli.command, + Command::Attempt(AttemptCommand { request }) if request == "-" + )); + } + + #[test] + fn parses_probe_with_custom_transport() { + let cli = Cli::parse_from(["decodex", "probe", "ws://127.0.0.1:9000"]); + + assert!(matches!( + cli.command, + Command::Probe(ProbeCommand { transport }) if transport == "ws://127.0.0.1:9000" + )); + } + + #[test] + fn parses_status_with_json_limit_and_global_config() { + let cli = Cli::parse_from([ + "decodex", + "--config", + "./project.toml", + "status", + "--json", + "--limit", + "5", + ]); + + assert_eq!(cli.config, Some(PathBuf::from("./project.toml"))); + assert!(matches!(cli.command, Command::Status(StatusCommand { json: true, limit: 5 }))); + } +} diff --git a/apps/decodex/src/commit_message.rs b/apps/decodex/src/commit_message.rs new file mode 100644 index 00000000..98b00bf5 --- /dev/null +++ b/apps/decodex/src/commit_message.rs @@ -0,0 +1,294 @@ +use serde::{Deserialize, Serialize}; + +use crate::prelude::{Result, eyre}; + +pub(crate) const COMMIT_MESSAGE_SCHEMA: &str = "decodex/commit/1"; +pub(crate) const MANUAL_AUTHORITY: &str = "manual"; + +#[derive(Serialize)] +struct CommitMessage<'a> { + schema: &'static str, + summary: &'a str, + authority: &'a str, + #[serde(skip_serializing_if = "Vec::is_empty")] + related: Vec, + #[serde(skip_serializing_if = "is_false")] + breaking: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CommitMessageRecord { + schema: String, + summary: String, + authority: String, + #[serde(default)] + related: Vec, + #[serde(default)] + breaking: bool, +} + +pub(crate) fn build_commit_message( + summary: &str, + authority: &str, + related: &[String], + breaking: bool, +) -> Result { + let summary = normalize_single_line_field("summary", summary)?; + let authority = normalize_commit_authority("authority", authority)?; + let related = related + .iter() + .map(|value| normalize_issue_identifier("related", value)) + .collect::>>()?; + + serde_json::to_string(&CommitMessage { + schema: COMMIT_MESSAGE_SCHEMA, + summary: summary.as_str(), + authority: authority.as_str(), + related, + breaking, + }) + .map_err(Into::into) +} + +pub(crate) fn build_landing_commit_message( + summary: &str, + authority: &str, + related: &[String], + breaking: bool, +) -> Result { + let summary = normalize_single_line_field("summary", summary)?; + let landed_summary = landing_summary(&summary); + + build_commit_message(&landed_summary, authority, related, breaking) +} + +pub(crate) fn build_landed_merge_commit_message( + head_message: &str, + expected_authority: &str, +) -> Result { + let record = parse_commit_message_record(head_message, expected_authority)?; + let landed_summary = landing_summary(&record.summary); + let authority = normalize_commit_authority("expected_authority", expected_authority)?; + + build_commit_message(&landed_summary, &authority, &record.related, record.breaking) +} + +pub(crate) fn looks_like_issue_identifier(value: &str) -> bool { + let Some((prefix, number)) = value.rsplit_once('-') else { + return false; + }; + + !prefix.is_empty() + && !number.is_empty() + && prefix.chars().all(|character| character.is_ascii_alphanumeric()) + && number.chars().all(|character| character.is_ascii_digit()) +} + +pub(crate) fn normalize_single_line_field(field_name: &str, value: &str) -> Result { + let trimmed = value.trim(); + + if trimmed.is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + if trimmed != value { + eyre::bail!("`{field_name}` must not include surrounding whitespace."); + } + if trimmed.contains('\n') || trimmed.contains('\r') { + eyre::bail!("`{field_name}` must stay on one line."); + } + + Ok(trimmed.to_owned()) +} + +pub(crate) fn normalize_issue_identifier(field_name: &str, value: &str) -> Result { + let normalized = normalize_single_line_field(field_name, value)?; + + if !looks_like_issue_identifier(&normalized) { + eyre::bail!("`{field_name}` must look like an issue identifier such as `XY-123`."); + } + + Ok(normalized) +} + +pub(crate) fn normalize_commit_authority(field_name: &str, value: &str) -> Result { + let normalized = normalize_single_line_field(field_name, value)?; + + if normalized == MANUAL_AUTHORITY || looks_like_issue_identifier(&normalized) { + return Ok(normalized); + } + + eyre::bail!( + "`{field_name}` must look like an issue identifier such as `XY-123` or be exactly `{MANUAL_AUTHORITY}`." + ); +} + +fn landing_summary(summary: &str) -> String { + let summary = summary.strip_prefix("Land ").unwrap_or(summary); + + format!("Land {summary}") +} + +fn parse_commit_message_record( + message: &str, + expected_authority: &str, +) -> Result { + let message = normalize_single_line_field("commit_message", message)?; + let mut record: CommitMessageRecord = serde_json::from_str(&message)?; + + if record.schema != COMMIT_MESSAGE_SCHEMA { + eyre::bail!( + "`commit_message.schema` must be `{COMMIT_MESSAGE_SCHEMA}`, not `{}`.", + record.schema + ); + } + + record.summary = normalize_single_line_field("summary", &record.summary)?; + + let authority = normalize_commit_authority("authority", &record.authority)?; + let expected_authority = normalize_commit_authority("expected_authority", expected_authority)?; + + if !authority.eq_ignore_ascii_case(&expected_authority) { + eyre::bail!( + "`commit_message.authority` `{authority}` does not match expected authority `{expected_authority}`." + ); + } + + for related in &mut record.related { + *related = normalize_issue_identifier("related", related)?; + } + + Ok(record) +} + +fn is_false(value: &bool) -> bool { + !value +} + +#[cfg(test)] +mod tests { + use crate::commit_message::{self}; + + #[test] + fn build_commit_message_omits_empty_related_and_false_breaking() { + let message = + commit_message::build_commit_message("tighten workflow defaults", "XY-225", &[], false) + .expect("commit message should build"); + + assert_eq!( + message, + r#"{"schema":"decodex/commit/1","summary":"tighten workflow defaults","authority":"XY-225"}"# + ); + } + + #[test] + fn build_commit_message_includes_optional_fields() { + let message = commit_message::build_commit_message( + "tighten workflow defaults", + "XY-225", + &[String::from("XY-12"), String::from("XY-99")], + true, + ) + .expect("commit message should build"); + + assert_eq!( + message, + r#"{"schema":"decodex/commit/1","summary":"tighten workflow defaults","authority":"XY-225","related":["XY-12","XY-99"],"breaking":true}"# + ); + } + + #[test] + fn build_landing_commit_message_normalizes_land_prefix() { + for (summary, related, breaking, expected) in [ + ( + "tighten workflow defaults", + vec![String::from("XY-12")], + true, + r#"{"schema":"decodex/commit/1","summary":"Land tighten workflow defaults","authority":"XY-225","related":["XY-12"],"breaking":true}"#, + ), + ( + "Land tighten workflow defaults", + Vec::new(), + false, + r#"{"schema":"decodex/commit/1","summary":"Land tighten workflow defaults","authority":"XY-225"}"#, + ), + ] { + let message = + commit_message::build_landing_commit_message(summary, "XY-225", &related, breaking) + .expect("landing commit message should build"); + + assert_eq!(message, expected); + } + } + + #[test] + fn build_commit_message_rejects_multiline_summary() { + let error = commit_message::build_commit_message("one\ntwo", "XY-225", &[], false) + .expect_err("multiline summary should fail"); + + assert!(error.to_string().contains("must stay on one line")); + } + + #[test] + fn build_commit_message_accepts_manual_authority() { + let message = commit_message::build_commit_message( + "ship hotfix outside tracker", + "manual", + &[], + false, + ) + .expect("manual authority should build"); + + assert_eq!( + message, + r#"{"schema":"decodex/commit/1","summary":"ship hotfix outside tracker","authority":"manual"}"# + ); + } + + #[test] + fn looks_like_issue_identifier_requires_suffix_number() { + assert!(commit_message::looks_like_issue_identifier("XY-225")); + assert!(commit_message::looks_like_issue_identifier("A1-9")); + assert!(!commit_message::looks_like_issue_identifier("XY")); + assert!(!commit_message::looks_like_issue_identifier("XY-")); + assert!(!commit_message::looks_like_issue_identifier("-123")); + assert!(!commit_message::looks_like_issue_identifier("XY-12A")); + } + + #[test] + fn build_landed_merge_commit_message_normalizes_land_prefix() { + for (head_message, expected) in [ + ( + r#"{"schema":"decodex/commit/1","summary":"ship fix","authority":"XY-225","related":["XY-12"],"breaking":true}"#, + r#"{"schema":"decodex/commit/1","summary":"Land ship fix","authority":"XY-225","related":["XY-12"],"breaking":true}"#, + ), + ( + r#"{"schema":"decodex/commit/1","summary":"Land ship fix","authority":"XY-225"}"#, + r#"{"schema":"decodex/commit/1","summary":"Land ship fix","authority":"XY-225"}"#, + ), + ] { + let landed_message = + commit_message::build_landed_merge_commit_message(head_message, "XY-225") + .expect("landed merge message should build"); + + assert_eq!(landed_message, expected); + } + } + + #[test] + fn build_landed_merge_commit_message_rejects_invalid_head_subjects() { + for (head_message, authority, expected) in [ + ( + r#"{"schema":"decodex/commit/1","summary":"ship fix","authority":"XY-225"}"#, + "XY-226", + "does not match expected authority", + ), + ("ship fix", "XY-225", "expected value"), + ] { + let error = commit_message::build_landed_merge_commit_message(head_message, authority) + .expect_err("invalid landed head subject should fail"); + + assert!(error.to_string().contains(expected)); + } + } +} diff --git a/apps/decodex/src/config.rs b/apps/decodex/src/config.rs new file mode 100644 index 00000000..dfe0aeca --- /dev/null +++ b/apps/decodex/src/config.rs @@ -0,0 +1,1375 @@ +//! Service configuration for Decodex. + +#[cfg(unix)] use std::os::unix::ffi::OsStringExt as _; +use std::{ + env, + ffi::OsString, + fs, + io::ErrorKind, + path::{Component, Path, PathBuf}, + process::Command, +}; + +use serde::Deserialize; + +use crate::prelude::{Result, eyre}; + +const WORKFLOW_FILE_NAME: &str = "WORKFLOW.md"; +const PROJECT_CONFIG_FILE_NAME: &str = "project.toml"; + +/// Top-level service configuration for one target repository and tracker integration. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServiceConfig { + service_id: String, + repo_root: PathBuf, + worktree_root: PathBuf, + workflow_path: PathBuf, + tracker: ProjectTrackerConfig, + github: ProjectGitHubConfig, + codex: ProjectCodexConfig, +} +impl ServiceConfig { + /// Parse service configuration from TOML text. + pub fn parse_toml(input: &str) -> Result { + let config_dir = canonicalize_path_best_effort(&env::current_dir()?); + let document = toml::from_str::(input)?; + + Self::from_document(document, &config_dir) + } + + /// Resolve the canonical `project.toml` path for a Decodex project directory. + pub fn resolve_project_config_path(path: impl AsRef) -> Result { + resolve_project_config_file_path(path.as_ref()) + } + + /// Load service configuration from a project directory or its `project.toml` file. + pub fn from_path(path: impl AsRef) -> Result { + let path = Self::resolve_project_config_path(path)?; + let config_dir = config_parent_dir(&path)?; + let input = fs::read_to_string(&path)?; + let document = toml::from_str::(&input)?; + + Self::from_document(document, &config_dir) + } + + /// Stable identifier for this target service config. + pub fn service_id(&self) -> &str { + &self.service_id + } + + /// Absolute repository root used for the target checkout. + pub fn repo_root(&self) -> &Path { + &self.repo_root + } + + /// Worktree root where `decodex` creates issue lanes. + pub fn worktree_root(&self) -> &Path { + &self.worktree_root + } + + /// Absolute path to the project-owned `WORKFLOW.md`. + pub fn workflow_path(&self) -> &Path { + &self.workflow_path + } + + /// Tracker configuration for this project. + pub fn tracker(&self) -> &ProjectTrackerConfig { + &self.tracker + } + + /// GitHub configuration for this project. + pub fn github(&self) -> &ProjectGitHubConfig { + &self.github + } + + /// Codex defaults scoped to this project. + pub fn codex(&self) -> &ProjectCodexConfig { + &self.codex + } + + fn from_document(document: ServiceConfigDocument, config_dir: &Path) -> Result { + document.validate()?; + + let repo_root = document.paths.resolve_repo_root(config_dir)?; + + validate_nonempty_path("repo_root", &repo_root)?; + + Ok(Self { + service_id: document.service_id, + repo_root: repo_root.to_path_buf(), + worktree_root: document.paths.resolve_worktree_root(&repo_root)?, + workflow_path: config_dir.join(WORKFLOW_FILE_NAME), + tracker: document.tracker, + github: document.github, + codex: document.codex.resolve_paths(config_dir)?, + }) + } +} + +/// Tracker-specific settings for a target project. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectTrackerConfig { + api_key_env_var: String, +} +impl ProjectTrackerConfig { + /// Name of the environment variable that stores the tracker API key. + pub fn api_key_env_var(&self) -> &str { + &self.api_key_env_var + } + + /// Resolve the configured tracker API key env-var name into a concrete token string. + pub fn resolve_api_key(&self) -> Result { + resolve_secret_env_var("tracker.api_key_env_var", self.api_key_env_var()) + } + + fn validate(&self) -> Result<()> { + validate_env_var_name("tracker.api_key_env_var", self.api_key_env_var())?; + + Ok(()) + } +} + +/// GitHub settings for a target project. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectGitHubConfig { + token_env_var: String, +} +impl ProjectGitHubConfig { + /// Name of the environment variable that stores the GitHub token. + pub fn token_env_var(&self) -> &str { + &self.token_env_var + } + + /// Resolve the configured GitHub token env-var name into a concrete token string. + pub fn resolve_token(&self) -> Result { + resolve_secret_env_var("github.token_env_var", self.token_env_var()) + } + + fn validate(&self) -> Result<()> { + validate_env_var_name("github.token_env_var", self.token_env_var())?; + + Ok(()) + } +} + +/// Project-level Codex defaults from service configuration. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectCodexConfig { + #[serde(default = "default_internal_review_mode")] + internal_review_mode: InternalReviewMode, + #[serde(default = "default_external_review_enabled")] + external_review_enabled: bool, + accounts: Option, +} +impl ProjectCodexConfig { + /// Internal self-review behavior Decodex should request for agent runs. + pub fn internal_review_mode(&self) -> InternalReviewMode { + self.internal_review_mode + } + + /// Whether Decodex should drive the retained external `@codex review` loop. + pub fn external_review_enabled(&self) -> bool { + self.external_review_enabled + } + + /// Optional ChatGPT accounts used to seed Codex app-server auth. + pub fn accounts(&self) -> Option<&ProjectCodexAccountsConfig> { + self.accounts.as_ref() + } + + fn resolve_paths(mut self, config_dir: &Path) -> Result { + if let Some(accounts) = self.accounts.take() { + self.accounts = Some(accounts.resolve_paths(config_dir)?); + } + + Ok(self) + } + + fn validate(&self) -> Result<()> { + if let Some(accounts) = &self.accounts { + accounts.validate()?; + } + + Ok(()) + } +} + +impl Default for ProjectCodexConfig { + fn default() -> Self { + Self { + internal_review_mode: default_internal_review_mode(), + external_review_enabled: default_external_review_enabled(), + accounts: None, + } + } +} + +/// Optional JSONL ChatGPT accounts for Codex app-server runs. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectCodexAccountsConfig { + path: PathBuf, + usage_endpoint: Option, + refresh_endpoint: Option, +} +impl ProjectCodexAccountsConfig { + /// JSONL file containing one composite auth.json-style account per line. + pub fn path(&self) -> &Path { + &self.path + } + + /// Override for ChatGPT usage probes. Defaults to the Codex `/wham/usage` endpoint. + pub fn usage_endpoint(&self) -> Option<&str> { + self.usage_endpoint.as_deref() + } + + /// Override for ChatGPT OAuth refresh. Defaults to the Codex auth token endpoint. + pub fn refresh_endpoint(&self) -> Option<&str> { + self.refresh_endpoint.as_deref() + } + + fn validate(&self) -> Result<()> { + validate_nonempty_path("codex.accounts.path", &self.path)?; + validate_optional_nonempty_string( + "codex.accounts.usage_endpoint", + self.usage_endpoint.as_deref(), + )?; + validate_optional_nonempty_string( + "codex.accounts.refresh_endpoint", + self.refresh_endpoint.as_deref(), + )?; + + Ok(()) + } + + fn resolve_paths(mut self, config_dir: &Path) -> Result { + self.path = resolve_config_path(config_dir, &self.path)?; + + Ok(self) + } +} + +/// Optional service-level path overrides. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectPathsConfig { + repo_root: Option, + worktree_root: Option, +} +impl ProjectPathsConfig { + fn validate(&self) -> Result<()> { + if self.repo_root.is_none() { + eyre::bail!("`paths.repo_root` is required for every Decodex project config."); + } + + if let Some(repo_root) = self.repo_root.as_deref() { + validate_nonempty_path("paths.repo_root", repo_root)?; + } + if let Some(worktree_root) = self.worktree_root.as_deref() { + validate_nonempty_path("paths.worktree_root", worktree_root)?; + } + + Ok(()) + } + + fn resolve_repo_root(&self, config_dir: &Path) -> Result { + let Some(path) = self.repo_root.as_deref() else { + eyre::bail!("`paths.repo_root` is required for every Decodex project config."); + }; + let repo_root = resolve_relative_path(config_dir, path); + let repo_root = canonicalize_path_best_effort(&repo_root); + + validate_nonempty_path("paths.repo_root", &repo_root)?; + + Ok(repo_root) + } + + fn resolve_worktree_root(&self, repo_root: &Path) -> Result { + let worktree_root = self.worktree_root.as_deref().map_or_else( + || repo_root.join(".worktrees"), + |path| resolve_relative_path(repo_root, path), + ); + + validate_nonempty_path("paths.worktree_root", &worktree_root)?; + + Ok(worktree_root) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +struct ServiceConfigDocument { + service_id: String, + tracker: ProjectTrackerConfig, + github: ProjectGitHubConfig, + #[serde(default)] + codex: ProjectCodexConfig, + #[serde(default)] + paths: ProjectPathsConfig, +} +impl ServiceConfigDocument { + fn validate(&self) -> Result<()> { + validate_service_id("service_id", &self.service_id)?; + + self.tracker.validate()?; + self.github.validate()?; + self.codex.validate()?; + self.paths.validate()?; + + Ok(()) + } +} + +/// Internal self-review mode for agent runs. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InternalReviewMode { + /// Use the existing runtime-owned checkpoint loop. + Loop, + /// Add a prompt-only self-review instruction without the checkpoint loop. + Prompt, + /// Disable internal self-review behavior. + Off, +} +impl InternalReviewMode { + /// Config string for this mode. + pub const fn as_str(self) -> &'static str { + match self { + Self::Loop => "loop", + Self::Prompt => "prompt", + Self::Off => "off", + } + } + + /// Whether this mode uses the structured checkpoint loop. + pub const fn requires_review_checkpoint(self) -> bool { + matches!(self, Self::Loop) + } +} + +impl Default for InternalReviewMode { + fn default() -> Self { + default_internal_review_mode() + } +} + +/// Canonical repository root for the current Git checkout. +pub fn canonical_repo_root_for_checkout(cwd: &Path) -> Result> { + let worktree_root = git_absolute_rev_parse(cwd, "show-toplevel")? + .map(|path| canonicalize_path_best_effort(&path)); + + if let Some(shared_repo_root) = shared_repo_root_for_checkout(cwd, worktree_root.as_deref())? { + return Ok(Some(shared_repo_root)); + } + + Ok(worktree_root) +} + +/// Absolute Git administrative directory for the current checkout. +pub fn git_dir_for_checkout(cwd: &Path) -> Result> { + Ok(git_absolute_rev_parse(cwd, "git-dir")?.map(|path| canonicalize_path_best_effort(&path))) +} + +/// Whether two Git checkouts belong to the same shared repository. +pub fn checkouts_share_repository(a: &Path, b: &Path) -> Result { + let a_common_dir = git_absolute_rev_parse(a, "git-common-dir")? + .map(|path| canonicalize_path_best_effort(&path)); + let b_common_dir = git_absolute_rev_parse(b, "git-common-dir")? + .map(|path| canonicalize_path_best_effort(&path)); + + Ok(a_common_dir.is_some() && a_common_dir == b_common_dir) +} + +const fn default_external_review_enabled() -> bool { + true +} + +const fn default_internal_review_mode() -> InternalReviewMode { + InternalReviewMode::Loop +} + +fn shared_repo_root_for_checkout( + cwd: &Path, + worktree_root: Option<&Path>, +) -> Result> { + let git_dir = + git_absolute_rev_parse(cwd, "git-dir")?.map(|path| canonicalize_path_best_effort(&path)); + let common_dir = git_absolute_rev_parse(cwd, "git-common-dir")? + .map(|path| canonicalize_path_best_effort(&path)); + let prefers_shared_repo_root = git_dir.is_some() && git_dir != common_dir; + + if prefers_shared_repo_root { + return shared_repo_root_for_linked_worktree(cwd, worktree_root, common_dir.as_deref()); + } + + Ok(None) +} + +fn shared_repo_root_for_linked_worktree( + cwd: &Path, + worktree_root: Option<&Path>, + common_dir: Option<&Path>, +) -> Result> { + let Some(worktree_root) = worktree_root else { + return Ok(None); + }; + let Some(common_dir) = common_dir else { + return Ok(None); + }; + + if let Some(shared_repo_root) = + repo_root_from_git_worktree_list(cwd, common_dir, worktree_root)? + { + return Ok(Some(shared_repo_root)); + } + if let Some(shared_repo_root) = + repo_root_from_gitdir_reference_search(common_dir, worktree_root)? + { + return Ok(Some(shared_repo_root)); + } + + Ok(None) +} + +fn repo_root_from_git_worktree_list( + cwd: &Path, + common_dir: &Path, + worktree_root: &Path, +) -> Result> { + for path in git_worktree_roots(cwd)? { + let path = canonicalize_path_best_effort(&path); + + if path == worktree_root || path == common_dir { + continue; + } + if git_absolute_rev_parse(&path, "git-common-dir")? + .map(|path| canonicalize_path_best_effort(&path)) + .as_deref() + != Some(common_dir) + { + continue; + } + if git_absolute_rev_parse(&path, "git-dir")? + .map(|path| canonicalize_path_best_effort(&path)) + .as_deref() + == Some(common_dir) + { + return Ok(Some(path)); + } + } + + Ok(None) +} + +fn repo_root_from_gitdir_reference_search( + common_dir: &Path, + worktree_root: &Path, +) -> Result> { + let Some(search_root) = nearest_shared_ancestor(common_dir, worktree_root) else { + return Ok(None); + }; + + find_checkout_root_referencing_common_dir(&search_root, common_dir, worktree_root) +} + +fn git_worktree_roots(cwd: &Path) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(cwd) + .args(["worktree", "list", "--porcelain", "-z"]) + .output()?; + + if !output.status.success() { + return Ok(Vec::new()); + } + + parse_git_worktree_list(&output.stdout) +} + +fn parse_git_worktree_list(output: &[u8]) -> Result> { + let mut roots = Vec::new(); + + for entry in output.split(|byte| *byte == 0).filter(|entry| !entry.is_empty()) { + let Some(path_bytes) = entry.strip_prefix(b"worktree ") else { + continue; + }; + let Some(path) = path_buf_from_git_bytes(path_bytes)? else { + continue; + }; + + roots.push(path); + } + + Ok(roots) +} + +fn nearest_shared_ancestor(a: &Path, b: &Path) -> Option { + a.ancestors().find(|ancestor| b.starts_with(ancestor)).map(Path::to_path_buf) +} + +fn find_checkout_root_referencing_common_dir( + search_root: &Path, + common_dir: &Path, + worktree_root: &Path, +) -> Result> { + const MAX_DIRS_TO_SCAN: usize = 4_096; + + let mut stack = vec![search_root.to_path_buf()]; + let mut scanned_dirs = 0_usize; + + while let Some(path) = stack.pop() { + if scanned_dirs >= MAX_DIRS_TO_SCAN { + return Ok(None); + } + + scanned_dirs += 1; + + if path != worktree_root + && path != common_dir + && git_dir_reference_matches_common_dir_best_effort(&path.join(".git"), common_dir) + { + return Ok(Some(path)); + } + + let entries = match fs::read_dir(&path) { + Ok(entries) => entries, + Err(error) if error.kind() == ErrorKind::NotFound => continue, + Err(error) => return Err(error.into()), + }; + + for entry in entries { + let entry = entry?; + let child = entry.path(); + + if !child.is_dir() + || child == common_dir + || child.starts_with(common_dir) + || child == worktree_root + || child.starts_with(worktree_root) + { + continue; + } + + stack.push(child); + } + } + + Ok(None) +} + +fn git_dir_reference_matches_common_dir_best_effort(dot_git: &Path, common_dir: &Path) -> bool { + git_dir_reference_matches_common_dir(dot_git, common_dir).unwrap_or_default() +} + +fn git_dir_reference_matches_common_dir(dot_git: &Path, common_dir: &Path) -> Result { + if dot_git.is_dir() { + return Ok(fs::canonicalize(dot_git)? == common_dir); + } + if !dot_git.is_file() { + return Ok(false); + } + + let gitdir = parse_gitdir_file(dot_git)?; + + Ok(fs::canonicalize(gitdir)? == common_dir) +} + +fn parse_gitdir_file(dot_git: &Path) -> Result { + let contents = fs::read_to_string(dot_git)?; + let prefix = "gitdir:"; + let Some(gitdir) = contents.lines().find_map(|line| line.strip_prefix(prefix)) else { + eyre::bail!("Git dir file `{}` is missing a `gitdir:` entry.", dot_git.display()); + }; + let gitdir = gitdir.trim(); + + if gitdir.is_empty() { + eyre::bail!("Git dir file `{}` has an empty `gitdir:` entry.", dot_git.display()); + } + + let gitdir = PathBuf::from(gitdir); + + if gitdir.is_absolute() { + return Ok(gitdir); + } + + let Some(parent) = dot_git.parent() else { + eyre::bail!("Git dir file `{}` must have a parent directory.", dot_git.display()); + }; + + Ok(parent.join(gitdir)) +} + +fn git_absolute_rev_parse(cwd: &Path, mode: &str) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(cwd) + .args(["rev-parse", "--path-format=absolute", &format!("--{mode}")]) + .output()?; + + if !output.status.success() { + return Ok(None); + } + + path_buf_from_git_line_output(&output.stdout) +} + +fn path_buf_from_git_line_output(output: &[u8]) -> Result> { + let resolved = output.strip_suffix(b"\n").unwrap_or(output); + let resolved = resolved.strip_suffix(b"\r").unwrap_or(resolved); + + path_buf_from_git_bytes(resolved) +} + +fn canonicalize_path_best_effort(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +#[cfg(unix)] +fn path_buf_from_git_bytes(path: &[u8]) -> Result> { + if path.is_empty() { + return Ok(None); + } + + Ok(Some(PathBuf::from(OsString::from_vec(path.to_vec())))) +} + +#[cfg(not(unix))] +fn path_buf_from_git_bytes(path: &[u8]) -> Result> { + let resolved = String::from_utf8(path.to_vec())?; + + if resolved.is_empty() { + return Ok(None); + } + + Ok(Some(PathBuf::from(resolved))) +} + +fn validate_nonempty_path(field_name: &str, value: &Path) -> Result<()> { + if value.as_os_str().is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + + Ok(()) +} + +fn validate_optional_nonempty_string(field_name: &str, value: Option<&str>) -> Result<()> { + let Some(value) = value else { + return Ok(()); + }; + + if value.trim().is_empty() { + eyre::bail!("`{field_name}` must not be empty when configured."); + } + + Ok(()) +} + +fn resolve_project_config_file_path(path: &Path) -> Result { + let metadata = fs::metadata(path).map_err(|error| { + eyre::eyre!("Failed to inspect Decodex project config path `{}`: {error}", path.display()) + })?; + + if metadata.is_dir() { + return Ok(path.join(PROJECT_CONFIG_FILE_NAME)); + } + if path.file_name().and_then(|name| name.to_str()) == Some(PROJECT_CONFIG_FILE_NAME) { + return Ok(path.to_path_buf()); + } + + eyre::bail!( + "Decodex project config must be a project directory or `{PROJECT_CONFIG_FILE_NAME}` file: `{}`.", + path.display() + ); +} + +fn config_parent_dir(config_path: &Path) -> Result { + let canonical_path = fs::canonicalize(config_path)?; + let Some(parent) = canonical_path.parent() else { + eyre::bail!("Config path `{}` must have a parent directory.", config_path.display()); + }; + + Ok(parent.to_path_buf()) +} + +fn resolve_relative_path(base: &Path, path: &Path) -> PathBuf { + let resolved = if path.is_absolute() { path.to_path_buf() } else { base.join(path) }; + + normalize_path(&resolved) +} + +fn resolve_config_path(base: &Path, path: &Path) -> Result { + let expanded = expand_home_path(path)?; + + Ok(resolve_relative_path(base, &expanded)) +} + +fn expand_home_path(path: &Path) -> Result { + let Some(path_text) = path.to_str() else { + return Ok(path.to_path_buf()); + }; + let Some(rest) = path_text.strip_prefix("~/") else { + if path_text == "~" { + return env::var_os("HOME") + .map(PathBuf::from) + .ok_or_else(|| eyre::eyre!("`HOME` is required to expand `~`.")); + } + + return Ok(path.to_path_buf()); + }; + let Some(home) = env::var_os("HOME") else { + eyre::bail!("`HOME` is required to expand `{path_text}`."); + }; + + Ok(PathBuf::from(home).join(rest)) +} + +fn normalize_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + + for component in path.components() { + match component { + Component::CurDir => {}, + Component::ParentDir => match normalized.components().next_back() { + Some(Component::Normal(_)) => { + normalized.pop(); + }, + Some(Component::RootDir | Component::Prefix(_)) => {}, + Some(Component::ParentDir) | None => normalized.push(component.as_os_str()), + Some(Component::CurDir) => {}, + }, + _ => normalized.push(component.as_os_str()), + } + } + + if normalized.as_os_str().is_empty() { PathBuf::from(".") } else { normalized } +} + +fn validate_service_id(field_name: &str, value: &str) -> Result<()> { + let trimmed = value.trim(); + + if trimmed.is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + if trimmed != value { + eyre::bail!("`{field_name}` must not include surrounding whitespace."); + } + + let mut chars = trimmed.chars(); + let Some(first) = chars.next() else { + eyre::bail!("`{field_name}` must not be empty."); + }; + + if !(first.is_ascii_lowercase() || first.is_ascii_digit()) { + eyre::bail!("`{field_name}` must start with a lowercase ASCII letter or digit."); + } + if chars.any(|character| { + !(character.is_ascii_lowercase() + || character.is_ascii_digit() + || matches!(character, '-' | '_')) + }) { + eyre::bail!( + "`{field_name}` must contain only lowercase ASCII letters, digits, hyphens, or underscores." + ); + } + + Ok(()) +} + +fn validate_env_var_name(field_name: &str, value: &str) -> Result<()> { + let trimmed = value.trim(); + + if trimmed.is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + if trimmed != value { + eyre::bail!("`{field_name}` must not include surrounding whitespace."); + } + if trimmed.starts_with('$') { + eyre::bail!( + "`{field_name}` must name the environment variable directly, without a `$` prefix." + ); + } + + let mut chars = trimmed.chars(); + let Some(first) = chars.next() else { + eyre::bail!("`{field_name}` must not be empty."); + }; + + if !(first == '_' || first.is_ascii_alphabetic()) { + eyre::bail!( + "`{field_name}` must start with an ASCII letter or underscore and contain only ASCII letters, digits, or underscores." + ); + } + if chars.any(|character| !(character == '_' || character.is_ascii_alphanumeric())) { + eyre::bail!("`{field_name}` must contain only ASCII letters, digits, or underscores."); + } + + Ok(()) +} + +fn resolve_secret_env_var(field_name: &str, env_var: &str) -> Result { + validate_env_var_name(field_name, env_var)?; + + let value = env::var(env_var).map_err(|error| { + eyre::eyre!( + "Failed to read environment variable `{env_var}` referenced by `{field_name}`: {error}" + ) + })?; + + if value.trim().is_empty() { + eyre::bail!( + "Environment variable `{env_var}` referenced by `{field_name}` must not be blank." + ); + } + + Ok(value) +} + +#[cfg(test)] +mod tests { + use std::{ + env, + ffi::OsString, + fs, + path::{Path, PathBuf}, + sync::{Mutex, MutexGuard, OnceLock}, + }; + + use tempfile::TempDir; + + use crate::{ + config::{self, InternalReviewMode, ServiceConfig}, + worktree::WorktreeManager, + }; + + struct TestEnvVarGuard { + key: String, + previous: Option, + } + impl TestEnvVarGuard { + fn lock() -> MutexGuard<'static, ()> { + static ENV_LOCK: OnceLock> = OnceLock::new(); + + ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env var mutex should not be poisoned") + } + + fn set(key: &str, value: &str) -> Self { + let _guard = Self::lock(); + let previous = env::var_os(key); + + unsafe { env::set_var(key, value) }; + + Self { key: key.to_owned(), previous } + } + } + + impl Drop for TestEnvVarGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(previous) => unsafe { env::set_var(&self.key, previous) }, + None => unsafe { env::remove_var(&self.key) }, + } + } + } + + fn write_config_file(dir: &Path, body: &str) -> PathBuf { + let config_path = dir.join("project.toml"); + let body = body_with_explicit_repo_root(body); + + fs::write(&config_path, body).expect("config should write"); + + config_path + } + + #[test] + fn loads_service_config_from_project_file_with_explicit_repo_root() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + "#, + ); + let config = + ServiceConfig::from_path(&config_path).expect("service config should load from disk"); + let canonical_root = + fs::canonicalize(temp_dir.path()).expect("temp dir should canonicalize"); + + assert_eq!(config.service_id(), "pubfi"); + assert_eq!(config.repo_root(), canonical_root); + assert_eq!(config.worktree_root(), canonical_root.join(".worktrees")); + assert_eq!(config.workflow_path(), canonical_root.join("WORKFLOW.md")); + assert_eq!(config.github().token_env_var(), "HOME"); + assert_eq!(config.codex().internal_review_mode(), InternalReviewMode::Loop); + assert!(config.codex().external_review_enabled()); + } + + #[test] + fn loads_service_config_from_project_directory() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + "#, + ); + let config = ServiceConfig::from_path(temp_dir.path()) + .expect("service config should load from project directory"); + + assert_eq!(config.service_id(), "pubfi"); + assert_eq!( + ServiceConfig::resolve_project_config_path(temp_dir.path()) + .expect("project directory should resolve"), + config_path + ); + } + + fn body_with_explicit_repo_root(body: &str) -> String { + if body.contains("repo_root") { + return body.to_owned(); + } + if body.contains("[paths]") { + return body.replacen("[paths]", "[paths]\nrepo_root = \".\"", 1); + } + + format!("{body}\n\n[paths]\nrepo_root = \".\"\n") + } + + #[test] + fn loads_service_config_with_relative_worktree_override() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + + [paths] + worktree_root = "var/worktrees" + "#, + ); + let config = + ServiceConfig::from_path(&config_path).expect("service config should load from disk"); + let canonical_root = + fs::canonicalize(temp_dir.path()).expect("temp dir should canonicalize"); + + assert_eq!(config.worktree_root(), canonical_root.join("var/worktrees")); + } + + #[test] + fn loads_service_config_from_external_project_file_with_explicit_repo_root() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("target-repo"); + let config_dir = temp_dir.path().join("codex/decodex/projects/rsnap"); + let config_path = config_dir.join("project.toml"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(&config_dir).expect("config dir should exist"); + fs::write( + &config_path, + r#" + service_id = "rsnap" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + + [paths] + repo_root = "../../../../target-repo" + worktree_root = "lanes" + "#, + ) + .expect("centralized config should write"); + + let config = + ServiceConfig::from_path(&config_path).expect("centralized config should load"); + let canonical_root = fs::canonicalize(&repo_root).expect("repo root should canonicalize"); + + assert_eq!(config.service_id(), "rsnap"); + assert_eq!(config.repo_root(), canonical_root); + assert_eq!(config.worktree_root(), canonical_root.join("lanes")); + assert_eq!( + config.workflow_path(), + fs::canonicalize(&config_dir) + .expect("config dir should canonicalize") + .join("WORKFLOW.md") + ); + } + + #[test] + fn rejects_project_config_with_nonstandard_file_name() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = temp_dir.path().join("rsnap.toml"); + + fs::write(&config_path, "").expect("config should write"); + + let error = ServiceConfig::from_path(&config_path) + .expect_err("nonstandard config file name should fail"); + + assert!( + error.to_string().contains("project.toml"), + "error should explain the fixed config file name: {error:?}" + ); + } + + #[test] + fn external_project_config_requires_explicit_repo_root() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = temp_dir.path().join("project.toml"); + + fs::write( + &config_path, + r#" + service_id = "rsnap" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + "#, + ) + .expect("centralized config should write"); + + let error = + ServiceConfig::from_path(&config_path).expect_err("repo_root should be required"); + + assert!( + error.to_string().contains("paths.repo_root"), + "error should explain the missing explicit repo root: {error:?}" + ); + } + + #[test] + fn parses_codex_review_settings() { + for (case_name, codex_body, expected_mode, expected_external_review) in [ + ( + "explicit off mode and disabled external review", + r#" + internal_review_mode = "off" + external_review_enabled = false"#, + InternalReviewMode::Off, + false, + ), + ( + "prompt mode keeps external review default enabled", + r#" + internal_review_mode = "prompt""#, + InternalReviewMode::Prompt, + true, + ), + ] { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + &format!( + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + + [codex] + {codex_body} + "# + ), + ); + let config = ServiceConfig::from_path(&config_path).expect(case_name); + + assert_eq!(config.codex().internal_review_mode(), expected_mode); + assert_eq!(config.codex().external_review_enabled(), expected_external_review); + } + } + + #[test] + fn parses_codex_accounts_settings() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + + [codex.accounts] + path = "accounts/codex-auth.jsonl" + usage_endpoint = "http://127.0.0.1:1234/wham/usage" + refresh_endpoint = "http://127.0.0.1:1234/oauth/token" + "#, + ); + let config = ServiceConfig::from_path(&config_path).expect("accounts should parse"); + let accounts = config.codex().accounts().expect("accounts should be configured"); + let expected_path = temp_dir + .path() + .canonicalize() + .expect("temp dir should canonicalize") + .join("accounts/codex-auth.jsonl"); + + assert_eq!(accounts.path(), expected_path); + assert_eq!(accounts.usage_endpoint(), Some("http://127.0.0.1:1234/wham/usage")); + assert_eq!(accounts.refresh_endpoint(), Some("http://127.0.0.1:1234/oauth/token")); + } + + #[test] + fn rejects_unknown_codex_internal_review_mode() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + + [codex] + internal_review_mode = "prompt_only" + "#, + ); + let error = ServiceConfig::from_path(&config_path) + .expect_err("unknown internal review mode should fail"); + + assert!(error.to_string().contains("prompt_only")); + } + + #[test] + fn rejects_empty_github_token_env_var() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "" + "#, + ); + let error = ServiceConfig::from_path(&config_path) + .expect_err("empty github token env-var should be rejected"); + + assert!(error.to_string().contains("github.token_env_var")); + } + + #[test] + fn rejects_blank_secret_env_var_values_when_resolving() { + #[derive(Clone, Copy)] + enum SecretTarget { + Github, + Tracker, + } + + for (case_name, env_var, env_value, target) in [ + ( + "empty github token env-var value", + "DECODEX_TEST_EMPTY_GITHUB_TOKEN", + "", + SecretTarget::Github, + ), + ( + "whitespace-only github token env-var value", + "DECODEX_TEST_BLANK_GITHUB_TOKEN", + " ", + SecretTarget::Github, + ), + ( + "whitespace-only tracker api key env-var value", + "DECODEX_TEST_BLANK_TRACKER_API_KEY", + " ", + SecretTarget::Tracker, + ), + ] { + let _guard = TestEnvVarGuard::set(env_var, env_value); + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + &format!( + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "{}" + + [github] + token_env_var = "{}" + "#, + match target { + SecretTarget::Github => "HOME", + SecretTarget::Tracker => env_var, + }, + match target { + SecretTarget::Github => env_var, + SecretTarget::Tracker => "HOME", + }, + ), + ); + let config = + ServiceConfig::from_path(&config_path).expect("service config should parse"); + let error = match target { + SecretTarget::Github => config.github().resolve_token(), + SecretTarget::Tracker => config.tracker().resolve_api_key(), + } + .expect_err(case_name); + + assert!( + error.to_string().contains("must not be blank"), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn rejects_invalid_service_ids() { + for (case_name, service_id, expected) in [ + ("empty service_id", "", "service_id"), + ( + "service_id with non-slug characters", + "pub:fi", + "lowercase ASCII letters, digits, hyphens, or underscores", + ), + ] { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let config_path = write_config_file( + temp_dir.path(), + &format!( + r#" + service_id = "{service_id}" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "HOME" + "# + ), + ); + let error = ServiceConfig::from_path(&config_path).expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[cfg(unix)] + #[test] + fn git_path_output_preserves_non_utf8_bytes() { + let path = super::path_buf_from_git_line_output(b"/tmp/\xFFlane\n") + .expect("git path output should parse") + .expect("git path output should not be empty"); + + assert_eq!(std::os::unix::ffi::OsStrExt::as_bytes(path.as_os_str()), b"/tmp/\xFFlane"); + } + + #[test] + fn canonical_repo_root_for_checkout_prefers_shared_repo_root_for_linked_worktree() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("target-repo"); + let worktree_root = repo_root.join(".worktrees"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + assert!( + std::process::Command::new("git") + .args(["init", "-b", "main"]) + .current_dir(temp_dir.path()) + .arg(&repo_root) + .status() + .expect("git init should run") + .success() + ); + assert!( + std::process::Command::new("git") + .args(["config", "user.name", "Decodex Tests"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + assert!( + std::process::Command::new("git") + .args(["config", "user.email", "decodex-tests@example.com"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + assert!( + std::process::Command::new("git") + .args(["config", "commit.gpgsign", "false"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + + fs::write(repo_root.join("README.md"), "bootstrap\n").expect("readme should write"); + + assert!( + std::process::Command::new("git") + .args(["add", "README.md"]) + .current_dir(&repo_root) + .status() + .expect("git add should run") + .success() + ); + assert!( + std::process::Command::new("git") + .args(["commit", "-m", "seed repo"]) + .current_dir(&repo_root) + .status() + .expect("git commit should run") + .success() + ); + + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let worktree = manager.ensure_worktree("XY-251", false).expect("worktree should create"); + let canonical_repo_root = + fs::canonicalize(&repo_root).expect("repo root should canonicalize"); + + assert_eq!( + config::canonical_repo_root_for_checkout(&worktree.path) + .expect("canonical repo root should resolve") + .expect("linked worktree should expose a canonical repo root"), + canonical_repo_root + ); + } +} diff --git a/apps/decodex/src/default_branch_sync.rs b/apps/decodex/src/default_branch_sync.rs new file mode 100644 index 00000000..45d4ba63 --- /dev/null +++ b/apps/decodex/src/default_branch_sync.rs @@ -0,0 +1,517 @@ +use std::{collections::BTreeSet, path::Path, process::Command}; + +use crate::{ + git_credentials::{GitAskpassGuard, GitCredentialEnvironment, GitCredentialSource}, + prelude::{Result, eyre}, +}; + +pub(crate) fn sync_repo_root_default_branch( + repo_root: &Path, + default_branch: &str, + credentials: Option>, +) -> Result<()> { + let (git_env, _askpass_guard) = materialize_git_credentials(credentials)?; + + preflight_repo_root_default_branch_sync_with_env(repo_root, default_branch, &git_env)?; + + fast_forward_default_branch(repo_root, default_branch, &git_env) +} + +pub(crate) fn preflight_repo_root_default_branch_sync( + repo_root: &Path, + default_branch: &str, + credentials: Option>, +) -> Result<()> { + let (git_env, _askpass_guard) = materialize_git_credentials(credentials)?; + + preflight_repo_root_default_branch_sync_with_env(repo_root, default_branch, &git_env) +} + +fn preflight_repo_root_default_branch_sync_with_env( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let current_branch = run_git_capture(repo_root, &["branch", "--show-current"], git_env)?; + + if current_branch.is_empty() { + eyre::bail!( + "Configured repo root `{}` is detached; landing closeout cannot fast-forward local `{default_branch}` there.", + repo_root.display() + ); + } + if current_branch != default_branch { + eyre::bail!( + "Configured repo root `{}` is on branch `{current_branch}`, but landing closeout must fast-forward local `{default_branch}` there.", + repo_root.display() + ); + } + + ensure_clean_default_branch_worktree(repo_root, default_branch, git_env)?; + fetch_default_branch(repo_root, default_branch, git_env)?; + ensure_no_untracked_overwrite_conflicts(repo_root, default_branch, git_env)?; + ensure_fast_forward_possible(repo_root, default_branch, git_env)?; + + Ok(()) +} + +fn ensure_clean_default_branch_worktree( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let status = + run_git_capture(repo_root, &["status", "--porcelain", "--untracked-files=no"], git_env)?; + + if status.is_empty() { + return Ok(()); + } + + eyre::bail!( + "Configured repo root `{}` has tracked local changes; landing closeout cannot fast-forward local `{default_branch}` until they are cleared.", + repo_root.display() + ); +} + +fn fetch_default_branch( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let refspec = format!("refs/heads/{default_branch}:refs/remotes/origin/{default_branch}"); + + run_git_checked( + repo_root, + &["fetch", "origin", refspec.as_str()], + format!("fetch the latest `{default_branch}` from `origin`"), + git_env, + ) +} + +fn ensure_no_untracked_overwrite_conflicts( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let untracked_files = + run_git_capture(repo_root, &["ls-files", "--others", "--exclude-standard"], git_env)?; + + if untracked_files.is_empty() { + return Ok(()); + } + + let tracking_ref = format!("refs/remotes/origin/{default_branch}"); + let incoming_paths = run_git_capture( + repo_root, + &["diff", "--name-only", "--diff-filter=ACMRTUXB", "HEAD", tracking_ref.as_str()], + git_env, + )?; + + if incoming_paths.is_empty() { + return Ok(()); + } + + let incoming_paths = incoming_paths.lines().collect::>(); + let conflicting_paths = untracked_files + .lines() + .filter(|untracked_path| { + incoming_paths.iter().any(|incoming_path| paths_conflict(untracked_path, incoming_path)) + }) + .map(str::to_owned) + .collect::>(); + + if conflicting_paths.is_empty() { + return Ok(()); + } + + eyre::bail!( + "Configured repo root `{}` has untracked local files that would be overwritten by fast-forwarding `{default_branch}`: {}.", + repo_root.display(), + conflicting_paths.join(", ") + ); +} + +fn paths_conflict(left: &str, right: &str) -> bool { + left == right + || left.strip_prefix(right).is_some_and(|suffix| suffix.starts_with('/')) + || right.strip_prefix(left).is_some_and(|suffix| suffix.starts_with('/')) +} + +fn ensure_fast_forward_possible( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let tracking_ref = format!("refs/remotes/origin/{default_branch}"); + let status = build_git_command( + repo_root, + &["merge-base", "--is-ancestor", "HEAD", tracking_ref.as_str()], + git_env, + ) + .status()?; + + if status.success() { + return Ok(()); + } + if status.code() == Some(1) { + eyre::bail!( + "Configured repo root `{}` cannot fast-forward local `{default_branch}` to `{tracking_ref}` because local `{default_branch}` contains commits that are not on origin.", + repo_root.display() + ); + } + + eyre::bail!( + "`git merge-base --is-ancestor HEAD {tracking_ref}` failed in `{}` with status `{}`.", + repo_root.display(), + status + ); +} + +fn fast_forward_default_branch( + repo_root: &Path, + default_branch: &str, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let tracking_ref = format!("refs/remotes/origin/{default_branch}"); + + run_git_checked( + repo_root, + &["merge", "--ff-only", tracking_ref.as_str()], + format!("fast-forward local `{default_branch}` to `{tracking_ref}`"), + git_env, + ) +} + +fn run_git_capture( + cwd: &Path, + args: &[&str], + git_env: &GitCredentialEnvironment, +) -> Result { + let output = build_git_command(cwd, args, git_env).output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + eyre::bail!("`git {}` failed in `{}`: {detail}", args.join(" "), cwd.display()); + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn run_git_checked( + cwd: &Path, + args: &[&str], + action: String, + git_env: &GitCredentialEnvironment, +) -> Result<()> { + let output = build_git_command(cwd, args, git_env).output()?; + + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + if detail.is_empty() { + eyre::bail!("Failed to {action} in `{}`.", cwd.display()); + } + + eyre::bail!("Failed to {action} in `{}`: {detail}", cwd.display()); +} + +fn build_git_command(cwd: &Path, args: &[&str], git_env: &GitCredentialEnvironment) -> Command { + let mut command = Command::new("git"); + + git_env.apply_to(&mut command); + command.arg("-C").arg(cwd).args(args); + + command +} + +fn materialize_git_credentials( + credentials: Option>, +) -> Result<(GitCredentialEnvironment, Option)> { + let Some(credentials) = credentials else { + return Ok((GitCredentialEnvironment::default(), None)); + }; + let (git_env, askpass_guard) = credentials.materialize_github_askpass("default-branch-sync")?; + + Ok((git_env, Some(askpass_guard))) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, + process::Command, + }; + + use tempfile::TempDir; + + use crate::{default_branch_sync, git_credentials::GitCredentialEnvironment, state}; + + #[test] + fn sync_repo_root_default_branch_fast_forwards_local_main() { + let (_temp_dir, repo_root, remote_root) = init_repo(); + let peer_root = clone_repo(&remote_root, "peer"); + + fs::write(peer_root.join("README.md"), "seed\nremote update\n") + .expect("peer update should write"); + + run_git(&peer_root, &["add", "README.md"]); + run_git(&peer_root, &["commit", "-m", "remote update"]); + run_git(&peer_root, &["push", "origin", "main"]); + + let before = git_stdout(&repo_root, &["rev-parse", "HEAD"]); + + default_branch_sync::sync_repo_root_default_branch(&repo_root, "main", None) + .expect("repo root main should fast-forward"); + + let after = git_stdout(&repo_root, &["rev-parse", "HEAD"]); + let remote = git_stdout(&repo_root, &["rev-parse", "refs/remotes/origin/main"]); + + assert_ne!(before, after, "sync should advance local main"); + assert_eq!(after, remote, "local main should match origin/main after sync"); + } + + #[test] + fn sync_repo_root_default_branch_rejects_non_default_branch_checkout() { + let (_temp_dir, repo_root, _remote_root) = init_repo(); + + run_git(&repo_root, &["checkout", "-b", "feature"]); + + let error = default_branch_sync::sync_repo_root_default_branch(&repo_root, "main", None) + .expect_err("non-default repo root branch should be rejected"); + + assert!(error.to_string().contains("is on branch `feature`")); + assert!(error.to_string().contains("fast-forward local `main`")); + } + + #[test] + fn sync_repo_root_default_branch_rejects_tracked_local_changes() { + let (_temp_dir, repo_root, _remote_root) = init_repo(); + + fs::write(repo_root.join("README.md"), "dirty\n").expect("tracked change should write"); + + let error = default_branch_sync::sync_repo_root_default_branch(&repo_root, "main", None) + .expect_err("tracked dirty repo root should be rejected"); + + assert!(error.to_string().contains("tracked local changes")); + } + + #[test] + fn preflight_repo_root_default_branch_sync_rejects_local_commits_not_on_origin() { + let (_temp_dir, repo_root, _remote_root) = init_repo(); + + fs::write(repo_root.join("README.md"), "seed\nlocal-only\n") + .expect("local-only update should write"); + + run_git(&repo_root, &["add", "README.md"]); + run_git(&repo_root, &["commit", "-m", "local only"]); + + let error = + default_branch_sync::preflight_repo_root_default_branch_sync(&repo_root, "main", None) + .expect_err("local-only commits should block ff-only preflight"); + + assert!(error.to_string().contains("cannot fast-forward local `main`")); + assert!(error.to_string().contains("not on origin")); + } + + #[test] + fn preflight_repo_root_default_branch_sync_accepts_clean_default_branch_checkout() { + let (_temp_dir, repo_root, _remote_root) = init_repo(); + + default_branch_sync::preflight_repo_root_default_branch_sync(&repo_root, "main", None) + .expect("clean repo root on the default branch should pass preflight"); + } + + #[test] + fn preflight_repo_root_default_branch_sync_accepts_untracked_decodex_runtime_markers() { + let (_temp_dir, repo_root, _remote_root) = init_repo(); + + fs::write(repo_root.join(state::RUN_ACTIVITY_MARKER_FILE), "runtime marker\n") + .expect("runtime marker should write"); + default_branch_sync::preflight_repo_root_default_branch_sync(&repo_root, "main", None) + .expect("untracked Decodex activity marker should not block clean-source preflight"); + } + + #[test] + fn default_branch_git_commands_use_routed_noninteractive_credentials() { + let git_env = GitCredentialEnvironment::with_github_credentials( + String::from("GITHUB_PAT_Y"), + String::from("ghp_test_token"), + PathBuf::from("/tmp/decodex-default-branch-askpass.sh"), + ); + let command = default_branch_sync::build_git_command( + Path::new("/repo"), + &["fetch", "origin", "refs/heads/main:refs/remotes/origin/main"], + &git_env, + ); + let args = + command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect::>(); + let envs = command + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .collect::>(); + + assert_eq!( + args, + ["-C", "/repo", "fetch", "origin", "refs/heads/main:refs/remotes/origin/main"] + ); + assert_eq!(envs.get("GH_TOKEN").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GITHUB_TOKEN").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GITHUB_PAT_Y").map(String::as_str), Some("ghp_test_token")); + assert_eq!(envs.get("GH_PROMPT_DISABLED").map(String::as_str), Some("1")); + assert_eq!(envs.get("GIT_TERMINAL_PROMPT").map(String::as_str), Some("0")); + assert_eq!(envs.get("GCM_INTERACTIVE").map(String::as_str), Some("never")); + assert_eq!( + envs.get("GIT_ASKPASS").map(String::as_str), + Some("/tmp/decodex-default-branch-askpass.sh") + ); + assert_eq!(envs.get("GIT_CONFIG_COUNT").map(String::as_str), Some("9")); + assert_eq!( + envs.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("url.https://github.com/.insteadOf") + ); + assert_eq!(envs.get("GIT_CONFIG_VALUE_0").map(String::as_str), Some("git@github.com:")); + assert_eq!(envs.get("GIT_CONFIG_KEY_6").map(String::as_str), Some("commit.gpgsign")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_6").map(String::as_str), Some("false")); + assert_eq!(envs.get("GIT_CONFIG_KEY_7").map(String::as_str), Some("tag.gpgsign")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_7").map(String::as_str), Some("false")); + assert_eq!(envs.get("GIT_CONFIG_KEY_8").map(String::as_str), Some("user.signingkey")); + assert_eq!(envs.get("GIT_CONFIG_VALUE_8").map(String::as_str), Some("")); + } + + #[test] + fn preflight_repo_root_default_branch_sync_rejects_untracked_overwrite_conflicts() { + let (_temp_dir, repo_root, remote_root) = init_repo(); + let peer_root = clone_repo(&remote_root, "peer"); + + fs::write(peer_root.join("conflict.txt"), "remote tracked file\n") + .expect("peer conflict file should write"); + + run_git(&peer_root, &["add", "conflict.txt"]); + run_git(&peer_root, &["commit", "-m", "add conflict file"]); + run_git(&peer_root, &["push", "origin", "main"]); + + fs::write(repo_root.join("conflict.txt"), "local untracked file\n") + .expect("repo-root untracked conflict file should write"); + + let error = + default_branch_sync::preflight_repo_root_default_branch_sync(&repo_root, "main", None) + .expect_err("incoming tracked paths must not overwrite local untracked files"); + + assert!(error.to_string().contains("untracked local files")); + assert!(error.to_string().contains("conflict.txt")); + } + + #[test] + fn preflight_repo_root_default_branch_sync_rejects_untracked_path_prefix_conflicts() { + let (_temp_dir, repo_root, remote_root) = init_repo(); + let peer_root = clone_repo(&remote_root, "peer"); + + fs::create_dir_all(peer_root.join("docs")).expect("peer nested directory should exist"); + fs::write(peer_root.join("docs/guide.md"), "remote tracked file\n") + .expect("peer nested file should write"); + + run_git(&peer_root, &["add", "docs/guide.md"]); + run_git(&peer_root, &["commit", "-m", "add nested file"]); + run_git(&peer_root, &["push", "origin", "main"]); + + fs::write(repo_root.join("docs"), "local untracked file\n") + .expect("repo-root conflicting untracked file should write"); + + let error = + default_branch_sync::preflight_repo_root_default_branch_sync(&repo_root, "main", None) + .expect_err( + "incoming tracked directories must not overwrite local untracked files", + ); + + assert!(error.to_string().contains("untracked local files")); + assert!(error.to_string().contains("docs")); + } + + fn init_repo() -> (TempDir, PathBuf, PathBuf) { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("repo"); + let remote_root = temp_dir.path().join("origin.git"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + + run_git( + temp_dir.path(), + &[ + "init", + "--bare", + "--initial-branch", + "main", + remote_root.to_str().expect("remote path utf-8"), + ], + ); + run_git(&repo_root, &["init", "--initial-branch", "main"]); + run_git(&repo_root, &["config", "user.name", "Decodex Tests"]); + run_git(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git(&repo_root, &["config", "commit.gpgsign", "false"]); + run_git(&repo_root, &["config", "tag.gpgsign", "false"]); + run_git( + &repo_root, + &["remote", "add", "origin", remote_root.to_str().expect("remote path utf-8")], + ); + + fs::write(repo_root.join("README.md"), "seed\n").expect("seed file should write"); + + run_git(&repo_root, &["add", "README.md"]); + run_git(&repo_root, &["commit", "-m", "seed"]); + run_git(&repo_root, &["push", "-u", "origin", "main"]); + + (temp_dir, repo_root, remote_root) + } + + fn clone_repo(remote_root: &Path, name: &str) -> PathBuf { + let clone_root = remote_root.parent().expect("remote should have parent").join(name); + + run_git( + remote_root.parent().expect("remote should have parent"), + &[ + "clone", + remote_root.to_str().expect("remote path utf-8"), + clone_root.to_str().expect("clone path utf-8"), + ], + ); + run_git(&clone_root, &["config", "user.name", "Decodex Tests"]); + run_git(&clone_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git(&clone_root, &["config", "commit.gpgsign", "false"]); + + clone_root + } + + fn git_stdout(cwd: &Path, args: &[&str]) -> String { + let output = + Command::new("git").arg("-C").arg(cwd).args(args).output().expect("git should run"); + + assert!( + output.status.success(), + "git {:?} failed in `{}`: {}", + args, + cwd.display(), + String::from_utf8_lossy(&output.stderr) + ); + + String::from_utf8_lossy(&output.stdout).trim().to_owned() + } + + fn run_git(cwd: &Path, args: &[&str]) { + let status = + Command::new("git").arg("-C").arg(cwd).args(args).status().expect("git should run"); + + assert!(status.success(), "git {:?} should succeed in `{}`", args, cwd.display()); + } +} diff --git a/apps/decodex/src/git_credentials.rs b/apps/decodex/src/git_credentials.rs new file mode 100644 index 00000000..c4da583c --- /dev/null +++ b/apps/decodex/src/git_credentials.rs @@ -0,0 +1,242 @@ +#[cfg(unix)] use std::os::unix::fs::PermissionsExt as _; +use std::{ + fs, + io::ErrorKind, + path::{Path, PathBuf}, + process::{self, Command}, + sync::atomic::{AtomicU64, Ordering}, +}; + +use crate::prelude::{Result, eyre}; + +const GITHUB_HTTPS_URL_BASE: &str = "https://github.com/"; +const GITHUB_SSH_URL_PREFIXES: &[&str] = &[ + "git@github.com:", + "git@github.com-x:", + "git@github.com-y:", + "ssh://git@github.com/", + "ssh://git@github.com-x/", + "ssh://git@github.com-y/", +]; +static NEXT_ASKPASS_ID: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy)] +pub(crate) struct GitCredentialSource<'a> { + token_env_var: &'a str, + token: &'a str, + askpass_root: &'a Path, +} +impl<'a> GitCredentialSource<'a> { + pub(crate) fn new(token_env_var: &'a str, token: &'a str, askpass_root: &'a Path) -> Self { + Self { token_env_var, token, askpass_root } + } + + pub(crate) fn materialize_github_askpass( + self, + label: &str, + ) -> Result<(GitCredentialEnvironment, GitAskpassGuard)> { + let askpass_path = scoped_github_askpass_path(self.askpass_root, label); + let askpass_guard = GitAskpassGuard::create(askpass_path.clone())?; + let git_env = GitCredentialEnvironment::with_github_credentials( + self.token_env_var.to_owned(), + self.token.to_owned(), + askpass_path, + ); + + Ok((git_env, askpass_guard)) + } +} + +#[derive(Clone, Default, Eq, PartialEq)] +pub(crate) enum GitSigningConfig { + #[default] + Preserve, + DisableInherited, + SigningKey(String), +} +impl GitSigningConfig { + pub(crate) fn from_local_git_config(repo_root: &Path) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["config", "--local", "--includes", "--get", "user.signingkey"]) + .output()?; + + if output.status.success() { + let signing_key = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + + return if signing_key.is_empty() { + Ok(Self::DisableInherited) + } else { + Ok(Self::SigningKey(signing_key)) + }; + } + if output.status.code() == Some(1) { + return Ok(Self::Preserve); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect local Git signing key in `{}`: {}", + repo_root.display(), + stderr.trim() + ); + } +} + +#[derive(Clone, Default, Eq, PartialEq)] +pub(crate) struct GitCredentialEnvironment { + github_token_env_var: Option, + github_token: Option, + git_askpass_path: Option, + signing_config: GitSigningConfig, +} +impl GitCredentialEnvironment { + pub(crate) fn with_github_credentials( + github_token_env_var: String, + github_token: String, + git_askpass_path: PathBuf, + ) -> Self { + Self { + github_token_env_var: Some(github_token_env_var), + github_token: Some(github_token), + git_askpass_path: Some(git_askpass_path), + signing_config: GitSigningConfig::DisableInherited, + } + } + + pub(crate) fn with_github_credentials_and_signing_config( + github_token_env_var: String, + github_token: String, + git_askpass_path: PathBuf, + signing_config: GitSigningConfig, + ) -> Self { + Self { + github_token_env_var: Some(github_token_env_var), + github_token: Some(github_token), + git_askpass_path: Some(git_askpass_path), + signing_config, + } + } + + pub(crate) fn apply_to(&self, command: &mut Command) { + command + .env("GH_PROMPT_DISABLED", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GCM_INTERACTIVE", "never"); + + if let Some(github_token) = self.github_token.as_deref() { + command.env("GH_TOKEN", github_token).env("GITHUB_TOKEN", github_token); + + if let Some(github_token_env_var) = self.github_token_env_var.as_deref() { + command.env(github_token_env_var, github_token); + } + } + if let Some(git_askpass_path) = self.git_askpass_path.as_deref() { + command.env("GIT_ASKPASS", git_askpass_path); + } + + let mut git_config_entries = Vec::new(); + + if self.github_token.is_some() && self.git_askpass_path.is_some() { + for ssh_prefix in GITHUB_SSH_URL_PREFIXES { + git_config_entries.push(( + format!("url.{GITHUB_HTTPS_URL_BASE}.insteadOf"), + (*ssh_prefix).to_owned(), + )); + } + } + + match &self.signing_config { + GitSigningConfig::Preserve => {}, + GitSigningConfig::DisableInherited => { + git_config_entries.push((String::from("commit.gpgsign"), String::from("false"))); + git_config_entries.push((String::from("tag.gpgsign"), String::from("false"))); + git_config_entries.push((String::from("user.signingkey"), String::new())); + }, + GitSigningConfig::SigningKey(signing_key) => { + git_config_entries.push((String::from("user.signingkey"), signing_key.clone())); + }, + } + + if !git_config_entries.is_empty() { + let git_config_count = git_config_entries.len(); + + for (index, (key, value)) in git_config_entries.into_iter().enumerate() { + command.env(format!("GIT_CONFIG_KEY_{index}"), key); + command.env(format!("GIT_CONFIG_VALUE_{index}"), value); + } + + command.env("GIT_CONFIG_COUNT", git_config_count.to_string()); + } + } +} + +pub(crate) struct GitAskpassGuard { + path: PathBuf, +} +impl GitAskpassGuard { + pub(crate) fn create(path: PathBuf) -> Result { + write_github_askpass_helper(&path)?; + + Ok(Self { path }) + } +} +impl Drop for GitAskpassGuard { + fn drop(&mut self) { + if let Err(error) = fs::remove_file(&self.path) + && error.kind() != ErrorKind::NotFound + { + tracing::warn!( + ?error, + askpass_path = %self.path.display(), + "Failed to remove Git askpass helper." + ); + } + } +} + +pub(crate) fn scoped_github_askpass_path(root: &Path, label: &str) -> PathBuf { + let safe_label = sanitize_path_component(label); + let id = NEXT_ASKPASS_ID.fetch_add(1, Ordering::Relaxed); + + root.join(format!(".decodex-git-askpass-{safe_label}-{}-{id}.sh", process::id())) +} + +pub(crate) fn write_github_askpass_helper(path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + fs::write( + path, + "#!/bin/sh\ncase \"$1\" in\n *https://github.com:*|*https://github.com/*|*https://github.com\\'*|*https://*@github.com:*|*https://*@github.com/*|*https://*@github.com\\'*) ;;\n *) exit 1 ;;\nesac\ncase \"$1\" in\n *Username*|*username*) printf '%s\\n' 'x-access-token' ;;\n *Password*|*password*) printf '%s\\n' \"$GH_TOKEN\" ;;\n *) exit 1 ;;\nesac\n", + )?; + + #[cfg(unix)] + { + let mut permissions = fs::metadata(path)?.permissions(); + + permissions.set_mode(0o700); + + fs::set_permissions(path, permissions)?; + } + + Ok(()) +} + +fn sanitize_path_component(value: &str) -> String { + let sanitized = value + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' || character == '_' { + character + } else { + '_' + } + }) + .collect::(); + + if sanitized.is_empty() { String::from("git") } else { sanitized } +} diff --git a/apps/decodex/src/github.rs b/apps/decodex/src/github.rs new file mode 100644 index 00000000..10167657 --- /dev/null +++ b/apps/decodex/src/github.rs @@ -0,0 +1,968 @@ +use std::{ + path::Path, + process::{Command, Output}, + thread, + time::{Duration, Instant}, +}; + +use serde::Deserialize; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + prelude::{Result, eyre}, + pull_request::PullRequestLandingState, +}; + +const PULL_REQUEST_LANDING_STATE_QUERY: &str = r#" +query($owner: String!, $name: String!, $number: Int!, $reviewThreadsAfter: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + url + state + isDraft + reviewDecision + baseRefName + mergeable + mergeStateStatus + headRefName + headRefOid + reviewRequests(first: 1) { + totalCount + } + reviewThreads(first: 100, after: $reviewThreadsAfter) { + nodes { + isResolved + isOutdated + } + pageInfo { + hasNextPage + endCursor + } + } + commits(last: 1) { + nodes { + commit { + statusCheckRollup { + state + } + } + } + } + } + } +} +"#; + +#[derive(Debug)] +pub(crate) struct PullRequestLocator { + pub(crate) owner: String, + pub(crate) repo: String, + pub(crate) number: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepositoryContext { + pub(crate) owner: String, + pub(crate) name: String, + pub(crate) default_branch: String, + pub(crate) merge_commit_allowed: bool, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct IssueCommentCreateResponse { + pub(crate) id: i64, + #[serde(rename = "created_at")] + pub(crate) created_at: String, +} + +#[derive(Debug, Deserialize)] +struct PullRequestLandingStateResponse { + data: PullRequestLandingStateData, +} + +#[derive(Debug, Deserialize)] +struct PullRequestLandingStateData { + repository: Option, +} + +#[derive(Debug, Deserialize)] +struct PullRequestLandingStateRepository { + #[serde(rename = "pullRequest")] + pull_request: Option, +} + +#[derive(Debug, Deserialize)] +struct PullRequestLandingStateNode { + url: String, + state: String, + #[serde(rename = "isDraft")] + is_draft: bool, + #[serde(rename = "reviewDecision")] + review_decision: Option, + #[serde(rename = "baseRefName")] + base_ref_name: String, + #[serde(rename = "mergeable")] + mergeable: String, + #[serde(rename = "mergeStateStatus")] + merge_state_status: String, + #[serde(rename = "headRefName")] + head_ref_name: String, + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "reviewRequests")] + review_requests: PullRequestReviewRequestConnection, + #[serde(rename = "reviewThreads")] + review_threads: PullRequestReviewThreadConnection, + commits: PullRequestCommitConnection, +} + +#[derive(Debug, Deserialize)] +struct PullRequestReviewRequestConnection { + #[serde(rename = "totalCount")] + total_count: usize, +} + +#[derive(Debug, Deserialize)] +struct PullRequestReviewThreadConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PullRequestPageInfo, +} + +#[derive(Debug, Deserialize)] +struct PullRequestReviewThreadNode { + #[serde(rename = "isResolved")] + is_resolved: bool, + #[serde(rename = "isOutdated")] + is_outdated: bool, +} + +#[derive(Debug, Deserialize)] +struct PullRequestPageInfo { + #[serde(rename = "hasNextPage")] + has_next_page: bool, + #[serde(rename = "endCursor")] + end_cursor: Option, +} + +#[derive(Debug, Deserialize)] +struct PullRequestCommitConnection { + nodes: Vec, +} + +#[derive(Debug, Deserialize)] +struct PullRequestCommitNode { + commit: PullRequestCommitPayload, +} + +#[derive(Debug, Deserialize)] +struct PullRequestCommitPayload { + #[serde(rename = "statusCheckRollup")] + status_check_rollup: Option, +} + +#[derive(Debug, Deserialize)] +struct PullRequestStatusCheckRollup { + state: String, +} + +#[derive(Debug, Deserialize)] +struct RepositoryViewResponse { + name: String, + owner: RepositoryViewOwner, + #[serde(rename = "defaultBranchRef")] + default_branch_ref: RepositoryViewBranchRef, + #[serde(rename = "mergeCommitAllowed")] + merge_commit_allowed: bool, +} + +#[derive(Debug, Deserialize)] +struct RepositoryViewOwner { + login: String, +} + +#[derive(Debug, Deserialize)] +struct RepositoryViewBranchRef { + name: String, +} + +#[derive(Debug, Deserialize)] +struct PullRequestMergeViewResponse { + state: String, + #[serde(rename = "headRefOid")] + head_ref_oid: Option, + #[serde(rename = "mergeCommit")] + merge_commit: Option, +} + +#[derive(Debug, Deserialize)] +struct PullRequestMergeCommit { + oid: String, +} + +#[derive(Debug, Deserialize)] +struct CommitViewResponse { + commit: CommitViewCommit, +} + +#[derive(Debug, Deserialize)] +struct CommitViewCommit { + message: String, +} + +pub(crate) fn configure_gh_command(command: &mut Command, github_token: &str) { + command + .env("GH_TOKEN", github_token) + .env("GITHUB_TOKEN", github_token) + .env("GH_PROMPT_DISABLED", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GCM_INTERACTIVE", "never"); +} + +pub(crate) fn parse_pull_request_url(pr_url: &str) -> Result { + let normalized = pr_url.trim().trim_end_matches('/'); + let suffix = normalized.strip_prefix("https://github.com/").ok_or_else(|| { + eyre::eyre!("Pull request URL `{pr_url}` must start with `https://github.com/`.") + })?; + let mut segments = suffix.split('/'); + let owner = segments + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| eyre::eyre!("Pull request URL `{pr_url}` is missing the owner."))?; + let repo = segments + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| eyre::eyre!("Pull request URL `{pr_url}` is missing the repository."))?; + let pull_segment = segments + .next() + .ok_or_else(|| eyre::eyre!("Pull request URL `{pr_url}` is missing the `pull` segment."))?; + + if pull_segment != "pull" { + eyre::bail!( + "Pull request URL `{pr_url}` must use `/pull/`, not `/{pull_segment}`." + ); + } + + let number = segments + .next() + .ok_or_else(|| { + eyre::eyre!("Pull request URL `{pr_url}` is missing the pull request number.") + })? + .parse::() + .map_err(|error| { + eyre::eyre!("Pull request URL `{pr_url}` has an invalid number: {error}") + })?; + + Ok(PullRequestLocator { owner: owner.to_owned(), repo: repo.to_owned(), number }) +} + +pub(crate) fn post_pull_request_issue_comment( + cwd: &Path, + pr_url: &str, + body: &str, + github_token: &str, +) -> Result<(i64, i64)> { + let locator = parse_pull_request_url(pr_url)?; + let endpoint = + format!("repos/{}/{}/issues/{}/comments", locator.owner, locator.repo, locator.number); + let mut command = Command::new("gh"); + + command.args(["api", endpoint.as_str(), "-f", &format!("body={body}")]); + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to post pull request comment on `{pr_url}`: {}", stderr.trim()); + } + + let response = serde_json::from_slice::(&output.stdout)?; + let created_at_unix_epoch = OffsetDateTime::parse(&response.created_at, &Rfc3339) + .map_err(|error| { + eyre::eyre!( + "Failed to parse GitHub comment timestamp `{}` for `{pr_url}`: {error}", + response.created_at + ) + })? + .unix_timestamp(); + + Ok((response.id, created_at_unix_epoch)) +} + +pub(crate) fn inspect_pull_request_landing_state( + cwd: &Path, + pr_url: &str, + github_token: &str, +) -> Result { + let locator = parse_pull_request_url(pr_url)?; + let mut review_threads_after: Option = None; + let mut landing_state: Option = None; + + loop { + let pull_request = query_pull_request_landing_state_page( + cwd, + &locator.owner, + &locator.repo, + locator.number, + review_threads_after.as_deref(), + pr_url, + github_token, + )?; + let next_cursor = match &mut landing_state { + Some(landing_state) => + merge_pull_request_landing_state_page(landing_state, &pull_request)?, + None => { + let next_cursor = next_pull_request_review_threads_cursor(&pull_request, pr_url)?; + + landing_state = Some(pull_request_landing_state_from_page(&pull_request)); + + next_cursor + }, + }; + let Some(next_cursor) = next_cursor else { + break; + }; + + review_threads_after = Some(next_cursor); + } + + landing_state.ok_or_else(|| { + eyre::eyre!("GitHub GraphQL response for `{pr_url}` did not include a pull request.") + }) +} + +pub(crate) fn inspect_repository_context( + cwd: &Path, + github_token: &str, +) -> Result { + let mut command = Command::new("gh"); + + command.args(["repo", "view", "--json", "name,owner,defaultBranchRef,mergeCommitAllowed"]); + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to inspect current GitHub repository context: {}", stderr.trim()); + } + + let response = serde_json::from_slice::(&output.stdout)?; + + Ok(RepositoryContext { + owner: response.owner.login, + name: response.name, + default_branch: response.default_branch_ref.name, + merge_commit_allowed: response.merge_commit_allowed, + }) +} + +pub(crate) fn pull_request_matches_repository( + pr_url: &str, + repository: &RepositoryContext, +) -> Result { + let locator = parse_pull_request_url(pr_url)?; + + Ok(locator.owner.eq_ignore_ascii_case(&repository.owner) + && locator.repo.eq_ignore_ascii_case(&repository.name)) +} + +pub(crate) fn delete_pull_request_head_branch_if_present( + cwd: &Path, + pr_url: &str, + branch_name: &str, + github_token: &str, +) -> Result<()> { + let locator = parse_pull_request_url(pr_url)?; + + delete_repository_branch_if_present( + cwd, + &locator.owner, + &locator.repo, + branch_name, + github_token, + ) +} + +pub(crate) fn admin_merge_pull_request( + cwd: &Path, + pr_url: &str, + reviewed_head_sha: &str, + merge_subject: Option<&str>, + github_token: &str, +) -> Result<()> { + let mut command = Command::new("gh"); + + configure_admin_merge_command(&mut command, pr_url, reviewed_head_sha, merge_subject); + + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + if detail.is_empty() { + eyre::bail!("Failed to admin-merge `{pr_url}`."); + } + + eyre::bail!("Failed to admin-merge `{pr_url}`: {detail}"); +} + +pub(crate) fn inspect_pull_request_merge_commit( + cwd: &Path, + pr_url: &str, + github_token: &str, +) -> Result { + let response = inspect_pull_request_merge_response(cwd, pr_url, github_token)?; + + if response.state != "MERGED" { + eyre::bail!("Pull request `{pr_url}` did not reach `MERGED` state after landing."); + } + + let Some(merge_commit) = response.merge_commit else { + eyre::bail!("Pull request `{pr_url}` does not expose a merge commit after merge."); + }; + + Ok(merge_commit.oid) +} + +pub(crate) fn wait_for_pull_request_merge_commit( + cwd: &Path, + pr_url: &str, + github_token: &str, + timeout: Duration, +) -> Result { + let deadline = Instant::now() + timeout; + + loop { + match inspect_pull_request_merge_commit(cwd, pr_url, github_token) { + Ok(merge_commit) => return Ok(merge_commit), + Err(error) if Instant::now() >= deadline => return Err(error), + Err(_error) => {}, + }; + + thread::sleep(Duration::from_secs(1)); + } +} + +pub(crate) fn inspect_commit_subject( + cwd: &Path, + pr_url: &str, + commit_oid: &str, + github_token: &str, +) -> Result { + let locator = parse_pull_request_url(pr_url)?; + let mut command = Command::new("gh"); + + command + .args(["api", &format!("repos/{}/{}/commits/{}", locator.owner, locator.repo, commit_oid)]); + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect merge commit `{commit_oid}` for `{pr_url}`: {}", + stderr.trim() + ); + } + + let response = serde_json::from_slice::(&output.stdout)?; + let subject = response + .commit + .message + .lines() + .next() + .map(|line| line.trim_end_matches('\r')) + .unwrap_or_default(); + + if subject.is_empty() { + eyre::bail!("Merge commit `{commit_oid}` for `{pr_url}` does not expose a subject line."); + } + + Ok(subject.to_owned()) +} + +pub(crate) fn wait_for_commit_subject( + cwd: &Path, + pr_url: &str, + commit_oid: &str, + github_token: &str, + timeout: Duration, +) -> Result { + let deadline = Instant::now() + timeout; + + loop { + match inspect_commit_subject(cwd, pr_url, commit_oid, github_token) { + Ok(subject) => return Ok(subject), + Err(error) if Instant::now() >= deadline => return Err(error), + Err(_error) => {}, + }; + + thread::sleep(Duration::from_secs(1)); + } +} + +pub(crate) fn pull_request_is_merged_at_head( + cwd: &Path, + pr_url: &str, + expected_head_sha: &str, + github_token: &str, +) -> Result { + let response = inspect_pull_request_merge_response(cwd, pr_url, github_token)?; + + Ok(response.state == "MERGED" && response.head_ref_oid.as_deref() == Some(expected_head_sha)) +} + +fn delete_repository_branch_if_present( + cwd: &Path, + owner: &str, + repo: &str, + branch_name: &str, + github_token: &str, +) -> Result<()> { + if branch_name.trim().is_empty() { + eyre::bail!("Refusing to delete an empty GitHub branch name."); + } + + let endpoint = + format!("repos/{owner}/{repo}/git/refs/heads/{}", github_api_ref_path(branch_name)); + let mut command = Command::new("gh"); + + command.args(["api", "--method", "DELETE", "--silent", endpoint.as_str()]); + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if output.status.success() || gh_delete_ref_missing_branch(&output) { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + eyre::bail!( + "Failed to delete retained remote branch `{branch_name}` from GitHub repository `{owner}/{repo}`: {detail}" + ); +} + +fn inspect_pull_request_merge_response( + cwd: &Path, + pr_url: &str, + github_token: &str, +) -> Result { + let mut command = Command::new("gh"); + + command.args(["pr", "view", pr_url, "--json", "state,headRefOid,mergeCommit"]); + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to inspect merge result for `{pr_url}`: {}", stderr.trim()); + } + + serde_json::from_slice::(&output.stdout).map_err(Into::into) +} + +fn configure_admin_merge_command( + command: &mut Command, + pr_url: &str, + reviewed_head_sha: &str, + merge_subject: Option<&str>, +) { + command.args(["pr", "merge", "--admin", "--merge", "--match-head-commit", reviewed_head_sha]); + + if let Some(merge_subject) = merge_subject { + command.args(["--subject", merge_subject]); + } + + command.args(["--body", ""]); + command.arg(pr_url); +} + +fn gh_delete_ref_missing_branch(output: &Output) -> bool { + let stderr = String::from_utf8_lossy(&output.stderr).to_ascii_lowercase(); + let stdout = String::from_utf8_lossy(&output.stdout).to_ascii_lowercase(); + let combined = format!("{stderr}\n{stdout}"); + + combined.contains("reference does not exist") + || combined.contains("reference not found") + || (combined.contains("http 422") && combined.contains("reference")) +} + +fn github_api_ref_path(ref_name: &str) -> String { + ref_name.split('/').map(github_api_path_component).collect::>().join("/") +} + +fn github_api_path_component(component: &str) -> String { + let mut encoded = String::new(); + + for byte in component.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + + encoded +} + +fn query_pull_request_landing_state_page( + cwd: &Path, + owner: &str, + repo: &str, + number: u64, + review_threads_after: Option<&str>, + pr_url: &str, + github_token: &str, +) -> Result { + let mut command = Command::new("gh"); + + command.args(["api", "graphql", "-f", &format!("query={PULL_REQUEST_LANDING_STATE_QUERY}")]); + command.args(["-F", &format!("owner={owner}")]); + command.args(["-F", &format!("name={repo}")]); + command.args(["-F", &format!("number={number}")]); + + if let Some(review_threads_after) = review_threads_after { + command.args(["-F", &format!("reviewThreadsAfter={review_threads_after}")]); + } + + command.current_dir(cwd); + + configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to inspect pull request landing state `{pr_url}`: {}", stderr.trim()); + } + + let response = serde_json::from_slice::(&output.stdout)?; + let Some(repository) = response.data.repository else { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a repository."); + }; + let Some(pull_request) = repository.pull_request else { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a pull request."); + }; + + Ok(pull_request) +} + +fn pull_request_landing_state_from_page( + pull_request: &PullRequestLandingStateNode, +) -> PullRequestLandingState { + PullRequestLandingState { + url: pull_request.url.clone(), + state: pull_request.state.clone(), + is_draft: pull_request.is_draft, + review_decision: pull_request.review_decision.clone(), + base_ref_name: pull_request.base_ref_name.clone(), + pending_review_requests: pull_request.review_requests.total_count, + mergeable: pull_request.mergeable.clone(), + merge_state_status: pull_request.merge_state_status.clone(), + head_ref_name: pull_request.head_ref_name.clone(), + head_ref_oid: pull_request.head_ref_oid.clone(), + status_check_rollup_state: pull_request + .commits + .nodes + .first() + .and_then(|node| node.commit.status_check_rollup.as_ref()) + .map(|rollup| rollup.state.clone()), + unresolved_review_threads: count_unresolved_review_threads(&pull_request.review_threads), + } +} + +fn merge_pull_request_landing_state_page( + landing_state: &mut PullRequestLandingState, + pull_request: &PullRequestLandingStateNode, +) -> Result> { + let page_state = pull_request_landing_state_from_page(pull_request); + + if landing_state.url != page_state.url + || landing_state.state != page_state.state + || landing_state.is_draft != page_state.is_draft + || landing_state.review_decision != page_state.review_decision + || landing_state.base_ref_name != page_state.base_ref_name + || landing_state.pending_review_requests != page_state.pending_review_requests + || landing_state.mergeable != page_state.mergeable + || landing_state.merge_state_status != page_state.merge_state_status + || landing_state.head_ref_name != page_state.head_ref_name + || landing_state.head_ref_oid != page_state.head_ref_oid + || landing_state.status_check_rollup_state != page_state.status_check_rollup_state + { + eyre::bail!("Pull request landing state changed while paginating `{}`.", landing_state.url); + } + + landing_state.unresolved_review_threads += page_state.unresolved_review_threads; + + next_pull_request_review_threads_cursor(pull_request, landing_state.url.as_str()) +} + +fn count_unresolved_review_threads(review_threads: &PullRequestReviewThreadConnection) -> usize { + review_threads.nodes.iter().filter(|thread| !thread.is_resolved && !thread.is_outdated).count() +} + +fn next_pull_request_review_threads_cursor( + pull_request: &PullRequestLandingStateNode, + pr_url: &str, +) -> Result> { + if !pull_request.review_threads.page_info.has_next_page { + return Ok(None); + } + + pull_request + .review_threads + .page_info + .end_cursor + .clone() + .map(Some) + .ok_or_else(|| { + eyre::eyre!( + "GitHub GraphQL response for `{pr_url}` reported additional review thread pages without an end cursor." + ) + }) +} + +#[cfg(test)] +mod tests { + use std::ffi::OsStr; + + #[test] + fn parses_pull_request_url() { + let locator = super::parse_pull_request_url("https://github.com/hack-ink/decodex/pull/20") + .expect("pull request URL should parse"); + + assert_eq!(locator.owner, "hack-ink"); + assert_eq!(locator.repo, "decodex"); + assert_eq!(locator.number, 20); + } + + #[test] + fn rejects_non_pull_github_url() { + let error = super::parse_pull_request_url("https://github.com/hack-ink/decodex/issues/20") + .expect_err("issue URL should be rejected"); + + assert!(error.to_string().contains("/pull/")); + } + + #[test] + fn rejects_missing_number() { + let error = super::parse_pull_request_url("https://github.com/hack-ink/decodex/pull/") + .expect_err("missing pull number should be rejected"); + + assert!(error.to_string().contains("missing the pull request number")); + } + + #[test] + fn configure_gh_command_sets_explicit_token_when_present() { + let mut command = std::process::Command::new("gh"); + + super::configure_gh_command(&mut command, "ghp_example"); + + let envs = command + .get_envs() + .filter_map(|(key, value)| Some((key.to_owned(), value?.to_owned()))) + .collect::>(); + + assert_eq!(envs.get(OsStr::new("GH_TOKEN")), Some(&OsStr::new("ghp_example").to_owned())); + assert_eq!( + envs.get(OsStr::new("GITHUB_TOKEN")), + Some(&OsStr::new("ghp_example").to_owned()) + ); + } + + #[test] + fn configure_gh_command_disables_prompt_for_explicit_token_auth() { + let mut command = std::process::Command::new("gh"); + + super::configure_gh_command(&mut command, "ghp_example"); + + assert!( + command + .get_envs() + .find_map(|(key, value)| (key == OsStr::new("GH_PROMPT_DISABLED")).then_some(value)) + .flatten() + .is_some_and(|value| value == OsStr::new("1")), + "configure_gh_command should disable interactive gh prompts" + ); + assert!( + command + .get_envs() + .find_map(|(key, value)| (key == OsStr::new("GIT_TERMINAL_PROMPT")).then_some(value)) + .flatten() + .is_some_and(|value| value == OsStr::new("0")), + "configure_gh_command should disable interactive git prompts" + ); + assert!( + command + .get_envs() + .find_map(|(key, value)| (key == OsStr::new("GCM_INTERACTIVE")).then_some(value)) + .flatten() + .is_some_and(|value| value == OsStr::new("never")), + "configure_gh_command should disable credential-manager prompts" + ); + } + + #[test] + fn repository_match_rejects_foreign_pull_request_url() { + let repository = super::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }; + + assert!( + !super::pull_request_matches_repository( + "https://github.com/other-org/other-repo/pull/9", + &repository + ) + .expect("foreign pull request URL should parse") + ); + } + + #[test] + fn repository_match_accepts_case_insensitive_pull_request_url() { + let repository = super::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }; + + assert!( + super::pull_request_matches_repository( + "https://github.com/Hack-Ink/Decodex/pull/9", + &repository + ) + .expect("same repository with different casing should parse") + ); + } + + #[test] + fn admin_merge_command_matches_reviewed_head_commit() { + let mut command = std::process::Command::new("gh"); + + super::configure_admin_merge_command( + &mut command, + "https://github.com/hack-ink/decodex/pull/50", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + None, + ); + + let args = + command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect::>(); + + assert_eq!( + args, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"), + String::from("--body"), + String::from(""), + String::from("https://github.com/hack-ink/decodex/pull/50"), + ] + ); + } + + #[test] + fn admin_merge_command_includes_subject_when_provided() { + let mut command = std::process::Command::new("gh"); + + super::configure_admin_merge_command( + &mut command, + "https://github.com/hack-ink/decodex/pull/50", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + Some(r#"{"schema":"decodex/commit/1","summary":"ship fix","authority":"manual"}"#), + ); + + let args = + command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect::>(); + + assert_eq!( + args, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"), + String::from("--subject"), + String::from( + r#"{"schema":"decodex/commit/1","summary":"ship fix","authority":"manual"}"# + ), + String::from("--body"), + String::from(""), + String::from("https://github.com/hack-ink/decodex/pull/50"), + ] + ); + } + + #[test] + fn github_api_ref_path_preserves_ref_slashes_and_encodes_segments() { + assert_eq!(super::github_api_ref_path("y/decodex XY-235"), "y/decodex%20XY-235"); + } + + #[test] + fn missing_remote_ref_errors_are_idempotent_cleanup() { + let output = std::process::Output { + status: std::process::Command::new("sh") + .args(["-c", "exit 1"]) + .status() + .expect("status command should run"), + stdout: Vec::new(), + stderr: b"gh: Reference does not exist (HTTP 422)".to_vec(), + }; + + assert!(super::gh_delete_ref_missing_branch(&output)); + } + + #[test] + fn generic_github_not_found_is_not_idempotent_cleanup() { + let output = std::process::Output { + status: std::process::Command::new("sh") + .args(["-c", "exit 1"]) + .status() + .expect("status command should run"), + stdout: Vec::new(), + stderr: b"gh: Not Found (HTTP 404)".to_vec(), + }; + + assert!(!super::gh_delete_ref_missing_branch(&output)); + } +} diff --git a/apps/decodex/src/lib.rs b/apps/decodex/src/lib.rs new file mode 100644 index 00000000..c9f5fbc2 --- /dev/null +++ b/apps/decodex/src/lib.rs @@ -0,0 +1,82 @@ +//! Decodex runtime bootstrap and CLI entrypoint. + +pub mod config; +pub mod state; +pub mod workflow; + +mod agent; +mod archive_hygiene; +mod cli; +mod commit_message; +mod default_branch_sync; +mod git_credentials; +mod github; +mod manual; +mod orchestrator; +mod pull_request; +mod prelude { + pub use color_eyre::{Result, eyre}; +} +mod runtime; +mod tracker; +mod worktree; + +use std::{fs, panic, process}; + +use clap::Parser; +use tracing_appender::{ + non_blocking::WorkerGuard, + rolling::{RollingFileAppender, Rotation}, +}; +use tracing_subscriber::EnvFilter; + +use crate::{cli::Cli, prelude::Result}; + +#[cfg(not(unix))] +compile_error!("Decodex supports only Unix targets (macOS and Linux). Windows is unsupported."); + +/// Run the Decodex CLI after initializing error reporting, logging, and the panic hook. +pub fn run() -> Result<()> { + color_eyre::install()?; + + let _guard = init_tracing()?; + + install_panic_hook(); + + Cli::parse().run() +} + +fn init_tracing() -> Result { + let log_dir = runtime::log_dir()?; + + fs::create_dir_all(&log_dir)?; + + let (non_blocking, guard) = tracing_appender::non_blocking( + RollingFileAppender::builder() + .rotation(Rotation::WEEKLY) + .max_log_files(3) + .filename_suffix("log") + .build(log_dir)?, + ); + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_ansi(false) + .with_writer(non_blocking) + .init(); + + Ok(guard) +} + +fn install_panic_hook() { + let default_hook = panic::take_hook(); + + panic::set_hook(Box::new(move |panic_info| { + default_hook(panic_info); + + process::abort(); + })); +} + +#[cfg(test)] mod test_support; diff --git a/apps/decodex/src/main.rs b/apps/decodex/src/main.rs new file mode 100644 index 00000000..71d91347 --- /dev/null +++ b/apps/decodex/src/main.rs @@ -0,0 +1,9 @@ +//! Decodex binary entrypoint. + +#![allow(unused_crate_dependencies)] + +use color_eyre::Result; + +fn main() -> Result<()> { + decodex::run() +} diff --git a/apps/decodex/src/manual.rs b/apps/decodex/src/manual.rs new file mode 100644 index 00000000..d85ab4a5 --- /dev/null +++ b/apps/decodex/src/manual.rs @@ -0,0 +1,3004 @@ +use std::{ + env, fs, + io::ErrorKind, + path::{Path, PathBuf}, + process::{Command, Stdio}, + time::Duration, +}; + +use color_eyre::{Report, eyre::WrapErr}; + +use crate::{ + commit_message::{self, MANUAL_AUTHORITY}, + config::{self, ServiceConfig}, + default_branch_sync, + git_credentials::GitCredentialSource, + github::{self, RepositoryContext}, + orchestrator, + prelude::{Result, eyre}, + pull_request::{self, PullRequestLandingState}, + runtime, + state::{RUN_ACTIVITY_MARKER_FILE, ReviewHandoffMarker, StateStore}, + tracker::{self, IssueTracker, TrackerIssue, linear::LinearClient}, + workflow::WorkflowDocument, + worktree::{self, WorktreeManager}, +}; + +const MANUAL_LAND_CLOSEOUT_MARKER_GIT_PATH: &str = "decodex/manual-land-closeout"; +const MANUAL_LAND_MERGE_VISIBILITY_TIMEOUT: Duration = Duration::from_secs(15 * 60); + +#[derive(Debug)] +pub(crate) struct ManualCommitRequest { + pub(crate) summary: String, + pub(crate) authority: Option, + pub(crate) manual_authority: bool, + pub(crate) related: Vec, + pub(crate) breaking: bool, +} + +#[derive(Debug)] +pub(crate) struct ManualLandRequest { + pub(crate) summary: String, + pub(crate) authority: Option, + pub(crate) manual_authority: bool, + pub(crate) pr_url: Option, + pub(crate) related: Vec, + pub(crate) breaking: bool, +} + +struct PreparedCloseout { + tracker: LinearClient, + issue: TrackerIssue, + completed_state: String, + service_id: String, + needs_attention_label: String, +} + +struct ManualLandContext { + cwd: PathBuf, + current_branch: String, + worktree_root: PathBuf, + project_worktree_root: PathBuf, + canonical_repo_root: PathBuf, + authority: ManualAuthority, + service_id: String, + workflow: WorkflowDocument, + github_token_env_var: String, + github_token: String, + repository: RepositoryContext, + prepared_closeout: Option, + pr_url: String, + review_branch: String, +} +impl ManualLandContext { + fn default_branch_git_credentials(&self) -> GitCredentialSource<'_> { + GitCredentialSource::new( + &self.github_token_env_var, + &self.github_token, + &self.worktree_root, + ) + } +} + +struct ManualLandRecoveryOutcome { + merge_commit: String, +} + +#[derive(Default)] +struct ManualLandCloseoutMarkerRecord { + pr_url: Option, + merge_commit: Option, + branch_name: Option, + landed_change: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LandExecutionMode { + MergeAndCloseout, + CloseoutOnly, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum ManualAuthority { + Issue(String), + Manual, +} +impl ManualAuthority { + fn commit_message_value(&self) -> &str { + match self { + Self::Issue(identifier) => identifier.as_str(), + Self::Manual => MANUAL_AUTHORITY, + } + } + + fn issue_identifier(&self) -> Option<&str> { + match self { + Self::Issue(identifier) => Some(identifier.as_str()), + Self::Manual => None, + } + } + + fn is_manual(&self) -> bool { + matches!(self, Self::Manual) + } +} + +pub(crate) fn run_commit(config_path: Option<&Path>, request: &ManualCommitRequest) -> Result<()> { + let cwd = env::current_dir()?; + let worktree_root = current_worktree_root(&cwd)?; + let authority = resolve_authority( + config_path, + request.authority.as_deref(), + request.manual_authority, + &worktree_root, + )?; + let message = commit_message::build_commit_message( + &request.summary, + authority.commit_message_value(), + &request.related, + request.breaking, + )?; + + run_git_checked_with_stdio(&cwd, &["commit", "-S", "-m", message.as_str()]) +} + +pub(crate) fn run_land(config_path: Option<&Path>, request: &ManualLandRequest) -> Result<()> { + let context = prepare_manual_land_context(config_path, request)?; + + if !github::pull_request_matches_repository(&context.pr_url, &context.repository)? { + eyre::bail!( + "Pull request `{}` does not belong to the current repository `{}/{}`.", + context.pr_url, + context.repository.owner, + context.repository.name, + ); + } + + if let Some(recovery) = finalize_already_merged_manual_land_recovery(&context, request)? { + println!( + "land ok: pr={} merge_commit={} default_branch={} local_default_branch_synced=true", + context.pr_url, recovery.merge_commit, context.repository.default_branch + ); + + return Ok(()); + } + + ensure_manual_land_checkout_is_managed_lane( + &context.worktree_root, + &context.project_worktree_root, + manual_land_cleanup_identifier(&context.authority, &context.current_branch), + )?; + + if context.current_branch == context.repository.default_branch { + eyre::bail!("`decodex land` must run from a reviewed lane branch, not the default branch."); + } + if context.review_branch != context.current_branch { + eyre::bail!( + "Review handoff expects branch `{}`, but the current branch is `{}`.", + context.review_branch, + context.current_branch, + ); + } + + let default_branch = context.repository.default_branch.clone(); + let landing_state = github::inspect_pull_request_landing_state( + &context.canonical_repo_root, + &context.pr_url, + &context.github_token, + )?; + let current_head = current_head_oid(&context.cwd)?; + let execution_mode = validate_landing_state( + &landing_state, + &context.pr_url, + &default_branch, + &context.current_branch, + ¤t_head, + )?; + + default_branch_sync::preflight_repo_root_default_branch_sync( + &context.canonical_repo_root, + &default_branch, + Some(context.default_branch_git_credentials()), + )?; + + let landed_change_record = commit_message::build_landing_commit_message( + &request.summary, + context.authority.commit_message_value(), + &request.related, + request.breaking, + )?; + let merge_commit = + execute_land_merge(&context, ¤t_head, landed_change_record.as_str(), execution_mode)?; + let landed_change_record = load_authoritative_landed_change_record(&context, &merge_commit)?; + + finalize_land_closeout( + &context, + &merge_commit, + &default_branch, + landed_change_record.as_str(), + )?; + + println!( + "land ok: pr={} merge_commit={} default_branch={} local_default_branch_synced=true", + context.pr_url, merge_commit, default_branch + ); + + Ok(()) +} + +fn prepare_manual_land_context( + config_path: Option<&Path>, + request: &ManualLandRequest, +) -> Result { + let cwd = env::current_dir()?; + let worktree_root = current_worktree_root(&cwd)?; + let current_branch = current_branch_name(&cwd)?; + let resolved_config_path = resolve_manual_config_path(config_path, &cwd)?; + let config = ServiceConfig::from_path(&resolved_config_path)?; + let canonical_repo_root = config::canonical_repo_root_for_checkout(&cwd)? + .unwrap_or_else(|| config.repo_root().to_path_buf()); + + ensure_cli_repo_context(&cwd, &config, &canonical_repo_root)?; + + let authority = resolve_land_authority( + Some(&resolved_config_path), + request.authority.as_deref(), + request.manual_authority, + &worktree_root, + )?; + let github_token = config.github().resolve_token()?; + let repository = github::inspect_repository_context(&canonical_repo_root, &github_token)?; + let workflow = WorkflowDocument::from_path(config.workflow_path())?; + let prepared_closeout = + prepare_manual_land_closeout(&config, &canonical_repo_root, workflow.clone(), &authority)?; + let state_store = runtime::open_runtime_store()?; + + runtime::register_project_config(&state_store, &resolved_config_path, true)?; + + let handoff = match prepared_closeout.as_ref() { + Some(prepared_closeout) => read_manual_land_handoff( + &state_store, + config.service_id(), + &prepared_closeout.issue.id, + ¤t_branch, + )?, + None => None, + }; + let pr_url = + resolve_pr_url(request.pr_url.as_deref(), handoff.as_ref(), authority.is_manual())?; + let review_branch = handoff + .as_ref() + .map(|marker| marker.branch_name().to_owned()) + .unwrap_or_else(|| current_branch.clone()); + + Ok(ManualLandContext { + cwd, + current_branch, + worktree_root, + project_worktree_root: config.worktree_root().to_path_buf(), + canonical_repo_root, + authority, + service_id: config.service_id().to_owned(), + workflow, + github_token_env_var: config.github().token_env_var().to_owned(), + github_token, + repository, + prepared_closeout, + pr_url, + review_branch, + }) +} + +fn prepare_manual_land_closeout( + config: &ServiceConfig, + _canonical_repo_root: &Path, + workflow: WorkflowDocument, + authority: &ManualAuthority, +) -> Result> { + let Some(authority_issue) = authority.issue_identifier() else { + return Ok(None); + }; + + prepare_closeout(config, workflow, authority_issue).map(Some) +} + +fn execute_land_merge( + context: &ManualLandContext, + current_head: &str, + landed_change_record: &str, + execution_mode: LandExecutionMode, +) -> Result { + match execution_mode { + LandExecutionMode::MergeAndCloseout => { + ensure_clean_worktree(&context.cwd)?; + + if !context.repository.merge_commit_allowed { + eyre::bail!( + "GitHub repository `{}/{}` does not allow merge commits, but `decodex land` requires an admin merge commit.", + context.repository.owner, + context.repository.name + ); + } + + if let Err(error) = github::admin_merge_pull_request( + &context.canonical_repo_root, + &context.pr_url, + current_head, + Some(landed_change_record), + &context.github_token, + ) { + if matches!( + github::pull_request_is_merged_at_head( + &context.canonical_repo_root, + &context.pr_url, + current_head, + &context.github_token, + ), + Ok(true) + ) { + return github::wait_for_pull_request_merge_commit( + &context.canonical_repo_root, + &context.pr_url, + &context.github_token, + MANUAL_LAND_MERGE_VISIBILITY_TIMEOUT, + ); + } + + return Err(error); + } + }, + LandExecutionMode::CloseoutOnly => {}, + } + + github::wait_for_pull_request_merge_commit( + &context.canonical_repo_root, + &context.pr_url, + &context.github_token, + MANUAL_LAND_MERGE_VISIBILITY_TIMEOUT, + ) +} + +fn load_authoritative_landed_change_record( + context: &ManualLandContext, + merge_commit: &str, +) -> Result { + github::wait_for_commit_subject( + &context.canonical_repo_root, + &context.pr_url, + merge_commit, + &context.github_token, + MANUAL_LAND_MERGE_VISIBILITY_TIMEOUT, + ) +} + +fn finalize_land_closeout( + context: &ManualLandContext, + merge_commit: &str, + default_branch: &str, + landed_change_record: &str, +) -> Result<()> { + if let Some(prepared_closeout) = context.prepared_closeout.as_ref() { + apply_closeout( + &context.cwd, + prepared_closeout, + &context.pr_url, + merge_commit, + &context.current_branch, + landed_change_record, + )?; + } + + default_branch_sync::sync_repo_root_default_branch( + &context.canonical_repo_root, + default_branch, + Some(context.default_branch_git_credentials()), + )?; + + if context.prepared_closeout.is_none() + && !manual_land_closeout_matches( + &context.cwd, + &context.pr_url, + merge_commit, + &context.current_branch, + landed_change_record, + )? { + write_manual_land_closeout_marker( + &context.cwd, + &context.pr_url, + merge_commit, + &context.current_branch, + landed_change_record, + )?; + } + + cleanup_manual_land_lane_checkout(context)?; + + if let Some(prepared_closeout) = context.prepared_closeout.as_ref() { + let state_store = runtime::open_runtime_store()?; + + clear_manual_closeout_runtime_state(&state_store, &prepared_closeout.issue.id)?; + clear_manual_closeout_issue_scope( + &prepared_closeout.tracker, + &prepared_closeout.issue, + &prepared_closeout.service_id, + &prepared_closeout.needs_attention_label, + )?; + } + + Ok(()) +} + +fn cleanup_manual_land_lane_checkout(context: &ManualLandContext) -> Result<()> { + let worktree_manager = WorktreeManager::new( + context.service_id.as_str(), + &context.canonical_repo_root, + &context.project_worktree_root, + ); + + github::delete_pull_request_head_branch_if_present( + &context.canonical_repo_root, + &context.pr_url, + &context.current_branch, + &context.github_token, + )?; + orchestrator::detach_worktree_head_from_branch_if_checked_out( + &context.worktree_root, + &context.current_branch, + )?; + orchestrator::delete_local_branch_if_present( + &context.canonical_repo_root, + &context.current_branch, + )?; + + worktree_manager.remove_worktree_path_with_hooks( + manual_land_cleanup_identifier(&context.authority, &context.current_branch), + &context.current_branch, + &context.worktree_root, + context.workflow.frontmatter().execution().workspace_hooks(), + )?; + + ensure_manual_land_left_no_merged_worktree_cleanup_debt(context)?; + + Ok(()) +} + +fn ensure_manual_land_left_no_merged_worktree_cleanup_debt( + context: &ManualLandContext, +) -> Result<()> { + let debts = worktree::merged_worktree_cleanup_debts( + &context.canonical_repo_root, + &context.project_worktree_root, + &context.repository.default_branch, + )?; + + if debts.is_empty() { + return Ok(()); + } + + let details = debts + .iter() + .map(|debt| { + format!( + "{} on {} ({})", + debt.path.display(), + debt.branch_name, + if debt.cleanliness.is_dirty() { "dirty" } else { "clean" } + ) + }) + .collect::>() + .join(", "); + + eyre::bail!( + "`decodex land` completed the merge but post-land worktree cleanup debt remains under `{}`: {details}. Remove or salvage those worktrees before continuing automation.", + context.project_worktree_root.display() + ); +} + +fn manual_land_cleanup_identifier<'a>( + authority: &'a ManualAuthority, + current_branch: &'a str, +) -> &'a str { + authority.issue_identifier().unwrap_or(current_branch) +} + +fn resolve_manual_config_path(explicit: Option<&Path>, cwd: &Path) -> Result { + if let Some(explicit) = explicit { + return Ok(explicit.to_path_buf()); + } + + let state_store = runtime::open_runtime_store()?; + + if let Some(registered) = runtime::registered_config_path_for_cwd(&state_store, cwd)? { + return Ok(registered); + } + + eyre::bail!( + "Decodex project config is required for this command. Pass `--config ` or register one with `decodex project add `." + ); +} + +fn resolve_authority( + config_path: Option<&Path>, + explicit: Option<&str>, + manual_authority: bool, + worktree_root: &Path, +) -> Result { + if manual_authority { + return Ok(ManualAuthority::Manual); + } + + if let Some(explicit) = explicit { + return Ok(ManualAuthority::Issue(commit_message::normalize_issue_identifier( + "authority", + explicit, + )?)); + } + if let Some(inferred) = infer_issue_identifier_from_worktree_root(worktree_root) { + return Ok(ManualAuthority::Issue(inferred)); + } + + if config_path.is_some() { + eyre::bail!( + "Failed to infer the issue authority from worktree `{}`. Pass `--authority ` or `--manual-authority`.", + worktree_root.display() + ); + } + + eyre::bail!( + "`--authority ` or `--manual-authority` is required outside an issue worktree." + ) +} + +fn resolve_land_authority( + config_path: Option<&Path>, + explicit: Option<&str>, + manual_authority: bool, + worktree_root: &Path, +) -> Result { + if manual_authority { + return Ok(ManualAuthority::Manual); + } + + let inferred = infer_issue_identifier_from_worktree_root(worktree_root); + + if let Some(explicit) = explicit { + let explicit = commit_message::normalize_issue_identifier("authority", explicit)?; + + if let Some(inferred) = inferred { + if !explicit.eq_ignore_ascii_case(&inferred) { + eyre::bail!( + "`decodex land` authority `{explicit}` does not match the current lane issue `{inferred}`." + ); + } + + return Ok(ManualAuthority::Issue(inferred)); + } + + return Ok(ManualAuthority::Issue(explicit)); + } + if let Some(inferred) = inferred { + return Ok(ManualAuthority::Issue(inferred)); + } + + if config_path.is_some() { + eyre::bail!( + "Failed to infer the lane issue from worktree `{}`. Pass `--authority ` or `--manual-authority`.", + worktree_root.display() + ); + } + + eyre::bail!( + "`--authority ` or `--manual-authority` is required outside an issue worktree." + ) +} + +fn ensure_cli_repo_context( + cwd: &Path, + config: &ServiceConfig, + canonical_repo_root: &Path, +) -> Result<()> { + let worktree_root = current_worktree_root(cwd)?; + + if worktree_root == canonical_repo_root + || config::checkouts_share_repository(&worktree_root, canonical_repo_root)? + { + let config_repo_root = config.repo_root(); + + if config_repo_root == canonical_repo_root + || config::checkouts_share_repository(config_repo_root, canonical_repo_root)? + { + return Ok(()); + } + } + + eyre::bail!( + "Current worktree `{}` does not match loaded config repo root `{}` for canonical repo root `{}`.", + worktree_root.display(), + config.repo_root().display(), + canonical_repo_root.display(), + ); +} + +fn current_worktree_root(cwd: &Path) -> Result { + let root = run_git_capture(cwd, &["rev-parse", "--show-toplevel"])?; + + Ok(PathBuf::from(root)) +} + +fn current_branch_name(cwd: &Path) -> Result { + let branch = run_git_capture(cwd, &["branch", "--show-current"])?; + + if branch.is_empty() { + eyre::bail!("Current Git checkout is detached; switch back to a lane branch first."); + } + + Ok(branch) +} + +fn current_head_oid(cwd: &Path) -> Result { + run_git_capture(cwd, &["rev-parse", "HEAD"]) +} + +fn resolve_pr_url( + explicit: Option<&str>, + handoff: Option<&ReviewHandoffMarker>, + manual_authority: bool, +) -> Result { + if let Some(explicit) = explicit { + return Ok(explicit.trim().to_owned()); + } + if let Some(handoff) = handoff { + return Ok(handoff.pr_url().to_owned()); + } + + if manual_authority { + eyre::bail!("`decodex land --manual-authority` requires `--pr `."); + } + + eyre::bail!( + "`decodex land` requires a PR URL. Run it from a handoff worktree or pass `--pr `." + ); +} + +fn read_manual_land_handoff( + state_store: &StateStore, + service_id: &str, + issue_id: &str, + current_branch: &str, +) -> Result> { + state_store.review_handoff_marker(service_id, issue_id, current_branch) +} + +fn infer_issue_identifier_from_worktree_root(worktree_root: &Path) -> Option { + let basename = worktree_root.file_name()?.to_str()?; + + looks_like_issue_identifier(basename).then(|| basename.to_owned()) +} + +fn looks_like_issue_identifier(value: &str) -> bool { + commit_message::looks_like_issue_identifier(value) +} + +fn ensure_clean_worktree(cwd: &Path) -> Result<()> { + let status = run_git_capture(cwd, &["status", "--porcelain"])?; + + if status.lines().any(is_landing_blocking_status_line) { + eyre::bail!("Worktree has uncommitted changes. Commit or stash them before landing."); + } + + Ok(()) +} + +fn is_landing_blocking_status_line(line: &str) -> bool { + let line = line.trim_end(); + + !line.is_empty() && !is_untracked_decodex_runtime_marker_status_line(line) +} + +fn is_untracked_decodex_runtime_marker_status_line(line: &str) -> bool { + let Some(path) = line.strip_prefix("?? ") else { + return false; + }; + + path == RUN_ACTIVITY_MARKER_FILE +} + +fn validate_landing_state( + landing_state: &PullRequestLandingState, + pr_url: &str, + expected_base_branch: &str, + current_branch: &str, + current_head: &str, +) -> Result { + let gate_view = landing_state.gate_view(); + + if landing_state.base_ref_name != expected_base_branch { + eyre::bail!( + "Pull request `{pr_url}` targets base branch `{}`, but `decodex land` only lands into `{expected_base_branch}`.", + landing_state.base_ref_name + ); + } + if landing_state.head_ref_name != current_branch { + eyre::bail!( + "Pull request `{pr_url}` points at branch `{}`, but the current branch is `{current_branch}`.", + landing_state.head_ref_name + ); + } + if landing_state.head_ref_oid != current_head { + eyre::bail!( + "Pull request `{pr_url}` points at head `{}`, but the current branch head is `{current_head}`.", + landing_state.head_ref_oid + ); + } + if gate_view.state == "MERGED" { + return Ok(LandExecutionMode::CloseoutOnly); + } + if gate_view.state != "OPEN" { + eyre::bail!("Pull request `{pr_url}` is `{}` and cannot be landed.", gate_view.state); + } + if gate_view.is_draft { + eyre::bail!("Pull request `{pr_url}` is still draft."); + } + if gate_view.pending_review_requests > 0 { + eyre::bail!( + "Pull request `{pr_url}` still has {} pending review request(s).", + gate_view.pending_review_requests + ); + } + if gate_view.unresolved_review_threads > 0 { + eyre::bail!( + "Pull request `{pr_url}` still has {} unresolved review thread(s).", + gate_view.unresolved_review_threads + ); + } + if gate_view.review_decision == Some("CHANGES_REQUESTED") { + eyre::bail!("Pull request `{pr_url}` still has active change requests."); + } + + if let Some(reason) = pull_request::merge_state_requires_review_repair( + gate_view.mergeable, + gate_view.merge_state_status, + ) { + eyre::bail!("Pull request `{pr_url}` requires review repair: {reason}."); + } + + if pull_request::failed_checks_require_repair( + gate_view.status_check_rollup_state, + gate_view.merge_state_status, + ) { + eyre::bail!("Pull request `{pr_url}` has failed required checks that need repair."); + } + if !pull_request::merge_state_allows_ready_to_land(gate_view.merge_state_status) { + eyre::bail!( + "Pull request `{pr_url}` is not ready to land: mergeStateStatus=`{}`.", + gate_view.merge_state_status + ); + } + if gate_view.mergeable != "MERGEABLE" { + eyre::bail!( + "Pull request `{pr_url}` is not mergeable: mergeable=`{}`.", + gate_view.mergeable + ); + } + + match gate_view.status_check_rollup_state { + Some(other) if pull_request::checks_require_wait(Some(other)) => eyre::bail!( + "Pull request `{pr_url}` is still waiting on checks: statusCheckRollup=`{other}`." + ), + Some("SUCCESS") | None => { + debug_assert!(pull_request::manual_landing_gates_satisfied(gate_view)); + + Ok(LandExecutionMode::MergeAndCloseout) + }, + Some(other) => eyre::bail!( + "Pull request `{pr_url}` still has non-green checks: statusCheckRollup=`{other}`." + ), + } +} + +fn finalize_already_merged_manual_land_recovery( + context: &ManualLandContext, + request: &ManualLandRequest, +) -> Result> { + if !request.manual_authority || request.pr_url.is_none() { + return Ok(None); + } + if !current_checkout_is_repo_root_default_branch(context)? { + return Ok(None); + } + + let landing_state = github::inspect_pull_request_landing_state( + &context.canonical_repo_root, + &context.pr_url, + &context.github_token, + )?; + + if landing_state.state != "MERGED" { + eyre::bail!( + "`decodex land --manual-authority --pr` can recover from the repo-root default branch only after the PR is `MERGED`; `{}` is `{}`.", + context.pr_url, + landing_state.state + ); + } + + let merge_commit = github::wait_for_pull_request_merge_commit( + &context.canonical_repo_root, + &context.pr_url, + &context.github_token, + MANUAL_LAND_MERGE_VISIBILITY_TIMEOUT, + )?; + + ensure_already_merged_manual_land_recovery_ready(context, &landing_state, &merge_commit)?; + + Ok(Some(ManualLandRecoveryOutcome { merge_commit })) +} + +fn current_checkout_is_repo_root_default_branch(context: &ManualLandContext) -> Result { + let canonical_checkout = fs::canonicalize(&context.worktree_root).wrap_err_with(|| { + format!("Failed to canonicalize current checkout `{}`.", context.worktree_root.display()) + })?; + let canonical_repo_root = + fs::canonicalize(&context.canonical_repo_root).wrap_err_with(|| { + format!( + "Failed to canonicalize configured repo root `{}`.", + context.canonical_repo_root.display() + ) + })?; + + Ok(canonical_checkout == canonical_repo_root + && context.current_branch == context.repository.default_branch) +} + +fn ensure_already_merged_manual_land_recovery_ready( + context: &ManualLandContext, + landing_state: &PullRequestLandingState, + merge_commit: &str, +) -> Result<()> { + ensure_already_merged_manual_land_recovery_state(context, landing_state)?; + + default_branch_sync::preflight_repo_root_default_branch_sync( + &context.canonical_repo_root, + &context.repository.default_branch, + Some(context.default_branch_git_credentials()), + )?; + + ensure_repo_root_default_branch_current( + &context.canonical_repo_root, + &context.repository.default_branch, + )?; + ensure_merge_commit_reachable_from_default_branch( + &context.canonical_repo_root, + &context.pr_url, + merge_commit, + &context.repository.default_branch, + )?; + ensure_manual_land_recovery_lane_cleanup_complete(context, landing_state)?; + ensure_manual_land_left_no_merged_worktree_cleanup_debt(context)?; + + Ok(()) +} + +fn ensure_already_merged_manual_land_recovery_state( + context: &ManualLandContext, + landing_state: &PullRequestLandingState, +) -> Result<()> { + if landing_state.base_ref_name != context.repository.default_branch { + eyre::bail!( + "Pull request `{}` targets base branch `{}`, but manual land recovery only accepts already-merged PRs into `{}`.", + context.pr_url, + landing_state.base_ref_name, + context.repository.default_branch + ); + } + if landing_state.state != "MERGED" { + eyre::bail!( + "Pull request `{}` is `{}`; manual land recovery only accepts already-merged PRs.", + context.pr_url, + landing_state.state + ); + } + if landing_state.head_ref_name.trim().is_empty() { + eyre::bail!( + "Pull request `{}` does not expose the landed head branch required to verify lane cleanup.", + context.pr_url + ); + } + if landing_state.head_ref_name == context.repository.default_branch { + eyre::bail!( + "Pull request `{}` uses the default branch `{}` as its head; manual land recovery cannot prove lane cleanup safely.", + context.pr_url, + context.repository.default_branch + ); + } + + Ok(()) +} + +fn ensure_repo_root_default_branch_current(repo_root: &Path, default_branch: &str) -> Result<()> { + let local_head = run_git_capture(repo_root, &["rev-parse", "HEAD"])?; + let tracking_ref = format!("refs/remotes/origin/{default_branch}"); + let remote_head = run_git_capture(repo_root, &["rev-parse", tracking_ref.as_str()])?; + + if local_head == remote_head { + return Ok(()); + } + + eyre::bail!( + "Configured repo root `{}` is on `{default_branch}` but is not current with `{tracking_ref}`; sync the default branch before retrying manual land recovery.", + repo_root.display() + ); +} + +fn ensure_merge_commit_reachable_from_default_branch( + repo_root: &Path, + pr_url: &str, + merge_commit: &str, + default_branch: &str, +) -> Result<()> { + let status = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["merge-base", "--is-ancestor", merge_commit, "HEAD"]) + .status()?; + + if status.success() { + return Ok(()); + } + if status.code() == Some(1) { + eyre::bail!( + "Configured repo root `{}` is on `{default_branch}` but does not contain merge commit `{merge_commit}` for `{pr_url}`.", + repo_root.display() + ); + } + + eyre::bail!( + "`git merge-base --is-ancestor {merge_commit} HEAD` failed in `{}` with status `{}`.", + repo_root.display(), + status + ); +} + +fn ensure_manual_land_recovery_lane_cleanup_complete( + context: &ManualLandContext, + landing_state: &PullRequestLandingState, +) -> Result<()> { + let pr_head_branch = landing_state.head_ref_name.as_str(); + + if local_branch_exists(&context.canonical_repo_root, pr_head_branch)? { + eyre::bail!( + "Manual land recovery for `{}` requires the landed lane cleanup to be complete, but local branch `{pr_head_branch}` still exists.", + context.pr_url + ); + } + + let worktree_paths = linked_worktree_paths_for_landed_head_under_root( + &context.canonical_repo_root, + &context.project_worktree_root, + pr_head_branch, + &landing_state.head_ref_oid, + )?; + + if worktree_paths.is_empty() { + return Ok(()); + } + + let details = + worktree_paths.iter().map(|path| path.display().to_string()).collect::>().join(", "); + + eyre::bail!( + "Manual land recovery for `{}` requires the landed lane cleanup to be complete, but branch `{pr_head_branch}` or its head `{}` is still checked out under `{}`: {details}.", + context.pr_url, + landing_state.head_ref_oid, + context.project_worktree_root.display() + ); +} + +fn local_branch_exists(repo_root: &Path, branch_name: &str) -> Result { + let ref_name = format!("refs/heads/{branch_name}"); + let status = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["show-ref", "--verify", "--quiet", ref_name.as_str()]) + .status()?; + + if status.success() { + return Ok(true); + } + if status.code() == Some(1) { + return Ok(false); + } + + eyre::bail!( + "`git show-ref --verify --quiet {ref_name}` failed in `{}` with status `{}`.", + repo_root.display(), + status + ); +} + +fn linked_worktree_paths_for_landed_head_under_root( + repo_root: &Path, + worktree_root: &Path, + branch_name: &str, + head_oid: &str, +) -> Result> { + let output = run_git_capture(repo_root, &["worktree", "list", "--porcelain"])?; + let mut matches = Vec::new(); + let mut current_path: Option = None; + let mut current_head: Option = None; + let mut current_branch: Option = None; + + for line in output.lines() { + if line.is_empty() { + push_matching_worktree_path( + &mut matches, + &mut current_path, + &mut current_head, + &mut current_branch, + worktree_root, + branch_name, + head_oid, + )?; + + continue; + } + + if let Some(path) = line.strip_prefix("worktree ") { + push_matching_worktree_path( + &mut matches, + &mut current_path, + &mut current_head, + &mut current_branch, + worktree_root, + branch_name, + head_oid, + )?; + + current_path = Some(PathBuf::from(path)); + } else if let Some(head) = line.strip_prefix("HEAD ") { + current_head = Some(head.to_owned()); + } else if let Some(branch) = line.strip_prefix("branch refs/heads/") { + current_branch = Some(branch.to_owned()); + } + } + + push_matching_worktree_path( + &mut matches, + &mut current_path, + &mut current_head, + &mut current_branch, + worktree_root, + branch_name, + head_oid, + )?; + + Ok(matches) +} + +fn push_matching_worktree_path( + matches: &mut Vec, + path: &mut Option, + head: &mut Option, + branch: &mut Option, + worktree_root: &Path, + branch_name: &str, + head_oid: &str, +) -> Result<()> { + if (branch.as_deref() == Some(branch_name) || head.as_deref() == Some(head_oid)) + && let Some(path) = path.take() + && checkout_path_is_under_worktree_root(&path, worktree_root)? + { + matches.push(path); + } + + *path = None; + *head = None; + *branch = None; + + Ok(()) +} + +fn checkout_path_is_under_worktree_root(path: &Path, worktree_root: &Path) -> Result { + if !path.exists() || !worktree_root.exists() { + return Ok(false); + } + + let canonical_path = fs::canonicalize(path)?; + let canonical_root = fs::canonicalize(worktree_root)?; + + Ok(canonical_path.starts_with(&canonical_root) && canonical_path != canonical_root) +} + +fn prepare_closeout( + config: &ServiceConfig, + workflow: WorkflowDocument, + authority: &str, +) -> Result { + let tracker_policy = workflow.frontmatter().tracker(); + let completed_state = tracker_policy.resolved_completed_state().to_owned(); + let needs_attention_label = tracker_policy.needs_attention_label().to_owned(); + let tracker = LinearClient::new(config.tracker().resolve_api_key()?)?; + let issue = tracker + .get_issue_by_identifier(&authority.to_ascii_uppercase())? + .ok_or_else(|| eyre::eyre!("Tracker does not contain issue `{authority}`."))?; + + ensure_manual_closeout_issue_scope(&tracker, &issue, config.service_id())?; + + Ok(PreparedCloseout { + tracker, + issue, + completed_state, + service_id: config.service_id().to_owned(), + needs_attention_label, + }) +} + +fn ensure_manual_land_checkout_is_managed_lane( + checkout_root: &Path, + project_worktree_root: &Path, + issue_identifier: &str, +) -> Result<()> { + let canonical_checkout = fs::canonicalize(checkout_root).wrap_err_with(|| { + format!("Failed to canonicalize current lane checkout `{}`.", checkout_root.display()) + })?; + let canonical_worktree_root = fs::canonicalize(project_worktree_root).wrap_err_with(|| { + format!( + "Failed to canonicalize configured worktree root `{}`.", + project_worktree_root.display() + ) + })?; + + if canonical_checkout.starts_with(&canonical_worktree_root) + && canonical_checkout != canonical_worktree_root + { + return Ok(()); + } + + eyre::bail!( + "`decodex land` for issue `{issue_identifier}` must run from a managed lane under worktree_root `{}` so successful land can clean up the worktree and branch.", + project_worktree_root.display() + ); +} + +fn ensure_manual_closeout_issue_scope( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let active_label = tracker::automation_active_label(service_id); + + if tracker::issue_has_label_with_server_confirmation(tracker, issue, &active_label)? { + return Ok(()); + } + + eyre::bail!( + "Issue `{}` is not owned by service `{service_id}`; `decodex land` requires label `{active_label}`.", + issue.identifier + ); +} + +fn apply_closeout( + checkout_root: &Path, + prepared: &PreparedCloseout, + pr_url: &str, + merge_commit: &str, + branch_name: &str, + landed_change_record: &str, +) -> Result<()> { + if prepared.issue.state.name != prepared.completed_state { + let state_id = + prepared.issue.state_id_for_name(&prepared.completed_state).ok_or_else(|| { + eyre::eyre!( + "Issue `{}` does not expose tracker state `{}` on its team.", + prepared.issue.identifier, + prepared.completed_state + ) + })?; + + prepared.tracker.update_issue_state(prepared.issue.id.as_str(), state_id)?; + } + if !manual_land_closeout_matches( + checkout_root, + pr_url, + merge_commit, + branch_name, + landed_change_record, + )? { + prepared.tracker.create_comment( + prepared.issue.id.as_str(), + format!( + "decodex land completed\n\n- pr_url: `{pr_url}`\n- merge_commit: `{merge_commit}`\n- branch: `{branch_name}`\n- landed_change: `{landed_change_record}`" + ) + .as_str(), + )?; + + write_manual_land_closeout_marker( + checkout_root, + pr_url, + merge_commit, + branch_name, + landed_change_record, + )?; + } + + Ok(()) +} + +fn clear_manual_closeout_issue_scope( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, + needs_attention_label: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let closeout_labels = [ + tracker::automation_active_label(service_id), + tracker::automation_queue_label(service_id), + needs_attention_label.to_owned(), + ]; + + for label_name in closeout_labels { + clear_manual_closeout_issue_label(tracker, issue, &label_name)?; + } + + Ok(()) +} + +fn clear_manual_closeout_issue_label( + tracker: &T, + issue: &TrackerIssue, + label_name: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + if let Err(error) = tracker::set_issue_label_presence(tracker, issue, label_name, false) + && !linear_label_not_on_issue_error(&error) + { + return Err(error); + } + + Ok(()) +} + +fn clear_manual_closeout_runtime_state(state_store: &StateStore, issue_id: &str) -> Result<()> { + state_store.succeed_active_run_attempts_for_issue(issue_id).wrap_err_with(|| { + format!("Failed to finalize active runtime attempts for issue `{issue_id}`.") + })?; + state_store + .clear_lease(issue_id) + .wrap_err_with(|| format!("Failed to clear runtime lease for issue `{issue_id}`."))?; + state_store.clear_worktree(issue_id).wrap_err_with(|| { + format!("Failed to clear runtime worktree state for issue `{issue_id}`.") + })?; + + Ok(()) +} + +fn linear_label_not_on_issue_error(error: &Report) -> bool { + error + .chain() + .any(|source| source.to_string().to_ascii_lowercase().contains("label not on issue")) +} + +fn manual_land_closeout_marker_path(checkout_root: &Path) -> Result { + let Some(git_dir) = config::git_dir_for_checkout(checkout_root)? else { + eyre::bail!( + "Current checkout `{}` does not expose a Git administrative directory.", + checkout_root.display() + ); + }; + + Ok(git_dir.join(MANUAL_LAND_CLOSEOUT_MARKER_GIT_PATH)) +} + +fn manual_land_closeout_matches( + checkout_root: &Path, + pr_url: &str, + merge_commit: &str, + branch_name: &str, + landed_change_record: &str, +) -> Result { + let Some(marker) = read_manual_land_closeout_marker(checkout_root)? else { + return Ok(false); + }; + + Ok(marker.pr_url.as_deref() == Some(pr_url) + && marker.merge_commit.as_deref() == Some(merge_commit) + && marker.branch_name.as_deref() == Some(branch_name) + && marker.landed_change.as_deref() == Some(landed_change_record)) +} + +fn read_manual_land_closeout_marker( + checkout_root: &Path, +) -> Result> { + let marker_path = manual_land_closeout_marker_path(checkout_root)?; + let marker_body = match fs::read_to_string(&marker_path) { + Ok(marker_body) => marker_body, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).wrap_err_with(|| { + format!("Failed to read manual land closeout marker `{}`.", marker_path.display()) + }); + }, + }; + let mut marker = ManualLandCloseoutMarkerRecord::default(); + + for line in marker_body.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + + match key { + "pr_url" => marker.pr_url = Some(value.to_owned()), + "merge_commit" => marker.merge_commit = Some(value.to_owned()), + "branch_name" => marker.branch_name = Some(value.to_owned()), + "landed_change" => marker.landed_change = Some(value.to_owned()), + _ => {}, + } + } + + Ok(Some(marker)) +} + +fn write_manual_land_closeout_marker( + checkout_root: &Path, + pr_url: &str, + merge_commit: &str, + branch_name: &str, + landed_change_record: &str, +) -> Result<()> { + let marker_path = manual_land_closeout_marker_path(checkout_root)?; + let Some(marker_dir) = marker_path.parent() else { + eyre::bail!( + "Manual land closeout marker path `{}` has no parent directory.", + marker_path.display() + ); + }; + + fs::create_dir_all(marker_dir).wrap_err_with(|| { + format!( + "Failed to create manual land closeout marker directory `{}`.", + marker_dir.display() + ) + })?; + fs::write( + &marker_path, + format!( + "pr_url={pr_url}\nmerge_commit={merge_commit}\nbranch_name={branch_name}\nlanded_change={landed_change_record}\n" + ), + ) + .wrap_err_with(|| { + format!("Failed to write manual land closeout marker `{}`.", marker_path.display()) + })?; + + Ok(()) +} + +fn run_git_capture(cwd: &Path, args: &[&str]) -> Result { + let output = Command::new("git").arg("-C").arg(cwd).args(args).output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + eyre::bail!("`git {}` failed in `{}`: {detail}", args.join(" "), cwd.display()); + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn run_git_checked_with_stdio(cwd: &Path, args: &[&str]) -> Result<()> { + let status = Command::new("git") + .arg("-C") + .arg(cwd) + .args(args) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status()?; + + if status.success() { + return Ok(()); + } + + eyre::bail!("`git {}` failed in `{}`.", args.join(" "), cwd.display()); +} + +#[cfg(test)] +mod tests { + use std::{ + cell::RefCell, + collections::HashMap, + env, fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::Command, + }; + + use tempfile::TempDir; + + use crate::{ + config::ServiceConfig, + manual::{self, LandExecutionMode, ManualAuthority, ManualLandContext, ManualLandRequest}, + prelude::eyre, + pull_request::PullRequestLandingState, + runtime, state, + test_support::TestEnvVarGuard, + tracker::{IssueTracker, TrackerIssue, TrackerLabel, TrackerState, TrackerTeam}, + workflow::WorkflowDocument, + worktree::WorktreeManager, + }; + + struct TestTracker { + issues_by_label: HashMap>, + label_removals: RefCell>>, + label_removal_error: Option, + } + + struct MergedManualLandBranch { + branch_name: String, + head_oid: String, + merge_commit: String, + worktree_path: PathBuf, + } + + impl TestTracker { + fn new() -> Self { + Self { + issues_by_label: HashMap::new(), + label_removals: RefCell::new(Vec::new()), + label_removal_error: None, + } + } + + fn with_label_issues(mut self, label_name: &str, issues: Vec) -> Self { + self.issues_by_label.insert(label_name.to_owned(), issues); + + self + } + + fn with_label_removal_error(mut self, message: &str) -> Self { + self.label_removal_error = Some(message.to_owned()); + + self + } + } + + impl IssueTracker for TestTracker { + fn list_issues_with_label( + &self, + label_name: &str, + ) -> crate::prelude::Result> { + Ok(self.issues_by_label.get(label_name).cloned().unwrap_or_default()) + } + + fn find_team_label_id( + &self, + _team_id: &str, + _label_name: &str, + ) -> crate::prelude::Result> { + Ok(None) + } + + fn get_issue_by_identifier( + &self, + _issue_identifier: &str, + ) -> crate::prelude::Result> { + Ok(None) + } + + fn refresh_issues( + &self, + _issue_ids: &[String], + ) -> crate::prelude::Result> { + Ok(Vec::new()) + } + + fn list_comments( + &self, + _issue_id: &str, + ) -> crate::prelude::Result> { + Ok(Vec::new()) + } + + fn update_issue_state( + &self, + _issue_id: &str, + _state_id: &str, + ) -> crate::prelude::Result<()> { + Ok(()) + } + + fn add_issue_labels( + &self, + _issue_id: &str, + _label_ids: &[String], + ) -> crate::prelude::Result<()> { + Ok(()) + } + + fn remove_issue_labels( + &self, + _issue_id: &str, + label_ids: &[String], + ) -> crate::prelude::Result<()> { + self.label_removals.borrow_mut().push(label_ids.to_vec()); + + if let Some(message) = self.label_removal_error.as_ref() { + eyre::bail!("{message}"); + } + + Ok(()) + } + + fn create_comment(&self, _issue_id: &str, _body: &str) -> crate::prelude::Result<()> { + Ok(()) + } + } + + fn init_git_checkout(temp_dir: &TempDir, directory_name: &str) -> PathBuf { + let checkout = temp_dir.path().join(directory_name); + + assert!( + Command::new("git") + .args(["init", "-b", "main"]) + .current_dir(temp_dir.path()) + .arg(&checkout) + .status() + .expect("git init should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "user.name", "Decodex Tests"]) + .current_dir(&checkout) + .status() + .expect("git config should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "user.email", "decodex-tests@example.com"]) + .current_dir(&checkout) + .status() + .expect("git config should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "commit.gpgsign", "false"]) + .current_dir(&checkout) + .status() + .expect("git config should run") + .success() + ); + + checkout + } + + fn git_success(cwd: &Path, args: &[&str]) { + assert!( + Command::new("git") + .args(args) + .current_dir(cwd) + .status() + .expect("git command should run") + .success(), + "git {:?} should succeed", + args + ); + } + + fn git_add_and_commit(cwd: &Path, pathspec: &str, message: &str) { + assert!( + Command::new("git") + .args(["add", pathspec]) + .current_dir(cwd) + .status() + .expect("git add should run") + .success() + ); + assert!( + Command::new("git") + .args(["commit", "-m", message]) + .current_dir(cwd) + .status() + .expect("git commit should run") + .success() + ); + } + + fn init_git_checkout_with_origin(temp_dir: &TempDir) -> PathBuf { + let remote_root = temp_dir.path().join("origin.git"); + let checkout = temp_dir.path().join("repo"); + + assert!( + Command::new("git") + .args(["init", "--bare", "--initial-branch", "main"]) + .arg(&remote_root) + .status() + .expect("bare origin should init") + .success() + ); + assert!( + Command::new("git") + .args(["clone"]) + .arg(&remote_root) + .arg(&checkout) + .status() + .expect("repo should clone") + .success() + ); + + git_success(&checkout, &["config", "user.name", "Decodex Tests"]); + git_success(&checkout, &["config", "user.email", "decodex-tests@example.com"]); + git_success(&checkout, &["config", "commit.gpgsign", "false"]); + + fs::write(checkout.join("README.md"), "bootstrap\n").expect("readme should write"); + + git_add_and_commit(&checkout, "README.md", "bootstrap repo"); + git_success(&checkout, &["push", "origin", "main"]); + + checkout + } + + fn repo_root_manual_land_context(repo_root: &Path, worktree_root: &Path) -> ManualLandContext { + ManualLandContext { + cwd: repo_root.to_path_buf(), + current_branch: String::from("main"), + worktree_root: repo_root.to_path_buf(), + project_worktree_root: worktree_root.to_path_buf(), + canonical_repo_root: repo_root.to_path_buf(), + authority: ManualAuthority::Manual, + service_id: String::from("decodex"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/64"), + review_branch: String::from("main"), + } + } + + fn merge_manual_land_test_branch( + repo_root: &Path, + worktree_root: &Path, + ) -> MergedManualLandBranch { + let worktree_manager = WorktreeManager::new("decodex", repo_root, worktree_root); + let worktree = worktree_manager + .ensure_worktree("manual-land-cleanup", false) + .expect("manual land worktree should create"); + + fs::write(worktree.path.join("feature.txt"), "manual land\n") + .expect("feature file should write"); + + git_add_and_commit(&worktree.path, "feature.txt", "manual land feature"); + + let head_oid = manual::run_git_capture(&worktree.path, &["rev-parse", "HEAD"]) + .expect("PR head should read"); + + git_success(repo_root, &["merge", "--no-ff", &worktree.branch_name, "-m", "land feature"]); + + let merge_commit = + manual::run_git_capture(repo_root, &["rev-parse", "HEAD"]).expect("merge head"); + + git_success(repo_root, &["push", "origin", "main"]); + + MergedManualLandBranch { + branch_name: worktree.branch_name, + head_oid, + merge_commit, + worktree_path: worktree.path, + } + } + + fn remove_test_lane_checkout(repo_root: &Path, worktree_path: &Path, branch_name: &str) { + git_success(worktree_path, &["checkout", "--detach"]); + git_success(repo_root, &["branch", "-D", branch_name]); + git_success( + repo_root, + &[ + "worktree", + "remove", + "--force", + worktree_path.to_str().expect("worktree path should be UTF-8"), + ], + ); + } + + fn create_dirty_merged_worktree_debt(repo_root: &Path, worktree_root: &Path) { + let worktree_manager = WorktreeManager::new("decodex", repo_root, worktree_root); + let worktree = + worktree_manager.ensure_worktree("XY-999", false).expect("debt worktree should create"); + + fs::write(worktree.path.join("debt.txt"), "debt\n").expect("debt file should write"); + + git_add_and_commit(&worktree.path, "debt.txt", "debt feature"); + git_success(repo_root, &["merge", "--no-ff", &worktree.branch_name, "-m", "land debt"]); + git_success(repo_root, &["push", "origin", "main"]); + + fs::write(worktree.path.join("debt.txt"), "dirty debt\n") + .expect("debt worktree should become dirty"); + } + + fn merged_manual_land_state(branch_name: &str, head_oid: &str) -> PullRequestLandingState { + let mut landing_state = sample_landing_state(); + + landing_state.state = String::from("MERGED"); + landing_state.base_ref_name = String::from("main"); + landing_state.head_ref_name = branch_name.to_owned(); + landing_state.head_ref_oid = head_oid.to_owned(); + + landing_state + } + + fn install_fake_landing_state_gh( + temp_dir: &TempDir, + state: &str, + branch_name: &str, + head_oid: &str, + merge_commit: &str, + ) -> TestEnvVarGuard { + let fake_gh_dir = temp_dir.path().join("fake-recovery-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"graphql\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"view\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "url": "https://github.com/hack-ink/decodex/pull/64", + "state": state, + "isDraft": false, + "reviewDecision": "APPROVED", + "baseRefName": "main", + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "headRefName": branch_name, + "headRefOid": head_oid, + "reviewRequests": { "totalCount": 0 }, + "reviewThreads": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null }, + }, + "commits": { + "nodes": [ + { + "commit": { + "statusCheckRollup": { "state": "SUCCESS" }, + }, + }, + ], + }, + }, + }, + }, + }), + serde_json::json!({ + "state": state, + "headRefOid": head_oid, + "mergeCommit": { "oid": merge_commit }, + }), + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + { + PermissionsExt::set_mode(&mut permissions, 0o755); + } + + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())) + } + + fn sample_workflow() -> WorkflowDocument { + WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 8 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Test workflow. +"#, + ) + .expect("sample workflow should parse") + } + + fn install_fake_admin_merge_gh( + temp_dir: &TempDir, + merged_head_oid: &str, + ) -> (TestEnvVarGuard, PathBuf) { + install_fake_admin_merge_gh_with_merge_exit_code( + temp_dir, + merged_head_oid, + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + r#"{"schema":"decodex/commit/1","summary":"ship hotfix","authority":"manual"}"#, + 0, + ) + } + + fn install_fake_admin_merge_gh_with_merge_exit_code( + temp_dir: &TempDir, + merged_head_oid: &str, + pr_head_oid: &str, + merge_subject: &str, + merge_exit_code: i32, + ) -> (TestEnvVarGuard, PathBuf) { + let fake_gh_dir = temp_dir.path().join("fake-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let invocation_log_path = temp_dir.path().join("gh-invocation.log"); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +printf '%s\\n' \"$*\" >> '{}'\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"merge\" ]; then\n\ + exit {}\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"view\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"api\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + invocation_log_path.display(), + merge_exit_code, + serde_json::json!({ + "state": "MERGED", + "headRefOid": pr_head_oid, + "mergeCommit": { "oid": merged_head_oid }, + }), + serde_json::json!({ + "commit": { "message": format!("{merge_subject}\n\n") }, + }), + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + { + PermissionsExt::set_mode(&mut permissions, 0o755); + } + + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + ( + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())), + invocation_log_path, + ) + } + + #[test] + fn issue_identifier_helpers_recognize_lane_directory_names() { + let inferred = + manual::infer_issue_identifier_from_worktree_root(Path::new("/tmp/.worktrees/XY-225")) + .expect("issue identifier should infer from worktree basename"); + + assert_eq!(inferred, "XY-225"); + assert!(!manual::looks_like_issue_identifier("decodex")); + assert!(!manual::looks_like_issue_identifier("feature-branch")); + assert!(manual::looks_like_issue_identifier("xy-225")); + } + + #[test] + fn landing_cleanliness_ignores_untracked_decodex_runtime_markers() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + + fs::write(checkout.join(state::RUN_ACTIVITY_MARKER_FILE), "agent_run\n") + .expect("activity marker should write"); + manual::ensure_clean_worktree(&checkout) + .expect("untracked activity marker should not block landing"); + } + + #[test] + fn landing_cleanliness_rejects_blocking_worktree_statuses() { + fn assert_blocks(checkout: &Path, case_name: &str) { + let error = manual::ensure_clean_worktree(checkout).expect_err(case_name); + + assert!( + error.to_string().contains("uncommitted changes"), + "unexpected error for `{case_name}`: {error:?}" + ); + } + { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + + fs::write(checkout.join("scratch.txt"), "debug\n").expect("scratch file should write"); + + assert_blocks(&checkout, "untracked non-runtime files should block landing"); + } + { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + let nested_dir = checkout.join("nested"); + + fs::create_dir_all(&nested_dir).expect("nested directory should create"); + fs::write(nested_dir.join(state::RUN_ACTIVITY_MARKER_FILE), "agent_run\n") + .expect("nested activity marker should write"); + + assert_blocks(&checkout, "nested runtime marker should still block landing"); + } + { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + let marker_path = checkout.join(state::RUN_ACTIVITY_MARKER_FILE); + + fs::write(&marker_path, "idle\n").expect("activity marker should write"); + + git_add_and_commit( + &checkout, + state::RUN_ACTIVITY_MARKER_FILE, + "track activity marker for test", + ); + + fs::write(&marker_path, "agent_run\n").expect("activity marker should update"); + + assert_blocks(&checkout, "tracked runtime marker changes should block landing"); + } + } + + #[test] + fn landing_state_validation_blocks_base_drift_except_after_merge() { + let error = manual::validate_landing_state( + &sample_landing_state(), + "https://github.com/hack-ink/decodex/pull/64", + "main", + "XY-225", + "deadbeef", + ) + .expect_err("non-default-base PR should be rejected"); + + assert!(error.to_string().contains("targets base branch `release/1.x`")); + assert!(error.to_string().contains("only lands into `main`")); + + let mut landing_state = sample_landing_state(); + + landing_state.state = String::from("MERGED"); + + let mode = manual::validate_landing_state( + &landing_state, + "https://github.com/hack-ink/decodex/pull/64", + "release/1.x", + "XY-225", + "deadbeef", + ) + .expect("merged PR should resume closeout"); + + assert_eq!(mode, LandExecutionMode::CloseoutOnly); + } + + #[test] + fn execute_land_merge_uses_admin_merge() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh(&temp_dir, "cafebabe"); + let context = manual::ManualLandContext { + cwd: checkout.clone(), + current_branch: String::from("xy-225"), + worktree_root: temp_dir.path().join(".worktrees"), + project_worktree_root: temp_dir.path().join(".worktrees"), + canonical_repo_root: checkout, + authority: ManualAuthority::Manual, + service_id: String::from("decodex"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/64"), + review_branch: String::from("xy-225"), + }; + let merge_commit = manual::execute_land_merge( + &context, + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + r#"{"schema":"decodex/commit/1","summary":"ship hotfix","authority":"manual"}"#, + LandExecutionMode::MergeAndCloseout, + ) + .expect("manual land should admin-merge successfully"); + + assert_eq!(merge_commit, "cafebabe"); + assert_eq!( + fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .collect::>(), + vec![ + "pr merge --admin --merge --match-head-commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef --subject {\"schema\":\"decodex/commit/1\",\"summary\":\"ship hotfix\",\"authority\":\"manual\"} --body https://github.com/hack-ink/decodex/pull/64", + "pr view https://github.com/hack-ink/decodex/pull/64 --json state,headRefOid,mergeCommit", + ] + ); + } + + #[test] + fn execute_land_merge_tolerates_already_merged_merge_race() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_with_merge_exit_code( + &temp_dir, + "cafebabe", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + r#"{"schema":"decodex/commit/1","summary":"ship hotfix","authority":"manual"}"#, + 1, + ); + let context = manual::ManualLandContext { + cwd: checkout.clone(), + current_branch: String::from("xy-225"), + worktree_root: temp_dir.path().join(".worktrees"), + project_worktree_root: temp_dir.path().join(".worktrees"), + canonical_repo_root: checkout, + authority: ManualAuthority::Manual, + service_id: String::from("decodex"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/64"), + review_branch: String::from("xy-225"), + }; + let merge_commit = manual::execute_land_merge( + &context, + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + r#"{"schema":"decodex/commit/1","summary":"ship hotfix","authority":"manual"}"#, + LandExecutionMode::MergeAndCloseout, + ) + .expect("manual land should accept an already-merged PR race"); + + assert_eq!(merge_commit, "cafebabe"); + assert_eq!( + fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .collect::>(), + vec![ + "pr merge --admin --merge --match-head-commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef --subject {\"schema\":\"decodex/commit/1\",\"summary\":\"ship hotfix\",\"authority\":\"manual\"} --body https://github.com/hack-ink/decodex/pull/64", + "pr view https://github.com/hack-ink/decodex/pull/64 --json state,headRefOid,mergeCommit", + "pr view https://github.com/hack-ink/decodex/pull/64 --json state,headRefOid,mergeCommit", + ] + ); + } + + #[test] + fn load_authoritative_landed_change_record_uses_merge_commit_subject() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_with_merge_exit_code( + &temp_dir, + "cafebabe", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + r#"{"schema":"decodex/commit/1","summary":"actual merge subject","authority":"manual"}"#, + 0, + ); + let context = manual::ManualLandContext { + cwd: checkout.clone(), + current_branch: String::from("xy-225"), + worktree_root: temp_dir.path().join(".worktrees"), + project_worktree_root: temp_dir.path().join(".worktrees"), + canonical_repo_root: checkout, + authority: ManualAuthority::Manual, + service_id: String::from("decodex"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/64"), + review_branch: String::from("xy-225"), + }; + let landed_change_record = + manual::load_authoritative_landed_change_record(&context, "cafebabe") + .expect("merge commit subject should load"); + + assert_eq!( + landed_change_record, + r#"{"schema":"decodex/commit/1","summary":"actual merge subject","authority":"manual"}"# + ); + assert_eq!( + fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .collect::>(), + vec!["api repos/hack-ink/decodex/commits/cafebabe"] + ); + } + + #[test] + fn land_authority_validates_issue_override_against_lane() { + let error = manual::resolve_land_authority( + Some(Path::new("/tmp/project.toml")), + Some("XY-999"), + false, + Path::new("/tmp/.worktrees/XY-225"), + ) + .expect_err("mismatched explicit authority should be rejected"); + + assert!(error.to_string().contains("does not match the current lane issue `XY-225`")); + + let authority = manual::resolve_land_authority( + Some(Path::new("/tmp/project.toml")), + Some("XY-225"), + false, + Path::new("/tmp/.worktrees/xy-225"), + ) + .expect("same issue with different casing should be accepted"); + + assert_eq!(authority, ManualAuthority::Issue(String::from("xy-225"))); + } + + #[test] + fn resolve_authority_accepts_manual_authority() { + let authority = manual::resolve_authority( + Some(Path::new("/tmp/project.toml")), + None, + true, + Path::new("/tmp/worktree"), + ) + .expect("manual authority should resolve"); + + assert_eq!(authority, ManualAuthority::Manual); + } + + #[test] + fn resolve_pr_url_requires_explicit_pr_for_manual_authority() { + let error = manual::resolve_pr_url(None, None, true) + .expect_err("manual authority land should require explicit pr"); + + assert!(error.to_string().contains("--manual-authority")); + } + + #[test] + fn prepare_closeout_matches_authority_case_insensitively() { + assert_eq!("xy-225".to_ascii_uppercase(), "XY-225"); + } + + #[test] + fn manual_closeout_scope_requires_service_ownership() { + let issue = sample_issue("issue-1", "XY-225", false, &[]); + let error = + manual::ensure_manual_closeout_issue_scope(&TestTracker::new(), &issue, "pubfi") + .expect_err("service ownership should be required"); + + assert!(error.to_string().contains("decodex:active:pubfi")); + + let issue = sample_issue("issue-1", "XY-225", false, &[]); + let tracker = + TestTracker::new().with_label_issues("decodex:active:pubfi", vec![issue.clone()]); + + manual::ensure_manual_closeout_issue_scope(&tracker, &issue, "pubfi") + .expect("server-confirmed service ownership should pass"); + } + + #[test] + fn manual_closeout_clear_removes_present_transient_decodex_labels() { + for (case_name, labels, expected_label_ids) in [ + ( + "all transient labels present", + &["decodex:active:pubfi", "decodex:queued:pubfi", "decodex:needs-attention"][..], + &["team-label-0", "team-label-1", "team-label-2"][..], + ), + ( + "optional transient labels absent", + &["decodex:active:pubfi"][..], + &["team-label-0"][..], + ), + ] { + let issue = sample_issue("issue-1", "XY-225", true, labels); + let tracker = TestTracker::new(); + + manual::clear_manual_closeout_issue_scope( + &tracker, + &issue, + "pubfi", + "decodex:needs-attention", + ) + .expect(case_name); + + let expected_removals = expected_label_ids + .iter() + .map(|label_id| vec![(*label_id).to_owned()]) + .collect::>(); + + assert_eq!(tracker.label_removals.borrow().as_slice(), expected_removals.as_slice()); + } + } + + #[test] + fn manual_closeout_clear_classifies_label_removal_errors() { + for (case_name, labels, message, expected_label_ids, expected_error) in [ + ( + "missing label removal is idempotent", + &["decodex:active:pubfi", "decodex:queued:pubfi", "decodex:needs-attention"][..], + "Linear GraphQL request failed: Label not on issue", + &["team-label-0", "team-label-1", "team-label-2"][..], + None, + ), + ( + "other label removal errors are preserved", + &["decodex:active:pubfi"][..], + "Linear GraphQL request failed: Timeout", + &["team-label-0"][..], + Some("Timeout"), + ), + ] { + let issue = sample_issue("issue-1", "XY-225", true, labels); + let tracker = TestTracker::new().with_label_removal_error(message); + let result = manual::clear_manual_closeout_issue_scope( + &tracker, + &issue, + "pubfi", + "decodex:needs-attention", + ); + + if let Some(expected_error) = expected_error { + let error = result.expect_err(case_name); + + assert!(error.to_string().contains(expected_error)); + } else { + result.expect(case_name); + } + + let expected_removals = expected_label_ids + .iter() + .map(|label_id| vec![(*label_id).to_owned()]) + .collect::>(); + + assert_eq!(tracker.label_removals.borrow().as_slice(), expected_removals.as_slice()); + } + } + + #[test] + fn manual_closeout_runtime_clear_removes_lane_state() { + let state_store = state::StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("issue-1", "XY-225", true, &["decodex:active:pubfi"]); + let other_issue = sample_issue("issue-2", "XY-226", true, &["decodex:active:pubfi"]); + let handoff = state::ReviewHandoffMarker::new( + "run-1", + 1, + "y/decodex-xy-225", + "https://github.com/hack-ink/decodex/pull/67", + "main", + "y/decodex-xy-225", + "deadbeef", + ); + + state_store + .upsert_lease("decodex", &issue.id, "run-1", "In Progress") + .expect("issue lease should persist"); + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("issue running attempt should persist"); + state_store + .record_run_attempt("run-1-starting", &issue.id, 2, "starting") + .expect("issue starting attempt should persist"); + state_store + .record_run_attempt("run-1-failed", &issue.id, 3, "failed") + .expect("issue terminal attempt should persist"); + state_store + .upsert_worktree("decodex", &issue.id, "y/decodex-xy-225", "/tmp/worktrees/xy-225") + .expect("issue worktree should persist"); + state_store + .upsert_review_handoff_marker("decodex", &issue.id, &handoff) + .expect("issue handoff should persist"); + state_store + .upsert_lease("decodex", &other_issue.id, "run-2", "In Progress") + .expect("other issue lease should persist"); + state_store + .record_run_attempt("run-2", &other_issue.id, 1, "running") + .expect("other issue running attempt should persist"); + + manual::clear_manual_closeout_runtime_state(&state_store, &issue.id) + .expect("manual closeout runtime state should clear"); + + assert!( + state_store + .list_leases("decodex") + .expect("leases should list") + .iter() + .all(|lease| lease.issue_id() != issue.id) + ); + assert!( + state_store + .list_leases("decodex") + .expect("leases should list") + .iter() + .any(|lease| lease.issue_id() == other_issue.id) + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_none() + ); + assert!( + state_store + .review_handoff_marker("decodex", &issue.id, "y/decodex-xy-225") + .expect("handoff lookup should succeed") + .is_none() + ); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should succeed") + .expect("run attempt should remain") + .status(), + "succeeded" + ); + assert_eq!( + state_store + .run_attempt("run-1-starting") + .expect("run attempt lookup should succeed") + .expect("run attempt should remain") + .status(), + "succeeded" + ); + assert_eq!( + state_store + .run_attempt("run-1-failed") + .expect("run attempt lookup should succeed") + .expect("run attempt should remain") + .status(), + "failed" + ); + assert_eq!( + state_store + .run_attempt("run-2") + .expect("run attempt lookup should succeed") + .expect("run attempt should remain") + .status(), + "running" + ); + } + + #[test] + fn manual_land_issue_closeout_removes_managed_lane_worktree_and_branch() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout(&temp_dir, "repo"); + let worktree_root = repo_root.join(".worktrees"); + + fs::write(repo_root.join("README.md"), "bootstrap\n").expect("readme should write"); + + git_add_and_commit(&repo_root, "README.md", "bootstrap repo"); + + let worktree_manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let worktree = + worktree_manager.ensure_worktree("XY-225", false).expect("worktree should create"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh(&temp_dir, "cafebabe"); + let context = manual::ManualLandContext { + cwd: worktree.path.clone(), + current_branch: worktree.branch_name.clone(), + worktree_root: worktree.path.clone(), + project_worktree_root: worktree_root.clone(), + canonical_repo_root: repo_root.clone(), + authority: ManualAuthority::Issue(String::from("XY-225")), + service_id: String::from("pubfi"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/64"), + review_branch: worktree.branch_name.clone(), + }; + + manual::cleanup_manual_land_lane_checkout(&context) + .expect("manual land cleanup should remove the lane checkout"); + + let gh_invocations = + fs::read_to_string(invocation_log_path).expect("fake gh invocation log should read"); + + assert!( + gh_invocations + .contains("api --method DELETE --silent repos/hack-ink/decodex/git/refs/heads/"), + "manual land cleanup should delete the remote branch through gh api" + ); + assert!(!worktree.path.exists(), "manual land cleanup should remove the worktree"); + assert!( + manual::run_git_capture(&repo_root, &["branch", "--list", &worktree.branch_name]) + .expect("local branch list should run") + .is_empty(), + "manual land cleanup should delete the local lane branch" + ); + } + + #[test] + fn manual_land_manual_authority_removes_managed_lane_worktree_and_branch() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout(&temp_dir, "repo"); + let worktree_root = repo_root.join(".worktrees"); + + fs::write(repo_root.join("README.md"), "bootstrap\n").expect("readme should write"); + + git_add_and_commit(&repo_root, "README.md", "bootstrap repo"); + + let worktree_manager = WorktreeManager::new("decodex", &repo_root, &worktree_root); + let worktree = worktree_manager + .ensure_worktree("manual-land-cleanup", false) + .expect("worktree should create"); + let (_path_guard, _invocation_log_path) = + install_fake_admin_merge_gh(&temp_dir, "cafebabe"); + let context = manual::ManualLandContext { + cwd: worktree.path.clone(), + current_branch: worktree.branch_name.clone(), + worktree_root: worktree.path.clone(), + project_worktree_root: worktree_root.clone(), + canonical_repo_root: repo_root.clone(), + authority: ManualAuthority::Manual, + service_id: String::from("decodex"), + workflow: sample_workflow(), + github_token_env_var: String::from("GITHUB_TOKEN"), + github_token: String::from("test-token"), + repository: crate::github::RepositoryContext { + owner: String::from("hack-ink"), + name: String::from("decodex"), + default_branch: String::from("main"), + merge_commit_allowed: true, + }, + prepared_closeout: None, + pr_url: String::from("https://github.com/hack-ink/decodex/pull/65"), + review_branch: worktree.branch_name.clone(), + }; + + manual::cleanup_manual_land_lane_checkout(&context) + .expect("manual authority cleanup should remove the lane checkout"); + + assert!(!worktree.path.exists(), "manual authority cleanup should remove the worktree"); + assert!( + manual::run_git_capture(&repo_root, &["branch", "--list", &worktree.branch_name]) + .expect("local branch list should run") + .is_empty(), + "manual authority cleanup should delete the local lane branch" + ); + } + + #[test] + fn manual_land_manual_authority_recovery_accepts_merged_pr_after_cleanup() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let merged_pr = merge_manual_land_test_branch(&repo_root, &worktree_root); + + remove_test_lane_checkout(&repo_root, &merged_pr.worktree_path, &merged_pr.branch_name); + + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let landing_state = merged_manual_land_state(&merged_pr.branch_name, &merged_pr.head_oid); + + manual::ensure_already_merged_manual_land_recovery_ready( + &context, + &landing_state, + &merged_pr.merge_commit, + ) + .expect("already-merged manual land recovery should succeed after cleanup debt is gone"); + } + + #[test] + fn manual_land_manual_authority_recovery_entrypoint_accepts_merged_pr() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let merged_pr = merge_manual_land_test_branch(&repo_root, &worktree_root); + + remove_test_lane_checkout(&repo_root, &merged_pr.worktree_path, &merged_pr.branch_name); + + let _path_guard = install_fake_landing_state_gh( + &temp_dir, + "MERGED", + &merged_pr.branch_name, + &merged_pr.head_oid, + &merged_pr.merge_commit, + ); + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let request = ManualLandRequest { + summary: String::from("land manual PR"), + authority: None, + manual_authority: true, + pr_url: Some(context.pr_url.clone()), + related: Vec::new(), + breaking: false, + }; + let outcome = manual::finalize_already_merged_manual_land_recovery(&context, &request) + .expect("entrypoint should accept already-merged PR recovery") + .expect("manual-authority recovery should run from repo-root main"); + + assert_eq!(outcome.merge_commit, merged_pr.merge_commit); + } + + #[test] + fn manual_land_manual_authority_recovery_rejects_unmerged_pr() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let mut landing_state = sample_landing_state(); + + landing_state.base_ref_name = String::from("main"); + landing_state.head_ref_name = String::from("x/decodex-manual-land-cleanup"); + + let error = manual::ensure_already_merged_manual_land_recovery_ready( + &context, + &landing_state, + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + ) + .expect_err("unmerged PRs must not use default-branch recovery"); + + assert!(error.to_string().contains("only accepts already-merged PRs")); + } + + #[test] + fn manual_land_manual_authority_recovery_rejects_incomplete_lane_cleanup() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let merged_pr = merge_manual_land_test_branch(&repo_root, &worktree_root); + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let landing_state = merged_manual_land_state(&merged_pr.branch_name, &merged_pr.head_oid); + let error = manual::ensure_already_merged_manual_land_recovery_ready( + &context, + &landing_state, + &merged_pr.merge_commit, + ) + .expect_err("recovery should reject when the landed lane branch remains"); + + assert!(error.to_string().contains("landed lane cleanup to be complete")); + assert!(error.to_string().contains(&merged_pr.branch_name)); + } + + #[test] + fn manual_land_manual_authority_recovery_rejects_detached_lane_worktree() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let merged_pr = merge_manual_land_test_branch(&repo_root, &worktree_root); + + git_success(&merged_pr.worktree_path, &["checkout", "--detach"]); + git_success(&repo_root, &["branch", "-D", &merged_pr.branch_name]); + + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let landing_state = merged_manual_land_state(&merged_pr.branch_name, &merged_pr.head_oid); + let error = manual::ensure_already_merged_manual_land_recovery_ready( + &context, + &landing_state, + &merged_pr.merge_commit, + ) + .expect_err("recovery should reject a detached worktree at the landed PR head"); + + assert!(error.to_string().contains("landed lane cleanup to be complete")); + assert!(error.to_string().contains(&merged_pr.head_oid)); + } + + #[test] + fn manual_land_manual_authority_recovery_rejects_remaining_cleanup_debt() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout_with_origin(&temp_dir); + let worktree_root = repo_root.join(".worktrees"); + let merged_pr = merge_manual_land_test_branch(&repo_root, &worktree_root); + + remove_test_lane_checkout(&repo_root, &merged_pr.worktree_path, &merged_pr.branch_name); + create_dirty_merged_worktree_debt(&repo_root, &worktree_root); + + let context = repo_root_manual_land_context(&repo_root, &worktree_root); + let landing_state = merged_manual_land_state(&merged_pr.branch_name, &merged_pr.head_oid); + let error = manual::ensure_already_merged_manual_land_recovery_ready( + &context, + &landing_state, + &merged_pr.merge_commit, + ) + .expect_err("recovery should reject remaining merged worktree cleanup debt"); + + assert!(error.to_string().contains("post-land worktree cleanup debt remains")); + assert!(error.to_string().contains("XY-999")); + } + + #[test] + fn manual_land_issue_closeout_requires_managed_lane_checkout() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = init_git_checkout(&temp_dir, "repo"); + let worktree_root = repo_root.join(".worktrees"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + let error = manual::ensure_manual_land_checkout_is_managed_lane( + &repo_root, + &worktree_root, + "XY-225", + ) + .expect_err("issue closeout should require a managed lane checkout"); + + assert!(error.to_string().contains("must run from a managed lane")); + assert!(error.to_string().contains("XY-225")); + } + + #[test] + fn manual_land_closeout_marker_roundtrips() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + + manual::write_manual_land_closeout_marker( + &checkout, + "https://github.com/hack-ink/decodex/pull/67", + "deadbeef", + "xy-225", + r#"{"schema":"decodex/commit/1"}"#, + ) + .expect("closeout marker should write"); + + assert!( + manual::manual_land_closeout_matches( + &checkout, + "https://github.com/hack-ink/decodex/pull/67", + "deadbeef", + "xy-225", + r#"{"schema":"decodex/commit/1"}"#, + ) + .expect("closeout marker should read"), + ); + + let marker = manual::read_manual_land_closeout_marker(&checkout) + .expect("closeout marker should parse") + .expect("closeout marker should exist"); + + assert_eq!(marker.landed_change.as_deref(), Some(r#"{"schema":"decodex/commit/1"}"#)); + assert!( + !checkout.join(".decodex/manual-land-closeout").exists(), + "closeout marker should live under git admin state, not the working tree" + ); + } + + #[test] + fn manual_land_closeout_marker_rejects_mismatched_receipts() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let checkout = init_git_checkout(&temp_dir, "repo"); + + manual::write_manual_land_closeout_marker( + &checkout, + "https://github.com/hack-ink/decodex/pull/67", + "deadbeef", + "xy-225", + r#"{"schema":"decodex/commit/1"}"#, + ) + .expect("closeout marker should write"); + + assert!( + !manual::manual_land_closeout_matches( + &checkout, + "https://github.com/hack-ink/decodex/pull/67", + "cafebabe", + "xy-225", + r#"{"schema":"decodex/commit/1"}"#, + ) + .expect("closeout marker should compare"), + ); + } + + #[test] + fn manual_land_handoff_lookup_prefers_current_branch_record() { + let issue = sample_issue("issue-1", "XY-225", true, &["decodex:active:pubfi"]); + let state_store = state::StateStore::open_in_memory().expect("state store should open"); + + state_store + .upsert_review_handoff_marker( + "decodex", + &issue.id, + &state::ReviewHandoffMarker::new( + String::from("run-current"), + 2, + String::from("xy-225"), + String::from("https://github.com/hack-ink/decodex/pull/67"), + String::from("main"), + String::from("xy-225"), + String::from("deadbeef"), + ), + ) + .expect("runtime handoff should persist"); + state_store + .upsert_review_handoff_marker( + "decodex", + &issue.id, + &state::ReviewHandoffMarker::new( + String::from("run-other"), + 3, + String::from("xy-225-next"), + String::from("https://github.com/hack-ink/decodex/pull/99"), + String::from("main"), + String::from("xy-225-next"), + String::from("cafebabe"), + ), + ) + .expect("unrelated runtime handoff should persist"); + + let handoff = + manual::read_manual_land_handoff(&state_store, "decodex", &issue.id, "xy-225") + .expect("manual land handoff should read") + .expect("current branch handoff should be found"); + + assert_eq!(handoff.branch_name(), "xy-225"); + assert_eq!(handoff.pr_url(), "https://github.com/hack-ink/decodex/pull/67"); + } + + #[test] + fn resolve_manual_config_path_uses_registered_project_for_linked_worktree() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let _home_guard = TestEnvVarGuard::set( + "HOME", + temp_dir.path().to_str().expect("temp dir path should be utf-8"), + ); + let repo_root = temp_dir.path().join("target-repo"); + let worktree_root = repo_root.join(".worktrees"); + let config_dir = temp_dir.path().join(".codex/decodex/projects/pubfi"); + let config_path = config_dir.join("project.toml"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + fs::create_dir_all(&config_dir).expect("config dir should exist"); + + assert!( + Command::new("git") + .args(["init", "-b", "main"]) + .current_dir(temp_dir.path()) + .arg(&repo_root) + .status() + .expect("git init should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "user.name", "Decodex Tests"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "user.email", "decodex-tests@example.com"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + assert!( + Command::new("git") + .args(["config", "commit.gpgsign", "false"]) + .current_dir(&repo_root) + .status() + .expect("git config should run") + .success() + ); + + fs::write( + &config_path, + format!( + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "PATH" + + [paths] + repo_root = "{}" + "#, + repo_root.display() + ), + ) + .expect("central project config should write"); + fs::write(config_dir.join("WORKFLOW.md"), "test workflow\n") + .expect("central workflow should write"); + fs::write(repo_root.join("README.md"), "bootstrap\n").expect("readme should write"); + + git_success(&repo_root, &["add", "README.md"]); + git_success(&repo_root, &["commit", "-m", "bootstrap repo"]); + + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let worktree = manager.ensure_worktree("XY-225", false).expect("worktree should create"); + let state_store = runtime::open_runtime_store().expect("state store should open"); + let canonical_config = + fs::canonicalize(&config_path).expect("central config should canonicalize"); + + runtime::register_project_config(&state_store, &config_path, true) + .expect("central config should register"); + + assert_eq!( + manual::resolve_manual_config_path(None, &worktree.path) + .expect("registered config path should resolve"), + canonical_config + ); + } + + #[test] + fn ensure_cli_repo_context_rejects_foreign_config_repo_root() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let current_repo = init_git_checkout(&temp_dir, "current-repo"); + let foreign_repo = init_git_checkout(&temp_dir, "foreign-repo"); + let config_path = foreign_repo.join("project.toml"); + + fs::write( + &config_path, + r#" + service_id = "pubfi" + + [tracker] + api_key_env_var = "HOME" + + [github] + token_env_var = "PATH" + + [paths] + repo_root = "." + "#, + ) + .expect("foreign config should write"); + + let config = ServiceConfig::from_path(&config_path).expect("config should parse"); + let canonical_repo_root = + fs::canonicalize(¤t_repo).expect("current repo root should canonicalize"); + let error = manual::ensure_cli_repo_context(¤t_repo, &config, &canonical_repo_root) + .expect_err("foreign config repo root should be rejected"); + + assert!(error.to_string().contains("does not match loaded config repo root")); + assert!(error.to_string().contains(&foreign_repo.display().to_string())); + } + + fn sample_landing_state() -> PullRequestLandingState { + PullRequestLandingState { + url: String::from("https://github.com/hack-ink/decodex/pull/64"), + state: String::from("OPEN"), + is_draft: false, + review_decision: Some(String::from("APPROVED")), + base_ref_name: String::from("release/1.x"), + pending_review_requests: 0, + mergeable: String::from("MERGEABLE"), + merge_state_status: String::from("CLEAN"), + head_ref_name: String::from("XY-225"), + head_ref_oid: String::from("deadbeef"), + status_check_rollup_state: Some(String::from("SUCCESS")), + unresolved_review_threads: 0, + } + } + + fn sample_issue( + id: &str, + identifier: &str, + labels_complete: bool, + labels: &[&str], + ) -> TrackerIssue { + TrackerIssue { + id: id.to_owned(), + identifier: identifier.to_owned(), + #[cfg(test)] + project_slug: None, + title: String::from("Sample issue"), + description: String::from(""), + priority: None, + created_at: String::from("2026-04-13T00:00:00Z"), + updated_at: String::from("2026-04-13T00:00:00Z"), + state: TrackerState { id: String::from("state-1"), name: String::from("In Review") }, + team: TrackerTeam { + id: String::from("team-1"), + name: String::from("Core"), + states: vec![TrackerState { + id: String::from("state-1"), + name: String::from("In Review"), + }], + labels: labels + .iter() + .enumerate() + .map(|(index, label)| TrackerLabel { + id: format!("team-label-{index}"), + name: (*label).to_owned(), + }) + .collect(), + }, + labels_complete, + labels: labels + .iter() + .enumerate() + .map(|(index, label)| TrackerLabel { + id: format!("issue-label-{index}"), + name: (*label).to_owned(), + }) + .collect(), + blockers: Vec::new(), + } + } +} diff --git a/apps/decodex/src/orchestrator.rs b/apps/decodex/src/orchestrator.rs new file mode 100644 index 00000000..aeb81f14 --- /dev/null +++ b/apps/decodex/src/orchestrator.rs @@ -0,0 +1,210 @@ +#[cfg(unix)] use std::os::fd::AsRawFd; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + env, + error::Error, + fmt::{self, Display, Formatter}, + fs::{self, File}, + io::{ErrorKind, Read, Write}, + net::{SocketAddr, TcpListener, TcpStream}, + path::{Path, PathBuf}, + process::{Child, Command, ExitStatus, Stdio}, + slice, + sync::{ + Arc, Mutex, + mpsc::{self, Receiver, RecvTimeoutError, Sender}, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use color_eyre::Report; +use libc::pid_t; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{agent, default_branch_sync, git_credentials, state}; +#[rustfmt::skip] +use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_COMMENT_TOOL_NAME, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{InternalReviewMode, ServiceConfig}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; + +include!("orchestrator/types.rs"); + +include!("orchestrator/entrypoints.rs"); + +include!("orchestrator/operator_http.rs"); + +include!("orchestrator/pull_request_review.rs"); + +include!("orchestrator/daemon.rs"); + +include!("orchestrator/reconciliation.rs"); + +include!("orchestrator/run_cycle.rs"); + +include!("orchestrator/runtime_validation.rs"); + +include!("orchestrator/execution.rs"); + +include!("orchestrator/dispatch_policy.rs"); + +include!("orchestrator/prompting.rs"); + +include!("orchestrator/git_ops.rs"); + +include!("orchestrator/status.rs"); + +include!("orchestrator/selection.rs"); + +pub(crate) const DEFAULT_STATUS_RUN_LIMIT: usize = 10; +pub(crate) const DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT: usize = 25; +pub(crate) const EXTERNAL_REVIEW_ACTOR_LOGIN: &str = "codex"; +pub(crate) const EXTERNAL_REVIEW_REQUEST_BODY: &str = "@codex review"; +pub(crate) const EXTERNAL_REVIEW_PASS_PHRASE: &str = "Didn't find any major issues."; +pub(crate) const EXTERNAL_REVIEW_ACK_TIMEOUT_SECS: i64 = 60; +pub(crate) const EXTERNAL_REVIEW_MERGE_VISIBILITY_TIMEOUT_SECS: i64 = 15 * 60; + +const CONTINUATION_RETRY_DELAY_MS: u64 = 1_000; +const FAILURE_RETRY_BASE_DELAY_MS: u64 = 10_000; +const AGENT_GIT_ASKPASS_PREFIX: &str = ".decodex-git-askpass-"; +const CONTINUATION_PENDING_RUN_STATUS: &str = "continuation_pending"; +const TERMINAL_GUARDED_RUN_STATUS: &str = "terminal_guarded"; +const TERMINAL_GUARD_MARKER_FILE: &str = ".decodex-terminal-guarded"; +const TRACKER_RATE_LIMIT_BACKOFF_SECS: u64 = 15 * 60; +const TRACKER_RATE_LIMIT_WARNING: &str = "tracker_rate_limited"; +const OPERATOR_DASHBOARD_ENDPOINT_PATH: &str = "/"; +const OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH: &str = "/dashboard"; +const OPERATOR_DASHBOARD_WS_ENDPOINT_PATH: &str = "/dashboard/control"; +const OPERATOR_STATE_ENDPOINT_PATH: &str = "/state"; +const OPERATOR_LIVE_ENDPOINT_PATH: &str = "/livez"; +const OPERATOR_READY_ENDPOINT_PATH: &str = "/readyz"; +const OPERATOR_STATE_MAX_REQUEST_BYTES: usize = 8_192; +const OPERATOR_DASHBOARD_WS_CLIENT_MESSAGE_MAX_BYTES: usize = 64 * 1_024; +const OPERATOR_STATE_HEADER_TERMINATOR: &[u8] = b"\r\n\r\n"; +const OPERATOR_DASHBOARD_WS_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); +const OPERATOR_RUN_ACTIVITY_STREAM_INTERVAL: Duration = Duration::from_secs(1); +const PULL_REQUEST_REVIEW_STATE_QUERY: &str = r#" +query($owner: String!, $name: String!, $number: Int!, $reviewThreadsAfter: String) { + repository(owner: $owner, name: $name) { + mergeCommitAllowed + pullRequest(number: $number) { + url + state + isDraft + reviewDecision + mergeable + mergeStateStatus + headRefName + headRefOid + mergeCommit { + oid + } + headRepository { + name + } + headRepositoryOwner { + login + } + reactionGroups { + content + users(first: 100) { + totalCount + nodes { + login + } + } + } + comments(first: 100) { + nodes { + databaseId + body + createdAt + author { + login + } + reactionGroups { + content + users(first: 100) { + totalCount + nodes { + login + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + reviews(last: 100) { + nodes { + body + state + submittedAt + author { + login + } + } + } + reviewRequests(first: 1) { + totalCount + } + reviewThreads(first: 100, after: $reviewThreadsAfter) { + nodes { + isResolved + isOutdated + } + pageInfo { + hasNextPage + endCursor + } + } + commits(last: 1) { + nodes { + commit { + statusCheckRollup { + state + } + } + } + } + } + } +} +"#; +const PULL_REQUEST_ISSUE_COMMENTS_QUERY: &str = r#" +query($owner: String!, $name: String!, $number: Int!, $commentsAfter: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + url + comments(first: 100, after: $commentsAfter) { + nodes { + databaseId + body + createdAt + author { + login + } + reactionGroups { + content + users(first: 100) { + totalCount + nodes { + login + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} +"#; + +#[cfg(test)] mod tests; diff --git a/apps/decodex/src/orchestrator/daemon.rs b/apps/decodex/src/orchestrator/daemon.rs new file mode 100644 index 00000000..f707d727 --- /dev/null +++ b/apps/decodex/src/orchestrator/daemon.rs @@ -0,0 +1,1444 @@ +use crate::cli::AttemptRequest; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RetryEntryRetentionDecision { + Retain, + Drop, + Block, +} + +struct DaemonTickRuntimeContext<'a, T, I> { + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + worktree_manager: &'a WorktreeManager, + review_state_inspector: &'a I, +} + +fn load_daemon_tick_context( + config_path: &Path, + workflow_cache: &mut Option, +) -> Result { + let config = ServiceConfig::from_path(config_path)?; + let workflow = load_daemon_tick_workflow(&config, workflow_cache)?; + let api_key = config.tracker().resolve_api_key()?; + let tracker = LinearClient::new(api_key)?; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + Ok(DaemonTickContext { config, workflow, tracker, worktree_manager }) +} + +fn load_daemon_tick_workflow( + config: &ServiceConfig, + workflow_cache: &mut Option, +) -> Result { + let workflow_path = config.workflow_path().to_path_buf(); + let cached_same_path = workflow_cache + .as_ref() + .filter(|cached| cached.path == workflow_path) + .map(|cached| cached.document.clone()); + + match WorkflowDocument::from_path(&workflow_path) { + Ok(workflow) => { + if cached_same_path.as_ref().is_some_and(|cached| cached != &workflow) { + tracing::info!( + workflow_path = %workflow_path.display(), + "Reloaded project WORKFLOW.md for future control-plane decisions." + ); + } + + *workflow_cache = + Some(CachedWorkflowDocument { path: workflow_path, document: workflow.clone() }); + + Ok(workflow) + }, + Err(error) => + if let Some(cached_workflow) = cached_same_path { + tracing::warn!( + workflow_path = %workflow_path.display(), + ?error, + "Failed to reload project WORKFLOW.md; keeping the last known good workflow active for control-plane decisions." + ); + + Ok(cached_workflow) + } else { + Err(error) + }, + } +} + +fn run_daemon_tick( + config_path: &Path, + state_store: &StateStore, + active_children: &mut Vec, + retry_queue: &mut RetryQueue, + context: &DaemonTickContext, +) -> Result<()> { + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(context.config.github().token_env_var().to_owned()), + }; + + run_daemon_tick_with_review_state_inspector( + config_path, + state_store, + active_children, + retry_queue, + DaemonTickRuntimeContext { + tracker: &context.tracker, + project: &context.config, + workflow: &context.workflow, + worktree_manager: &context.worktree_manager, + review_state_inspector: &review_state_inspector, + }, + ) +} + +fn run_daemon_tick_with_review_state_inspector( + config_path: &Path, + state_store: &StateStore, + active_children: &mut Vec, + retry_queue: &mut RetryQueue, + context: DaemonTickRuntimeContext<'_, T, I>, +) -> Result<()> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + inspect_or_clear_active_children( + active_children, + retry_queue, + context.tracker, + context.project, + context.workflow, + state_store, + context.worktree_manager, + )?; + + if active_children.is_empty() { + recover_and_reconcile_idle_daemon_state( + context.tracker, + context.project, + context.workflow, + state_store, + context.worktree_manager, + )?; + } + + reconcile_post_review_orchestration_with_inspector( + context.tracker, + context.project, + context.workflow, + state_store, + context.review_state_inspector, + )?; + + while active_children.len() + < context.workflow.frontmatter().execution().max_concurrent_agents() as usize + { + if !spawn_next_daemon_child( + config_path, + state_store, + active_children, + retry_queue, + context.tracker, + context.project, + context.workflow, + )? { + break; + } + } + + Ok(()) +} + +fn recover_and_reconcile_idle_daemon_state( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + worktree_manager: &WorktreeManager, +) -> Result<()> +where + T: IssueTracker, +{ + let _ = recover_runtime_state_from_tracker_and_worktrees( + tracker, + project, + workflow, + state_store, + )?; + + reconcile_project_state(tracker, project, workflow, state_store, worktree_manager) +} + +fn build_operator_state_snapshot_for_publish( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + limit: usize, + warnings: &[&str], + connector_backoffs: &[OperatorConnectorBackoffStatus], +) -> Result +where + T: IssueTracker, +{ + let mut snapshot = if warnings.is_empty() { + build_control_plane_operator_status_snapshot(tracker, project, workflow, state_store, limit)? + } else { + build_operator_status_snapshot(project, state_store, limit)? + }; + + if !warnings.is_empty() { + hydrate_history_lanes_from_local_ledger(project, state_store, &mut snapshot)?; + } + + for warning in warnings { + add_operator_snapshot_warning(&mut snapshot, warning); + } + + snapshot.connector_backoffs.extend(connector_backoffs.iter().cloned()); + + if !warnings.is_empty() { + add_operator_snapshot_warning(&mut snapshot, "external_observer_status_skipped"); + } + + refresh_operator_project_summary(&mut snapshot); + + Ok(snapshot) +} + +fn operator_snapshot_ready_stale_after(poll_interval: Duration) -> Duration { + poll_interval.checked_mul(2).unwrap_or(Duration::MAX) +} + +fn inspect_or_clear_active_children( + active_children: &mut Vec, + retry_queue: &mut RetryQueue, + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + worktree_manager: &WorktreeManager, +) -> Result<()> +where + T: IssueTracker, +{ + let mut index = 0; + + while index < active_children.len() { + let child_exit_status = active_children[index].child.try_wait()?; + let child_exited = child_exit_status.is_some(); + + if child_exited && child_exit_status.is_some_and(|status| !status.success()) { + mark_run_attempt_if_active(state_store, &active_children[index].run_id, "failed")?; + } + + let child_ref = ChildRunRef { + issue_id: &active_children[index].issue_id, + run_id: &active_children[index].run_id, + attempt_number: active_children[index].attempt_number, + }; + let actions = if child_exited { + inspect_exited_daemon_child_reconciliation( + tracker, + project, + workflow, + state_store, + child_ref.issue_id, + child_ref.run_id, + )? + } else { + inspect_active_daemon_child_reconciliation( + tracker, + project, + workflow, + state_store, + ActiveChildRunContext { + child: child_ref, + workflow: &active_children[index].workflow, + dispatch_mode: active_children[index].dispatch_mode, + }, + )? + }; + + if actions.is_empty() { + if child_exited { + if child_exit_status.is_some_and(|status| status.success()) { + mark_run_attempt_if_active( + state_store, + &active_children[index].run_id, + "succeeded", + )?; + } + + let daemon_child = active_children.swap_remove(index); + let child_ref = ChildRunRef { + issue_id: &daemon_child.issue_id, + run_id: &daemon_child.run_id, + attempt_number: daemon_child.attempt_number, + }; + + clear_orphaned_daemon_child_state(state_store, child_ref, false)?; + + if let Some(exit_status) = child_exit_status { + schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue, + tracker, + project, + workflow, + state_store, + }, + child_ref, + #[cfg(test)] + "", + &daemon_child.initial_issue_state, + daemon_child.dispatch_mode, + exit_status, + )?; + } + + continue; + } + + index += 1; + + continue; + } + + let mut daemon_child = active_children.swap_remove(index); + + if daemon_child.from_retry_queue { + retry_queue.release(&daemon_child.issue_id); + } + if !child_exited { + stop_daemon_child(&mut daemon_child.child)?; + } + + apply_active_run_reconciliation(tracker, project, state_store, worktree_manager, actions)?; + } + + Ok(()) +} + +fn inspect_active_daemon_child_reconciliation( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + child_context: ActiveChildRunContext<'_>, +) -> Result> +where + T: IssueTracker, +{ + inspect_active_daemon_child_reconciliation_at( + tracker, + project, + workflow, + state_store, + child_context, + OffsetDateTime::now_utc().unix_timestamp(), + ) +} + +fn inspect_active_daemon_child_reconciliation_at( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + child_context: ActiveChildRunContext<'_>, + now_unix_epoch: i64, +) -> Result> +where + T: IssueTracker, +{ + let child = child_context.child; + let Some(issue) = refresh_issue(tracker, child.issue_id)? else { + return Ok(Vec::new()); + }; + let Some(run_attempt) = state_store.run_attempt(child.run_id)? else { + return Ok(Vec::new()); + }; + let worktree_mapping = state_store.worktree_for_issue(&issue.id)?; + let action_workflow = active_reconciliation_workflow_for_lease( + workflow, + Some(ActiveWorkflowOverride { child, workflow: child_context.workflow }), + &issue, + &run_attempt, + ); + let retained_closeout = + terminal_issue_keeps_retained_closeout( + tracker, + &issue, + project, + action_workflow, + state_store, + )?; + let completed_closeout_child = matches!(child_context.dispatch_mode, IssueDispatchMode::Closeout) + && is_terminal_issue(&issue, action_workflow); + let disposition = if !retained_closeout && !completed_closeout_child + && is_terminal_issue(&issue, action_workflow) + { + Some(ActiveRunDisposition::Terminal) + } else if !retained_closeout && !completed_closeout_child + && is_issue_nonactive_for_active_dispatch( + tracker, + &issue, + project, + action_workflow, + child_context.dispatch_mode, + )? + { + Some(ActiveRunDisposition::NonActive) + } else if let Some(idle_for) = stalled_idle_duration( + state_store, + &run_attempt, + worktree_mapping.as_ref(), + now_unix_epoch, + )? { + if retained_review_handoff_matches_run( + state_store, + &run_attempt, + worktree_mapping.as_ref(), + )? { + Some(ActiveRunDisposition::RetainedReviewComplete) + } else { + Some(ActiveRunDisposition::Stalled { idle_for }) + } + } else { + None + }; + + Ok(disposition.map_or_else(Vec::new, |disposition| { + vec![ActiveRunReconciliation { + issue: issue.clone(), + run_attempt, + worktree_mapping, + disposition, + workflow: action_workflow.clone(), + }] + })) +} + +fn clear_orphaned_daemon_child_state( + state_store: &StateStore, + child: ChildRunRef<'_>, + mark_interrupted: bool, +) -> Result<()> { + let resolved_run_attempt = resolve_child_exit_run_attempt(state_store, child)?; + + if resolved_run_attempt.is_none() { + tracing::debug!( + issue_id = child.issue_id, + run_id = child.run_id, + attempt = child.attempt_number, + "Daemon child exited without a matching recorded run attempt; skipping orphan cleanup." + ); + } + if mark_interrupted && let Some(run_attempt) = resolved_run_attempt.as_ref() { + mark_run_attempt_if_active(state_store, run_attempt.run_id(), "interrupted")?; + } + + let existing_lease = state_store.lease_for_issue(child.issue_id)?; + let issue_unowned_or_matches_run = existing_lease.as_ref().is_none_or(|lease| { + resolved_run_attempt + .as_ref() + .is_some_and(|run_attempt| lease.run_id() == run_attempt.run_id()) + || lease.run_id() == child.run_id + }); + + if existing_lease.is_some() && issue_unowned_or_matches_run { + state_store.clear_lease(child.issue_id)?; + } + if resolved_run_attempt.is_some() + && issue_unowned_or_matches_run + && let Some(mapping) = state_store.worktree_for_issue(child.issue_id)? + && !mapping.worktree_path().try_exists()? + { + tracing::debug!( + issue_id = child.issue_id, + run_id = child.run_id, + attempt = child.attempt_number, + branch = mapping.branch_name(), + worktree_path = %mapping.worktree_path().display(), + "Cleared daemon child worktree mapping after the checkout was removed." + ); + + state_store.clear_worktree(child.issue_id)?; + } + + Ok(()) +} + +fn resolve_child_exit_run_attempt( + state_store: &StateStore, + child: ChildRunRef<'_>, +) -> Result> { + state_store.run_attempt(child.run_id) +} + +fn spawn_next_daemon_child( + config_path: &Path, + state_store: &StateStore, + active_children: &mut Vec, + retry_queue: &mut RetryQueue, + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, +) -> Result +where + T: IssueTracker, +{ + let next_run = plan_next_daemon_run(retry_queue, tracker, project, workflow, state_store)?; + + match next_run { + Some((summary, from_retry_queue)) => { + if summary.dispatch_mode != IssueDispatchMode::Closeout { + ensure_project_has_no_merged_worktree_cleanup_debt(project)?; + } + + state_store.configure_dispatch_slot_root( + project.service_id(), + project.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + )?; + + if !state_store.try_acquire_lease( + project.service_id(), + &summary.issue_id, + &summary.run_id, + &summary.issue_state, + )? { + return Ok(false); + } + + let daemon_spawn_state = + materialize_daemon_spawn_state(project, workflow, state_store, &summary) + .inspect_err(|_error| { + let _ = state_store.clear_lease(&summary.issue_id); + })?; + + state_store.record_run_attempt( + &summary.run_id, + &summary.issue_id, + summary.attempt_number, + "starting", + )?; + state_store.upsert_worktree( + project.service_id(), + &summary.issue_id, + &daemon_spawn_state.worktree.branch_name, + &daemon_spawn_state.worktree.path.display().to_string(), + )?; + + let mut child = spawn_planned_daemon_child( + config_path, + state_store, + workflow, + &summary, + daemon_spawn_state.retry_budget_base, + )?; + + if let Err(error) = state::write_run_operation_marker_for_process( + &daemon_spawn_state.worktree.path, + &summary.run_id, + summary.attempt_number, + child.id(), + RUN_OPERATION_AGENT_RUN, + ) { + let _ = child.kill(); + let _ = child.wait(); + let _ = state_store.update_run_status(&summary.run_id, "failed"); + let _ = state_store.clear_lease(&summary.issue_id); + + return Err(error); + } + + state_store.update_run_status(&summary.run_id, "running")?; + + tracing::info!( + issue = summary.issue_identifier, + worktree = %daemon_spawn_state.worktree.path.display(), + retry = from_retry_queue, + "Spawned control-plane child for active issue lane." + ); + + active_children.push(DaemonRunChild { + child, + issue_id: summary.issue_id, + run_id: summary.run_id, + attempt_number: summary.attempt_number, + initial_issue_state: summary.initial_issue_state, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode: summary.dispatch_mode, + from_retry_queue, + workflow: workflow.clone(), + }); + + Ok(true) + }, + None => { + if retry_queue.is_empty() { + tracing::debug!("Daemon tick found no eligible issue."); + } else { + tracing::debug!("Daemon tick is holding a queued retry claim."); + } + + Ok(false) + }, + } +} + +fn spawn_planned_daemon_child( + config_path: &Path, + state_store: &StateStore, + workflow: &WorkflowDocument, + summary: &RunSummary, + retry_budget_base: i64, +) -> Result { + let issue_claim_handoff = + Some(state_store.clone_issue_claim_for_child(&summary.issue_id).inspect_err(|_error| { + let _ = state_store.update_run_status(&summary.run_id, "failed"); + let _ = state_store.clear_lease(&summary.issue_id); + })?); + let (dispatch_slot_handoff_file, dispatch_slot_index) = + state_store.clone_dispatch_slot_for_child(&summary.issue_id)?; + let dispatch_slot_handoff = Some(dispatch_slot_handoff_file); + let dispatch_slot_index_handoff = Some(dispatch_slot_index); + let mut child = spawn_run_once_child(SpawnRunOnceChildRequest { + config_path, + preferred_issue_id: summary.issue_id.as_str(), + preferred_issue_state: summary.issue_state.as_str(), + preferred_initial_issue_state: Some(summary.initial_issue_state.as_str()), + dispatch_mode: summary.dispatch_mode, + preferred_run_id: summary.run_id.as_str(), + preferred_attempt_number: summary.attempt_number, + preferred_retry_budget_base: retry_budget_base, + workflow, + issue_claim_handoff: issue_claim_handoff.as_ref(), + dispatch_slot_handoff: dispatch_slot_handoff.as_ref(), + dispatch_slot_index_handoff, + }) + .inspect_err(|_error| { + let _ = state_store.update_run_status(&summary.run_id, "failed"); + let _ = state_store.clear_lease(&summary.issue_id); + })?; + + state_store.release_handed_off_guards(&summary.issue_id).inspect_err(|_error| { + let _ = child.kill(); + let _ = child.wait(); + let _ = state_store.update_run_status(&summary.run_id, "failed"); + let _ = state_store.clear_lease(&summary.issue_id); + })?; + + Ok(child) +} + +fn plan_next_daemon_run( + retry_queue: &mut RetryQueue, + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result> +where + T: IssueTracker, +{ + match plan_due_retry_run(retry_queue, tracker, project, workflow, state_store)? { + RetryDispatchDecision::Dispatch(summary) => Ok(Some((*summary, true))), + RetryDispatchDecision::Blocked { excluded_issue_ids } => { + let excluded_issue_ids = + excluded_issue_ids.iter().map(String::as_str).collect::>(); + let issue_run = plan_project_issue_run_with_exclusions( + tracker, + project, + workflow, + state_store, + true, + &excluded_issue_ids, + )?; + + Ok(issue_run + .map(|issue_run| (run_summary_from_issue_run(project.service_id(), &issue_run), false))) + }, + RetryDispatchDecision::Continue => { + let issue_run = plan_project_issue_run_with_exclusions( + tracker, + project, + workflow, + state_store, + true, + &[], + )?; + + Ok(issue_run + .map(|issue_run| (run_summary_from_issue_run(project.service_id(), &issue_run), false))) + }, + } +} + +fn materialize_daemon_spawn_state( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + summary: &RunSummary, +) -> Result { + let worktree = materialize_run_summary_worktree(project, workflow, summary)?; + let retry_budget_base = + retry_budget_base_for_issue_worktree(state_store, &summary.issue_id, &worktree.path)?; + + Ok(MaterializedDaemonSpawnState { worktree, retry_budget_base }) +} + +fn materialize_run_summary_worktree( + project: &ServiceConfig, + workflow: &WorkflowDocument, + summary: &RunSummary, +) -> Result { + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + let worktree = worktree_manager.ensure_worktree_with_hooks( + &summary.issue_identifier, + false, + workflow.frontmatter().execution().workspace_hooks(), + )?; + + if worktree.path != summary.worktree_path { + eyre::bail!( + "planned worktree path `{}` diverged from materialized path `{}` for issue `{}`", + summary.worktree_path.display(), + worktree.path.display(), + summary.issue_identifier + ); + } + if worktree.branch_name != summary.branch_name { + eyre::bail!( + "planned branch `{}` diverged from materialized branch `{}` for issue `{}`", + summary.branch_name, + worktree.branch_name, + summary.issue_identifier + ); + } + + Ok(worktree) +} + +fn spawn_run_once_child(request: SpawnRunOnceChildRequest<'_>) -> Result { + let executable = env::current_exe()?; + let lease_preacquired = + request.issue_claim_handoff.is_some() || request.dispatch_slot_handoff.is_some(); + let attempt_request = AttemptRequest { + dry_run: false, + issue_id: String::from(request.preferred_issue_id), + issue_state: String::from(request.preferred_issue_state), + initial_issue_state: request.preferred_initial_issue_state.map(String::from), + lease_preacquired, + #[cfg(unix)] + issue_claim_fd: request.issue_claim_handoff.map(AsRawFd::as_raw_fd), + #[cfg(not(unix))] + issue_claim_fd: None, + #[cfg(unix)] + dispatch_slot_fd: request.dispatch_slot_handoff.map(AsRawFd::as_raw_fd), + #[cfg(not(unix))] + dispatch_slot_fd: None, + dispatch_slot_index: request.dispatch_slot_index_handoff, + dispatch_mode: request.dispatch_mode.into(), + run_id: String::from(request.preferred_run_id), + attempt_number: request.preferred_attempt_number, + retry_budget_base: request.preferred_retry_budget_base, + workflow_snapshot: request.workflow.to_markdown()?, + }; + let payload = serde_json::to_vec(&attempt_request)?; + let mut command = Command::new(executable); + + command + .args(["--config"]) + .arg(request.config_path) + .args(["_attempt", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + let mut child = command.spawn()?; + let Some(mut stdin) = child.stdin.take() else { + let _ = child.kill(); + let _ = child.wait(); + + eyre::bail!("Spawned `_attempt` child without a writable stdin handle."); + }; + + if let Err(error) = stdin.write_all(&payload) { + let _ = child.kill(); + let _ = child.wait(); + + eyre::bail!("Failed to write `_attempt` request payload: {error}"); + } + + Ok(child) +} + +fn plan_due_retry_run( + retry_queue: &mut RetryQueue, + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result +where + T: IssueTracker, +{ + let now = Instant::now(); + + loop { + let Some(first_entry) = retry_queue.next_entry().cloned() else { + return Ok(RetryDispatchDecision::Continue); + }; + + if now >= first_entry.ready_at { + break; + } + + let Some(issue) = refresh_issue(tracker, &first_entry.issue_id)? else { + clear_retry_schedule_and_release(retry_queue, state_store, &first_entry.issue_id)?; + + continue; + }; + + if matches!( + evaluate_retry_entry_retention_policy( + tracker, + &issue, + project, + workflow, + state_store, + &first_entry, + )?, + RetryEntryRetentionDecision::Drop + ) { + clear_retry_schedule_and_release(retry_queue, state_store, &first_entry.issue_id)?; + + continue; + } + + tracing::debug!( + issue_id = first_entry.issue_id, + retry_kind = ?first_entry.kind, + retry_attempt = first_entry.attempt, + "Retry queue is holding the project claim until the next retry is due." + ); + + return Ok(RetryDispatchDecision::Blocked { + excluded_issue_ids: queued_retry_issue_ids(retry_queue), + }); + } + + let mut blocked_issue_id = None; + + for entry in retry_queue.ordered_entries() { + if now < entry.ready_at { + break; + } + + let preferred_issue_state = (entry.kind == RetryKind::Continuation + && !matches!( + entry.dispatch_mode, + IssueDispatchMode::ReviewRepair | IssueDispatchMode::Closeout + )) + .then_some(workflow.frontmatter().tracker().in_progress_state()); + let Some(summary) = run_target_issue_once(TargetIssueRunContext { + tracker, + project, + workflow, + state_store, + issue_id: &entry.issue_id, + preferred_issue_state, + preferred_initial_issue_state: entry.continuation_initial_issue_state.as_deref(), + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: entry.dispatch_mode, + preferred_run_identity: None, + preferred_retry_budget_base: None, + })? + else { + if retry_entry_is_temporarily_blocked(tracker, project, workflow, state_store, &entry)? + { + blocked_issue_id.get_or_insert_with(|| entry.issue_id.clone()); + + continue; + } + + clear_retry_schedule_and_release(retry_queue, state_store, &entry.issue_id)?; + + continue; + }; + + return Ok(RetryDispatchDecision::Dispatch(Box::new(summary))); + } + + Ok(blocked_issue_id.map_or(RetryDispatchDecision::Continue, |_issue_id| { + RetryDispatchDecision::Blocked { excluded_issue_ids: queued_retry_issue_ids(retry_queue) } + })) +} + +fn queued_retry_issue_ids(retry_queue: &RetryQueue) -> Vec { + retry_queue.ordered_entries().into_iter().map(|entry| entry.issue_id).collect() +} + +fn evaluate_post_review_retention_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + dispatch_mode: IssueDispatchMode, +) -> Result +where + T: IssueTracker + ?Sized, +{ + match dispatch_mode { + IssueDispatchMode::ReviewRepair => Ok( + if issue_passes_review_repair_dispatch_policy(tracker, issue, project, workflow)? { + RetryEntryRetentionDecision::Retain + } else { + RetryEntryRetentionDecision::Drop + }, + ), + IssueDispatchMode::Closeout => Ok(match evaluate_closeout_dispatch_policy_with_inspector( + tracker, + issue, + project, + workflow, + state_store, + &GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }, + )? { + CloseoutDispatchEligibility::Eligible => RetryEntryRetentionDecision::Retain, + CloseoutDispatchEligibility::Ineligible => RetryEntryRetentionDecision::Drop, + CloseoutDispatchEligibility::Blocked(_) => RetryEntryRetentionDecision::Block, + }), + _ => Ok(RetryEntryRetentionDecision::Drop), + } +} + +fn evaluate_retry_entry_retention_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + entry: &RetryEntry, +) -> Result +where + T: IssueTracker + ?Sized, +{ + if matches!( + entry.dispatch_mode, + IssueDispatchMode::ReviewRepair | IssueDispatchMode::Closeout + ) { + if entry.dispatch_mode == IssueDispatchMode::ReviewRepair + && issue_retry_budget_exhausted(workflow, state_store, &issue.id)? + { + return Ok(RetryEntryRetentionDecision::Drop); + } + + return evaluate_post_review_retention_policy( + tracker, + issue, + project, + workflow, + state_store, + entry.dispatch_mode, + ); + } + + let preferred_issue_state = (entry.kind == RetryKind::Continuation) + .then_some(workflow.frontmatter().tracker().in_progress_state()); + + if issue_passes_retry_retention_policy( + tracker, + issue, + project, + workflow, + state_store, + RetryIssueStateHint { + preferred_issue_state, + preferred_initial_issue_state: entry.continuation_initial_issue_state.as_deref(), + }, + )? { + Ok(RetryEntryRetentionDecision::Retain) + } else { + Ok(RetryEntryRetentionDecision::Drop) + } +} + +fn retry_entry_is_temporarily_blocked( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + entry: &RetryEntry, +) -> Result +where + T: IssueTracker, +{ + let Some(issue) = refresh_issue(tracker, &entry.issue_id)? else { + return Ok(false); + }; + + match evaluate_retry_entry_retention_policy( + tracker, + &issue, + project, + workflow, + state_store, + entry, + )? { + RetryEntryRetentionDecision::Drop => return Ok(false), + RetryEntryRetentionDecision::Block => return Ok(true), + RetryEntryRetentionDecision::Retain => {}, + } + + if state_store.issue_has_active_shared_claim(project.service_id(), &entry.issue_id)? { + return Ok(true); + } + + let concurrency = ConcurrencySnapshot::new(project.service_id(), state_store)?; + + Ok(!concurrency.has_global_capacity(workflow.frontmatter().execution())) +} + +fn schedule_retry_after_child_exit( + context: ChildExitRetryContext<'_, T>, + child: ChildRunRef<'_>, + #[cfg(test)] + _retry_project_slug: &str, + initial_issue_state: &str, + dispatch_mode: IssueDispatchMode, + exit_status: ExitStatus, +) -> Result<()> +where + T: IssueTracker, +{ + let Some(run_attempt) = resolve_child_exit_run_attempt(context.state_store, child)? else { + tracing::debug!( + issue_id = child.issue_id, + run_id = child.run_id, + attempt = child.attempt_number, + "Daemon child exited without a matching recorded run attempt; skipping retry scheduling." + ); + + return Ok(()); + }; + + if !exit_status.success() { + mark_run_attempt_if_active(context.state_store, run_attempt.run_id(), "failed")?; + } + + let Some(run_attempt) = context.state_store.run_attempt(run_attempt.run_id())? else { + return Ok(()); + }; + let issue_id = run_attempt.issue_id(); + let Some(issue) = refresh_issue(context.tracker, issue_id)? else { + clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?; + + return Ok(()); + }; + let continuation_pending = + exit_status.success() && run_attempt.status() == CONTINUATION_PENDING_RUN_STATUS; + + if !exit_status.success() && run_attempt.status() != "failed" { + clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?; + + return Ok(()); + } + + let retention_decision = child_exit_retry_retention_decision( + &context, + &issue, + initial_issue_state, + dispatch_mode, + continuation_pending, + )?; + + if retention_decision == RetryEntryRetentionDecision::Drop { + clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?; + + return Ok(()); + } + + let (kind, attempt, continuation_initial_issue_state) = if continuation_pending { + ( + RetryKind::Continuation, + u32::try_from(run_attempt.attempt_number()).unwrap_or(u32::MAX).max(1), + Some(initial_issue_state.to_owned()), + ) + } else if exit_status.success() { + clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?; + + return Ok(()); + } else { + let retry_budget_attempts = + child_exit_retry_budget_attempt_count(&context, &issue, child)?; + + if retry_budget_attempts >= context.workflow.frontmatter().execution().max_attempts() { + return terminalize_exhausted_child_exit_retry( + context, + issue, + child, + initial_issue_state, + dispatch_mode, + retry_budget_attempts, + ); + } + + (RetryKind::Failure, retry_budget_attempts, None) + }; + let delay = retry_delay(kind, attempt.max(1), context.workflow); + + tracing::info!( + issue_id, + retry_kind = ?kind, + retry_attempt = attempt.max(1), + retry_delay_ms = delay.as_millis(), + "Queued retry after control-plane child exit." + ); + + let retry_ready_at_unix_epoch = OffsetDateTime::now_utc().unix_timestamp().saturating_add( + i64::try_from((delay.as_millis().saturating_add(999)) / 1_000).unwrap_or(i64::MAX), + ); + + write_retry_schedule_for_run( + context.state_store, + issue_id, + run_attempt.run_id(), + run_attempt.attempt_number(), + kind, + retry_ready_at_unix_epoch, + )?; + + context.retry_queue.upsert(RetryEntry { + issue_id: issue_id.to_owned(), + #[cfg(test)] + retry_project_slug: String::new(), + continuation_initial_issue_state, + dispatch_mode, + kind, + attempt: attempt.max(1), + ready_at: Instant::now() + delay, + }); + + Ok(()) +} + +fn child_exit_retry_retention_decision( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, + initial_issue_state: &str, + dispatch_mode: IssueDispatchMode, + continuation_pending: bool, +) -> Result +where + T: IssueTracker, +{ + if matches!( + dispatch_mode, + IssueDispatchMode::ReviewRepair | IssueDispatchMode::Closeout + ) { + return evaluate_post_review_retention_policy( + context.tracker, + issue, + context.project, + context.workflow, + context.state_store, + dispatch_mode, + ); + } + + let preferred_issue_state = continuation_pending + .then_some(context.workflow.frontmatter().tracker().in_progress_state()); + + if issue_passes_retry_retention_policy( + context.tracker, + issue, + context.project, + context.workflow, + context.state_store, + RetryIssueStateHint { + preferred_issue_state, + preferred_initial_issue_state: continuation_pending.then_some(initial_issue_state), + }, + )? { + Ok(RetryEntryRetentionDecision::Retain) + } else { + Ok(RetryEntryRetentionDecision::Drop) + } +} + +fn child_exit_retry_budget_attempt_count( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, + child: ChildRunRef<'_>, +) -> Result +where + T: IssueTracker, +{ + let state_attempts = context.state_store.retry_budget_attempt_count(&issue.id)?.max(1); + let worktree = child_exit_worktree_spec(context, issue)?; + let Some(marker) = state::read_run_activity_marker_snapshot(&worktree.path)? else { + return Ok(u32::try_from(state_attempts).unwrap_or(u32::MAX).max(1)); + }; + let marker_attempts = state::read_run_retry_budget_attempt_count(&worktree.path)?.unwrap_or(0); + let marker_is_current_child = + marker.run_id() == child.run_id && marker.attempt_number() == child.attempt_number; + let marker_attempt_is_local = context.state_store.run_attempt(marker.run_id())?.is_some(); + let retry_budget_attempts = if marker_attempts > 0 + && !marker_is_current_child + && !marker_attempt_is_local + { + marker_attempts.saturating_add(state_attempts) + } else { + marker_attempts.max(state_attempts) + }; + + Ok(u32::try_from(retry_budget_attempts).unwrap_or(u32::MAX).max(1)) +} + +fn terminalize_exhausted_child_exit_retry( + context: ChildExitRetryContext<'_, T>, + issue: TrackerIssue, + child: ChildRunRef<'_>, + initial_issue_state: &str, + dispatch_mode: IssueDispatchMode, + retry_budget_attempts: u32, +) -> Result<()> +where + T: IssueTracker, +{ + apply_child_exit_terminal_failure_writeback( + &context, + &issue, + child, + initial_issue_state, + dispatch_mode, + i64::from(retry_budget_attempts), + )?; + clear_retry_schedule_and_release( + context.retry_queue, + context.state_store, + child.issue_id, + )?; + + Ok(()) +} + +fn apply_child_exit_terminal_failure_writeback( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, + child: ChildRunRef<'_>, + initial_issue_state: &str, + dispatch_mode: IssueDispatchMode, + retry_budget_attempts: i64, +) -> Result<()> +where + T: IssueTracker, +{ + let worktree = child_exit_worktree_spec(context, issue)?; + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: initial_issue_state.to_owned(), + worktree, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode, + attempt_number: child.attempt_number, + run_id: child.run_id.to_owned(), + retry_budget_base: 0, + }; + let worktree_path = relative_worktree_path(context.project, &issue_run.worktree); + let error = if worktree_has_tracked_changes(&issue_run.worktree.path) { + Report::new(RetainedPartialProgress { + issue_identifier: issue.identifier.clone(), + run_id: child.run_id.to_owned(), + worktree_path: worktree_path.clone(), + }) + } else { + Report::msg(format!( + "Daemon child `{}` for issue `{}` exited unsuccessfully after exhausting retry budget.", + child.run_id, issue.identifier + )) + }; + let outcome = apply_terminal_failure_writeback( + context.tracker, + TerminalFailureWritebackRuntime { + service_id: context.project.service_id(), + state_store: Some(context.state_store), + }, + context.workflow, + &issue_run, + &worktree_path, + false, + &error, + )?; + + if outcome.retry_guarded_by_state { + write_terminal_guard_marker( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + )?; + + context + .state_store + .update_run_status(&issue_run.run_id, TERMINAL_GUARDED_RUN_STATUS)?; + } + + write_retry_budget_marker( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + retry_budget_attempts, + )?; + + tracing::warn!( + project_id = context.project.service_id(), + issue_id = issue.id, + issue = issue.identifier, + run_id = child.run_id, + attempt = child.attempt_number, + retry_budget_attempt = retry_budget_attempts, + branch = issue_run.worktree.branch_name, + worktree_path = %worktree_path, + error_class = outcome.error_class, + "Daemon child failed and now requires operator attention." + ); + + Ok(()) +} + +fn child_exit_worktree_spec( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, +) -> Result +where + T: IssueTracker, +{ + if let Some(mapping) = context.state_store.worktree_for_issue(&issue.id)? { + return Ok(WorktreeSpec { + branch_name: mapping.branch_name().to_owned(), + issue_identifier: issue.identifier.clone(), + path: mapping.worktree_path().to_path_buf(), + reused_existing: true, + }); + } + + let worktree_manager = WorktreeManager::new( + context.project.service_id(), + context.project.repo_root(), + context.project.worktree_root(), + ); + + Ok(worktree_manager.plan_for_issue(&issue.identifier)) +} + +fn write_retry_schedule_for_run( + state_store: &StateStore, + issue_id: &str, + run_id: &str, + attempt_number: i64, + kind: RetryKind, + retry_ready_at_unix_epoch: i64, +) -> Result<()> { + let default_kind = match kind { + RetryKind::Continuation => "continuation", + RetryKind::Failure => "failure", + }; + let retry_kind_label = preserved_retry_schedule_kind( + state_store, + issue_id, + run_id, + attempt_number, + default_kind, + )?; + + if let Some(worktree) = state_store.worktree_for_issue(issue_id)? { + state::write_run_retry_schedule( + worktree.worktree_path(), + run_id, + attempt_number, + &retry_kind_label, + retry_ready_at_unix_epoch, + )?; + } + + Ok(()) +} + +fn preserved_retry_schedule_kind( + state_store: &StateStore, + issue_id: &str, + run_id: &str, + attempt_number: i64, + default_kind: &str, +) -> Result { + let Some(worktree) = state_store.worktree_for_issue(issue_id)? else { + return Ok(default_kind.to_owned()); + }; + let Some(marker) = state::read_run_activity_marker_snapshot(worktree.worktree_path())? else { + return Ok(default_kind.to_owned()); + }; + + if marker.run_id() == run_id + && marker.attempt_number() == attempt_number + && let Some(retry_kind) = marker.retry_kind() + { + return Ok(retry_kind.to_owned()); + } + + Ok(default_kind.to_owned()) +} + +fn clear_retry_schedule_and_release( + retry_queue: &mut RetryQueue, + state_store: &StateStore, + issue_id: &str, +) -> Result<()> { + clear_worktree_retry_schedule(state_store, issue_id)?; + + retry_queue.release(issue_id); + + Ok(()) +} + +fn retry_delay(kind: RetryKind, attempt: u32, workflow: &WorkflowDocument) -> Duration { + match kind { + RetryKind::Continuation => Duration::from_millis(CONTINUATION_RETRY_DELAY_MS), + RetryKind::Failure => { + let exponent = attempt.saturating_sub(1).min(31); + let multiplier = 1_u128 << exponent; + let requested = u128::from(FAILURE_RETRY_BASE_DELAY_MS).saturating_mul(multiplier); + let capped = requested + .min(u128::from(workflow.frontmatter().execution().max_retry_backoff_ms())); + + Duration::from_millis(capped as u64) + }, + } +} + +fn stop_daemon_child(child: &mut Child) -> Result<()> { + if child.try_wait()?.is_some() { + return Ok(()); + } + + let _ = child.kill(); + let _ = child.wait(); + + Ok(()) +} diff --git a/apps/decodex/src/orchestrator/dispatch_policy.rs b/apps/decodex/src/orchestrator/dispatch_policy.rs new file mode 100644 index 00000000..2c4904b1 --- /dev/null +++ b/apps/decodex/src/orchestrator/dispatch_policy.rs @@ -0,0 +1,744 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CloseoutDispatchEligibility { + Eligible, + Ineligible, + Blocked(&'static str), +} + +fn issue_passes_dispatch_policy( + tracker: &T, + issue: &TrackerIssue, + workflow: &WorkflowDocument, + queue_label: &str, + queue_membership_confirmed_by_source: bool, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + + if tracker_policy.terminal_states().iter().any(|state| state == &issue.state.name) { + return Ok(false); + } + if !tracker_policy.startable_states().iter().any(|state| state == &issue.state.name) { + return Ok(false); + } + if issue.has_label(tracker_policy.opt_out_label()) { + return Ok(false); + } + if issue.has_label(tracker_policy.needs_attention_label()) { + return Ok(false); + } + if issue.labels_complete { + if !issue.has_label(queue_label) { + return Ok(false); + } + } else if !queue_membership_confirmed_by_source + && !tracker::issue_has_label_with_server_confirmation(tracker, issue, queue_label)? + { + return Ok(false); + } + if !todo_blocker_rule_passes(issue, workflow) { + return Ok(false); + } + if !issue_has_generic_dispatch_briefing(issue) { + return Ok(false); + } + + Ok(true) +} + +fn issue_has_generic_dispatch_briefing(issue: &TrackerIssue) -> bool { + !description_is_machine_only_fenced_block(&issue.description) +} + +fn description_is_machine_only_fenced_block(description: &str) -> bool { + let trimmed = description.trim(); + + if trimmed.is_empty() { + return false; + } + + let mut saw_fence = false; + let mut inside_fence = false; + let mut current_fence_marker = b'`'; + let mut current_fence_ticks = 0; + let mut current_fence_info = String::new(); + let mut current_fence_body = String::new(); + + for line in trimmed.lines() { + let trimmed_line = line.trim(); + + if let Some((fence_marker, fence_ticks, fence_tail)) = parse_code_fence(trimmed_line) { + if inside_fence { + if fence_marker == current_fence_marker + && fence_ticks >= current_fence_ticks + && fence_tail.is_empty() + { + if !fenced_block_is_machine_readable(¤t_fence_info, ¤t_fence_body) { + return false; + } + + inside_fence = false; + current_fence_marker = b'`'; + current_fence_ticks = 0; + + current_fence_info.clear(); + current_fence_body.clear(); + + continue; + } + } else { + saw_fence = true; + inside_fence = true; + current_fence_marker = fence_marker; + current_fence_ticks = fence_ticks; + current_fence_info = fence_tail.to_ascii_lowercase(); + + current_fence_body.clear(); + + continue; + } + } + + if inside_fence { + current_fence_body.push_str(line); + current_fence_body.push('\n'); + + continue; + } + if !inside_fence && !trimmed_line.is_empty() { + return false; + } + } + + saw_fence && !inside_fence +} + +fn parse_code_fence(line: &str) -> Option<(u8, usize, &str)> { + let first_byte = *line.as_bytes().first()?; + + if first_byte != b'`' && first_byte != b'~' { + return None; + } + + let fence_ticks = line.bytes().take_while(|byte| *byte == first_byte).count(); + + if fence_ticks < 3 { + return None; + } + + Some((first_byte, fence_ticks, line[fence_ticks..].trim())) +} + +fn fenced_block_is_machine_readable(fence_info: &str, fence_body: &str) -> bool { + if !fence_info.is_empty() && fence_info != "json" { + return false; + } + + match serde_json::from_str::(fence_body.trim()) { + Ok(payload) => payload.is_object() || payload.is_array(), + Err(_) => false, + } +} + +fn render_issue_description_for_prompt(issue: &TrackerIssue) -> String { + if issue.description.trim().is_empty() { + return String::from("(no description)"); + } + if description_is_machine_only_fenced_block(&issue.description) { + return String::from( + "(machine-only tracker description omitted; this lane requires a separate generic issue briefing surface)", + ); + } + + issue.description.clone() +} + +fn issue_passes_review_repair_dispatch_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + + Ok(issue_has_service_ownership(tracker, issue, project.service_id())? + && issue.state.name == tracker_policy.success_state() + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label())) +} + +fn issue_passes_closeout_dispatch_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + + issue_passes_closeout_dispatch_policy_with_inspector( + tracker, + issue, + project, + workflow, + state_store, + &review_state_inspector, + ) +} + +fn issue_passes_closeout_dispatch_policy_with_inspector( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> Result +where + T: IssueTracker + ?Sized, + I: PullRequestReviewStateInspector + ?Sized, +{ + Ok(matches!( + evaluate_closeout_dispatch_policy_with_inspector( + tracker, + issue, + project, + workflow, + state_store, + review_state_inspector, + )?, + CloseoutDispatchEligibility::Eligible + )) +} + +fn closeout_dispatch_block_reason( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result> +where + T: IssueTracker + ?Sized, +{ + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + + closeout_dispatch_block_reason_with_inspector( + tracker, + issue, + project, + workflow, + state_store, + &review_state_inspector, + ) +} + +fn closeout_dispatch_block_reason_with_inspector( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> Result> +where + T: IssueTracker + ?Sized, + I: PullRequestReviewStateInspector + ?Sized, +{ + Ok(match evaluate_closeout_dispatch_policy_with_inspector( + tracker, + issue, + project, + workflow, + state_store, + review_state_inspector, + )? { + CloseoutDispatchEligibility::Blocked(reason) => Some(reason), + CloseoutDispatchEligibility::Eligible | CloseoutDispatchEligibility::Ineligible => + None, + }) +} + +fn evaluate_closeout_dispatch_policy_with_inspector( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> Result +where + T: IssueTracker + ?Sized, + I: PullRequestReviewStateInspector + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + let completed_state = tracker_policy.resolved_completed_state(); + let issue_state = issue.state.name.as_str(); + + if issue.has_label(tracker_policy.opt_out_label()) + || issue.has_label(tracker_policy.needs_attention_label()) + { + return Ok(CloseoutDispatchEligibility::Ineligible); + } + if !issue_has_service_ownership(tracker, issue, project.service_id())? { + return Ok(CloseoutDispatchEligibility::Ineligible); + } + if issue_state != tracker_policy.success_state() && issue_state != completed_state { + return Ok(CloseoutDispatchEligibility::Ineligible); + } + + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + let worktree = worktree_manager.plan_for_issue(&issue.identifier); + + if !worktree.path.exists() { + return Ok(CloseoutDispatchEligibility::Ineligible); + } + + let Some(review_handoff) = + state_store.review_handoff_marker(project.service_id(), &issue.id, &worktree.branch_name)? + else { + return Ok(CloseoutDispatchEligibility::Blocked( + "missing_review_handoff_record", + )); + }; + + if review_handoff.branch_name() != worktree.branch_name { + return Ok(CloseoutDispatchEligibility::Ineligible); + } + + Ok(match retained_closeout_pr_merge_gate_with_inspector( + &worktree.path, + &worktree.branch_name, + review_handoff.pr_url(), + review_state_inspector, + )? { + RetainedCloseoutPrMergeGate::Merged => + CloseoutDispatchEligibility::Eligible, + RetainedCloseoutPrMergeGate::NotMerged => + CloseoutDispatchEligibility::Blocked("pull_request_not_merged"), + RetainedCloseoutPrMergeGate::PullRequestStateReadFailed => + CloseoutDispatchEligibility::Blocked("pull_request_state_read_failed"), + }) +} + +fn issue_passes_retry_dispatch_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + hint: RetryIssueStateHint<'_>, +) -> Result +where + T: IssueTracker + ?Sized, +{ + issue_passes_retry_retention_policy( + tracker, + issue, + project, + workflow, + state_store, + hint, + ) +} + +fn issue_passes_retry_retention_policy( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + hint: RetryIssueStateHint<'_>, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + let continuation_startable_snapshot = hint + .preferred_issue_state + .is_some_and(|state| state == tracker_policy.in_progress_state()) + && hint.preferred_initial_issue_state.is_some_and(|state| state == issue.state.name) + && tracker_policy.startable_states().iter().any(|candidate| candidate == &issue.state.name); + + Ok(issue_has_service_ownership(tracker, issue, project.service_id())? + && (issue.state.name == tracker_policy.in_progress_state() + || continuation_startable_snapshot) + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()) + && !issue_is_terminal_retry_guarded(issue, project, state_store)?) +} + +fn issue_has_service_ownership( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, +) -> Result +where + T: IssueTracker + ?Sized, +{ + tracker::issue_has_label_with_server_confirmation( + tracker, + issue, + &tracker::automation_active_label(service_id), + ) +} + +fn issue_is_terminal_retry_guarded( + issue: &TrackerIssue, + project: &ServiceConfig, + state_store: &StateStore, +) -> Result { + Ok(state_store + .latest_run_attempt_for_issue(&issue.id)? + .is_some_and(|attempt| attempt.status() == TERMINAL_GUARDED_RUN_STATUS) + || terminal_guard_marker_path(project, &issue.identifier).exists()) +} + +fn terminal_guard_marker_path(project: &ServiceConfig, issue_identifier: &str) -> PathBuf { + project.worktree_root().join(issue_identifier).join(TERMINAL_GUARD_MARKER_FILE) +} + +fn write_terminal_guard_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, +) -> Result<()> { + let marker_path = worktree_path.join(TERMINAL_GUARD_MARKER_FILE); + let marker_body = format!("run_id={run_id}\nattempt_number={attempt_number}\n"); + + fs::write(marker_path, marker_body)?; + + Ok(()) +} + +fn write_retry_budget_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + retry_budget_attempt_count: i64, +) -> Result<()> { + state::write_run_retry_budget_attempt_count( + worktree_path, + run_id, + attempt_number, + retry_budget_attempt_count, + ) +} + +fn retry_budget_base_for_issue_worktree( + state_store: &StateStore, + issue_id: &str, + worktree_path: &Path, +) -> Result { + Ok(state_store + .retry_budget_attempt_count(issue_id)? + .max(state::read_run_retry_budget_attempt_count(worktree_path)?.unwrap_or(0))) +} + +fn issue_retry_budget_exhausted( + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_id: &str, +) -> Result { + if let Some(mapping) = state_store.worktree_for_issue(issue_id)? { + return issue_retry_budget_exhausted_for_worktree( + workflow, + state_store, + issue_id, + mapping.worktree_path(), + ); + } + + let retry_budget_attempts = state_store.retry_budget_attempt_count(issue_id)?; + + Ok(retry_budget_attempts >= i64::from(workflow.frontmatter().execution().max_attempts())) +} + +fn issue_retry_budget_exhausted_for_worktree( + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_id: &str, + worktree_path: &Path, +) -> Result { + let retry_budget_attempts = + retry_budget_base_for_issue_worktree(state_store, issue_id, worktree_path)?; + + Ok(retry_budget_attempts >= i64::from(workflow.frontmatter().execution().max_attempts())) +} + +fn clear_terminal_guard_marker(worktree_path: &Path) -> Result<()> { + let marker_path = worktree_path.join(TERMINAL_GUARD_MARKER_FILE); + + match fs::remove_file(&marker_path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn clear_recovered_issue_lease( + project_id: &str, + issue_id: &str, + expected_run_id: Option<&str>, + state_store: &StateStore, +) -> Result<()> { + let Some(lease) = state_store.lease_for_issue(issue_id)? else { + return Ok(()); + }; + + if lease.project_id() != project_id { + return Ok(()); + } + if expected_run_id.is_some_and(|run_id| lease.run_id() != run_id) { + return Ok(()); + } + + state_store.clear_lease(issue_id) +} + +fn is_issue_eligible( + tracker: &T, + issue: &TrackerIssue, + project_id: &str, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let queue_label = tracker::automation_queue_label(project_id); + + if !issue_passes_dispatch_policy(tracker, issue, workflow, &queue_label, true)? { + return Ok(false); + } + + Ok(state_store.lease_for_issue(&issue.id)?.is_none()) +} + +fn todo_blocker_rule_passes(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { + if issue.state.name != "Todo" { + return true; + } + + issue.blockers.iter().all(|blocker| state_name_is_terminal(&blocker.state.name, workflow)) +} + +fn refresh_issue(tracker: &T, issue_id: &str) -> Result> +where + T: IssueTracker, +{ + let issue_ids = [issue_id.to_owned()]; + let mut refreshed_issues = tracker.refresh_issues(&issue_ids)?; + + Ok(refreshed_issues.pop()) +} + +fn is_terminal_issue(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { + state_name_is_terminal(&issue.state.name, workflow) +} + +fn state_name_is_terminal(state_name: &str, workflow: &WorkflowDocument) -> bool { + workflow.frontmatter().tracker().terminal_states().iter().any(|state| state == state_name) +} + +fn is_issue_active_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { + let tracker_policy = workflow.frontmatter().tracker(); + + issue.state.name == tracker_policy.in_progress_state() + && !issue.has_label(tracker_policy.needs_attention_label()) +} + +fn is_issue_nonactive_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { + let tracker_policy = workflow.frontmatter().tracker(); + + issue.has_label(tracker_policy.opt_out_label()) + || issue.has_label(tracker_policy.needs_attention_label()) + || (issue.state.name != tracker_policy.in_progress_state() + && !tracker_policy.startable_states().iter().any(|state| state == &issue.state.name)) +} + +fn is_issue_nonactive_for_active_dispatch( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + dispatch_mode: IssueDispatchMode, +) -> Result +where + T: IssueTracker + ?Sized, +{ + match dispatch_mode { + IssueDispatchMode::ReviewRepair => { + Ok(!issue_passes_review_repair_dispatch_policy(tracker, issue, project, workflow)?) + }, + IssueDispatchMode::Normal | IssueDispatchMode::Retry | IssueDispatchMode::Closeout => { + Ok(is_issue_nonactive_for_run(issue, workflow)) + }, + } +} + +fn mark_run_attempt_if_active( + state_store: &StateStore, + run_id: &str, + reconciled_status: &str, +) -> Result<()> { + let Some(run_attempt) = state_store.run_attempt(run_id)? else { + return Ok(()); + }; + + if matches!(run_attempt.status(), "starting" | "running") { + state_store.update_run_status(run_id, reconciled_status)?; + } + + Ok(()) +} + +fn cleanup_worktree_mapping( + state_store: &StateStore, + worktree_manager: &WorktreeManager, + workflow: &WorkflowDocument, + issue_identifier: &str, + mapping: &WorktreeMapping, +) -> Result<()> { + worktree_manager.remove_worktree_path_with_hooks( + issue_identifier, + mapping.branch_name(), + mapping.worktree_path(), + workflow.frontmatter().execution().workspace_hooks(), + )?; + state_store.clear_worktree(mapping.issue_id())?; + + Ok(()) +} + +fn cleanup_terminal_worktree( + state_store: &StateStore, + worktree_manager: &WorktreeManager, + workflow: &WorkflowDocument, + issue_id: &str, + issue_identifier: &str, + branch_name: &str, + worktree_path: &Path, +) -> Result<()> { + worktree_manager.remove_worktree_path_with_hooks( + issue_identifier, + branch_name, + worktree_path, + workflow.frontmatter().execution().workspace_hooks(), + )?; + state_store.clear_worktree(issue_id)?; + + Ok(()) +} + +fn clear_worktree_retry_schedule( + state_store: &StateStore, + issue_id: &str, +) -> Result<()> { + let Some(worktree) = state_store.worktree_for_issue(issue_id)? else { + return Ok(()); + }; + + state::clear_run_retry_schedule(worktree.worktree_path()) +} + +fn cleanup_completed_post_review_lane( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result<()> +{ + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + let review_handoff = state_store + .review_handoff_marker( + project.service_id(), + &issue_run.issue.id, + &issue_run.worktree.branch_name, + )? + .ok_or_else(|| { + eyre::eyre!( + "Retained closeout cleanup for issue `{}` requires an existing runtime review handoff.", + issue_run.issue.identifier + ) + })?; + let default_branch = + review_handoff.target_base_ref_name().ok_or_else(|| { + eyre::eyre!( + "Retained closeout cleanup for issue `{}` requires the review handoff marker to record the PR target base branch.", + issue_run.issue.identifier + ) + })?; + let github_token = project.github().resolve_token()?; + let landing_state = github::inspect_pull_request_landing_state( + &issue_run.worktree.path, + review_handoff.pr_url(), + &github_token, + )?; + + if landing_state.state != "MERGED" { + eyre::bail!( + "Retained closeout cleanup for issue `{}` requires PR `{}` to be merged, but GitHub reports `{}`.", + issue_run.issue.identifier, + review_handoff.pr_url(), + landing_state.state + ); + } + if landing_state.base_ref_name != default_branch { + eyre::bail!( + "Retained closeout cleanup for issue `{}` expected PR `{}` target branch `{}`, but GitHub reports `{}`. Re-run review handoff/repair before cleanup.", + issue_run.issue.identifier, + review_handoff.pr_url(), + default_branch, + landing_state.base_ref_name + ); + } + + let git_credentials = GitCredentialSource::new( + project.github().token_env_var(), + &github_token, + project.worktree_root(), + ); + + default_branch_sync::sync_repo_root_default_branch( + project.repo_root(), + default_branch, + Some(git_credentials), + )?; + github::delete_pull_request_head_branch_if_present( + project.repo_root(), + review_handoff.pr_url(), + &issue_run.worktree.branch_name, + &github_token, + )?; + + detach_worktree_head_from_branch_if_checked_out( + &issue_run.worktree.path, + &issue_run.worktree.branch_name, + )?; + delete_local_branch_if_present(project.repo_root(), &issue_run.worktree.branch_name)?; + + worktree_manager.remove_worktree_path_with_hooks( + &issue_run.issue.identifier, + &issue_run.worktree.branch_name, + &issue_run.worktree.path, + workflow.frontmatter().execution().workspace_hooks(), + )?; + state_store.clear_worktree(&issue_run.issue.id)?; + + Ok(()) +} diff --git a/apps/decodex/src/orchestrator/entrypoints.rs b/apps/decodex/src/orchestrator/entrypoints.rs new file mode 100644 index 00000000..2984b69a --- /dev/null +++ b/apps/decodex/src/orchestrator/entrypoints.rs @@ -0,0 +1,605 @@ +use crate::runtime; + +struct ControlPlaneProjectTick { + snapshot: Option, + project_status: Option, +} + +impl TrackerConnectorBackoff { + fn to_operator_status( + &self, + project_id: &str, + now_unix_epoch: i64, + ) -> OperatorConnectorBackoffStatus { + OperatorConnectorBackoffStatus { + project_id: project_id.to_owned(), + connector: String::from("linear"), + sync_phase: self.sync_phase.to_owned(), + quota_class: String::from("linear_graphql_api"), + reset_at: format_optional_unix_timestamp(Some(self.reset_unix_epoch)) + .unwrap_or_else(|| self.reset_unix_epoch.to_string()), + reset_unix_epoch: self.reset_unix_epoch, + reset_source: self.reset_source.to_owned(), + retry_after_seconds: self.reset_unix_epoch.saturating_sub(now_unix_epoch).max(0), + next_action: String::from( + "Wait for the reset window; keep monitoring local running lanes.", + ), + warning: String::from(TRACKER_RATE_LIMIT_WARNING), + } + } +} + +pub(crate) fn run_once(request: RunOnceRequest<'_>) -> Result<()> { + let state_store = runtime::open_runtime_store()?; + let Some(config_path) = resolve_config_path(request.config_path, &state_store)? else { + if request.dry_run { + println!("dry run: no Decodex project config supplied or registered; nothing to execute."); + + return Ok(()); + } + + eyre::bail!( + "No Decodex project config found. Pass --config or register one with `decodex project add `." + ); + }; + + runtime::register_project_config(&state_store, &config_path, true)?; + + let preferred_run_identity = match (request.preferred_run_id, request.preferred_attempt_number) + { + (Some(run_id), Some(attempt_number)) => + Some(PreferredRunIdentity { run_id, attempt_number }), + (None, None) => None, + _ => eyre::bail!("preferred run identity requires both `run_id` and `attempt_number`."), + }; + + if let Some(summary) = run_configured_cycle(RunCycleRequest { + config_path: &config_path, + state_store: &state_store, + dry_run: request.dry_run, + preferred_issue_id: request.preferred_issue_id, + preferred_issue_state: request.preferred_issue_state, + preferred_initial_issue_state: request.preferred_initial_issue_state, + preferred_lease_acquired: request.preferred_lease_acquired, + preferred_issue_claim_fd: request.preferred_issue_claim_fd, + preferred_dispatch_slot_fd: request.preferred_dispatch_slot_fd, + preferred_dispatch_slot_index: request.preferred_dispatch_slot_index, + preferred_dispatch_mode: request.preferred_dispatch_mode, + preferred_run_identity, + preferred_retry_budget_base: request.preferred_retry_budget_base, + preferred_workflow_snapshot: request.preferred_workflow_snapshot, + })? { + println!("{}", format_run_once_summary(&summary, request.dry_run)); + + return Ok(()); + } + + let config = ServiceConfig::from_path(&config_path)?; + let workflow = load_configured_cycle_workflow(&config, request.preferred_workflow_snapshot)?; + + println!("{}", format_no_eligible_issue_message(&config, &workflow)); + + Ok(()) +} + +pub(crate) fn run_control_plane(request: ServeRequest<'_>) -> Result<()> { + if request.poll_interval.is_zero() { + eyre::bail!("serve interval must be greater than zero."); + } + + validate_daemon_runtime()?; + + let state_store = Arc::new(runtime::open_runtime_store()?); + + if let Some(config_path) = request.config_path { + let Some(config_path) = resolve_config_path(Some(config_path), &state_store)? else { + eyre::bail!( + "No Decodex project config found. Pass --config or register one with `decodex project add `." + ); + }; + + runtime::register_project_config(&state_store, &config_path, true)?; + } + + let operator_state_endpoint = OperatorStateEndpoint::start( + request.listen_address, + operator_snapshot_ready_stale_after(request.poll_interval), + Arc::clone(&state_store), + )?; + let runtime_db_path = runtime::runtime_db_path()?; + let global_config_path = runtime::global_config_path()?; + let project_config_dir = runtime::project_config_dir()?; + let mut project_runtimes: HashMap = HashMap::new(); + + tracing::info!( + poll_interval_s = request.poll_interval.as_secs(), + listen_address = %operator_state_endpoint.listen_address(), + path = OPERATOR_STATE_ENDPOINT_PATH, + runtime_db_path = %runtime_db_path.display(), + global_config_path = %global_config_path.display(), + project_config_dir = %project_config_dir.display(), + "Starting Decodex control-plane poll loop." + ); + + loop { + let tick_started_at = Instant::now(); + let snapshot = run_control_plane_tick(&state_store, &mut project_runtimes)?; + + if let Err(error) = operator_state_endpoint.publish_snapshot(&snapshot) { + let _ = error; + + tracing::warn!( + "Operator snapshot publish failed; sensitive runtime details were withheld from control-plane logs." + ); + } + + sleep_until_next_tick(request.poll_interval, tick_started_at); + } +} + +pub(crate) fn print_status( + config_path: Option<&Path>, + json: bool, + limit: usize, +) -> Result<()> { + if limit == 0 { + eyre::bail!("`status --limit` must be greater than zero."); + } + + let state_store = runtime::open_runtime_store()?; + let Some(config_path) = resolve_config_path(config_path, &state_store)? else { + eyre::bail!( + "No Decodex project config found. Pass --config or register one with `decodex project add `." + ); + }; + let config = ServiceConfig::from_path(&config_path)?; + let workflow = WorkflowDocument::from_path(config.workflow_path())?; + let tracker = LinearClient::new(config.tracker().resolve_api_key()?)?; + + runtime::register_project_config(&state_store, &config_path, true)?; + + let recovered_state = recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + &state_store, + ); + let mut snapshot_warnings = Vec::new(); + + match recovered_state { + Ok(recovered_state) => + hydrate_status_snapshot_state(&config, &state_store, recovered_state)?, + Err(error) => { + let _ = error; + + tracing::warn!( + "Skipped runtime recovery for operator status; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("runtime_recovery_unavailable"); + }, + } + + let mut snapshot = + build_live_operator_status_snapshot(&tracker, &config, &workflow, &state_store, limit)?; + + for warning in snapshot_warnings { + add_operator_snapshot_warning(&mut snapshot, warning); + } + + refresh_operator_project_summary(&mut snapshot); + + if json { + println!("{}", serde_json::to_string_pretty(&snapshot)?); + } else { + print!("{}", render_operator_status(&snapshot)); + } + + Ok(()) +} + +fn run_control_plane_tick( + state_store: &StateStore, + project_runtimes: &mut HashMap, +) -> Result { + let registered_projects = state_store.list_projects()?; + + Ok(collect_control_plane_snapshot(registered_projects, |project, project_warnings| { + let runtime = project_runtimes.entry(project.service_id().to_owned()).or_default(); + + run_control_plane_project_tick(project, state_store, runtime, project_warnings) + })) +} + +fn collect_control_plane_snapshot( + registered_projects: Vec, + mut run_enabled_project_tick: F, +) -> OperatorStatusSnapshot +where + F: FnMut(&ProjectRegistration, &mut Vec<&'static str>) -> ControlPlaneProjectTick, +{ + let registered_project_count = registered_projects.len(); + let mut snapshot_warnings = Vec::new(); + let mut project_statuses = Vec::new(); + let mut project_snapshots = Vec::new(); + + if !registered_projects.iter().any(ProjectRegistration::enabled) { + snapshot_warnings.push("no_enabled_projects"); + } + + for project in registered_projects { + if !project.enabled() { + project_statuses.push(operator_project_status_from_registration(&project, 0)); + + continue; + } + + let mut project_warnings = Vec::new(); + let project_tick = run_enabled_project_tick(&project, &mut project_warnings); + + snapshot_warnings.extend(project_warnings); + + if let Some(status) = project_tick.project_status { + project_statuses.push(status); + } + if let Some(snapshot) = project_tick.snapshot { + project_snapshots.push(snapshot); + } + } + + let mut snapshot = + aggregate_control_plane_snapshot(registered_project_count, project_snapshots); + + snapshot.projects = project_statuses; + + for warning in snapshot_warnings { + add_operator_snapshot_warning(&mut snapshot, warning); + } + + snapshot +} + +fn aggregate_control_plane_snapshot( + registered_project_count: usize, + mut project_snapshots: Vec, +) -> OperatorStatusSnapshot { + if registered_project_count == 1 && project_snapshots.len() == 1 { + return project_snapshots.remove(0); + } + + let mut snapshot = empty_control_plane_snapshot(DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT); + + for project_snapshot in project_snapshots { + append_control_plane_project_snapshot(&mut snapshot, project_snapshot); + } + + snapshot +} + +fn append_control_plane_project_snapshot( + snapshot: &mut OperatorStatusSnapshot, + project_snapshot: OperatorStatusSnapshot, +) { + for warning in project_snapshot.warnings { + add_operator_snapshot_warning(snapshot, &warning); + } + + snapshot.connector_backoffs.extend(project_snapshot.connector_backoffs); + snapshot.accounts.extend(project_snapshot.accounts); + snapshot.active_runs.extend(project_snapshot.active_runs); + snapshot.recent_runs.extend(project_snapshot.recent_runs); + snapshot.history_lanes.extend(project_snapshot.history_lanes); + snapshot.queued_candidates.extend(project_snapshot.queued_candidates); + snapshot.worktrees.extend(project_snapshot.worktrees); + snapshot.post_review_lanes.extend(project_snapshot.post_review_lanes); +} + +fn run_control_plane_project_tick( + project: &ProjectRegistration, + state_store: &StateStore, + runtime: &mut ProjectDaemonRuntime, + snapshot_warnings: &mut Vec<&'static str>, +) -> ControlPlaneProjectTick { + if tracker_backoff_active(runtime, Instant::now()) { + snapshot_warnings.push(TRACKER_RATE_LIMIT_WARNING); + + return control_plane_project_local_snapshot(project, state_store, runtime, snapshot_warnings); + } + + match load_daemon_tick_context(project.config_path(), &mut runtime.workflow_cache) { + Ok(context) => + control_plane_project_snapshot(project, state_store, runtime, &context, snapshot_warnings), + Err(error) => { + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Control-plane tick context failed; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("control_plane_tick_context_failed"); + + ControlPlaneProjectTick { + snapshot: None, + project_status: Some(operator_project_status_from_registration(project, 1)), + } + }, + } +} + +fn tracker_backoff_active(runtime: &mut ProjectDaemonRuntime, now: Instant) -> bool { + if runtime.tracker_backoff.as_ref().is_some_and(|backoff| backoff.until > now) { + return true; + } + + runtime.tracker_backoff = None; + + false +} + +fn remember_tracker_backoff( + runtime: &mut ProjectDaemonRuntime, + error: &Report, + now: Instant, + sync_phase: &'static str, +) -> bool { + let Some(backoff) = tracker_rate_limit_backoff(error, now, sync_phase) else { + return false; + }; + + runtime.tracker_backoff = Some(backoff); + + true +} + +fn tracker_rate_limit_backoff( + error: &Report, + now: Instant, + sync_phase: &'static str, +) -> Option { + let message = format!("{error:#}"); + + if !message.contains("Linear connector is rate limited") { + return None; + } + + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let fallback_reset_unix_epoch = + now_unix_epoch.saturating_add(TRACKER_RATE_LIMIT_BACKOFF_SECS as i64); + let (reset_unix_epoch, reset_source) = + match parse_linear_rate_limit_reset_unix_epoch(&message) { + Some(reset_unix_epoch) if reset_unix_epoch > now_unix_epoch => + (reset_unix_epoch, "linear"), + _ => (fallback_reset_unix_epoch, "local_default"), + }; + let retry_after_seconds = reset_unix_epoch - now_unix_epoch; + let retry_after_seconds = u64::try_from(retry_after_seconds).ok()?; + + Some(TrackerConnectorBackoff { + until: now + Duration::from_secs(retry_after_seconds), + reset_unix_epoch, + reset_source, + sync_phase, + }) +} + +fn active_connector_backoff_statuses( + project_id: &str, + runtime: &ProjectDaemonRuntime, +) -> Vec { + let Some(backoff) = runtime.tracker_backoff.as_ref() else { + return Vec::new(); + }; + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + + vec![backoff.to_operator_status(project_id, now_unix_epoch)] +} + +fn parse_linear_rate_limit_reset_unix_epoch(message: &str) -> Option { + let reset = message.split("rate limited until `").nth(1)?.split('`').next()?; + + reset.parse().ok() +} + +fn control_plane_project_local_snapshot( + project: &ProjectRegistration, + state_store: &StateStore, + runtime: &mut ProjectDaemonRuntime, + snapshot_warnings: &mut Vec<&'static str>, +) -> ControlPlaneProjectTick { + match load_daemon_tick_context(project.config_path(), &mut runtime.workflow_cache) { + Ok(context) => match build_operator_state_snapshot_for_publish( + &context.tracker, + &context.config, + &context.workflow, + state_store, + DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT, + snapshot_warnings, + &active_connector_backoff_statuses(project.service_id(), runtime), + ) { + Ok(snapshot) => ControlPlaneProjectTick { + project_status: snapshot + .projects + .first() + .cloned() + .map(|status| complete_project_status(project, status)), + snapshot: Some(snapshot), + }, + Err(error) => { + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Local operator snapshot build failed; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("operator_snapshot_build_failed"); + ControlPlaneProjectTick { + snapshot: None, + project_status: Some(operator_project_status_from_registration( + project, + snapshot_warnings.len(), + )), + } + }, + }, + Err(error) => { + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Control-plane local snapshot context failed; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("control_plane_tick_context_failed"); + ControlPlaneProjectTick { + snapshot: None, + project_status: Some(operator_project_status_from_registration( + project, + snapshot_warnings.len(), + )), + } + }, + } +} + +fn control_plane_project_snapshot( + project: &ProjectRegistration, + state_store: &StateStore, + runtime: &mut ProjectDaemonRuntime, + context: &DaemonTickContext, + snapshot_warnings: &mut Vec<&'static str>, +) -> ControlPlaneProjectTick { + if let Err(error) = run_daemon_tick( + project.config_path(), + state_store, + &mut runtime.active_children, + &mut runtime.retry_queue, + context, + ) { + if remember_tracker_backoff(runtime, &error, Instant::now(), "control_plane_tick") { + snapshot_warnings.push(TRACKER_RATE_LIMIT_WARNING); + + return control_plane_project_local_snapshot( + project, + state_store, + runtime, + snapshot_warnings, + ); + } + + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Control-plane project tick failed; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("control_plane_tick_failed"); + } + + match build_operator_state_snapshot_for_publish( + &context.tracker, + &context.config, + &context.workflow, + state_store, + DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT, + snapshot_warnings, + &[], + ) { + Ok(snapshot) => { + runtime.tracker_backoff = None; + + ControlPlaneProjectTick { + project_status: snapshot + .projects + .first() + .cloned() + .map(|status| complete_project_status(project, status)), + snapshot: Some(snapshot), + } + }, + Err(error) => { + if remember_tracker_backoff(runtime, &error, Instant::now(), "operator_snapshot_refresh") { + snapshot_warnings.push(TRACKER_RATE_LIMIT_WARNING); + + return control_plane_project_local_snapshot( + project, + state_store, + runtime, + snapshot_warnings, + ); + } + + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Operator snapshot build failed; sensitive runtime details were withheld." + ); + + snapshot_warnings.push("operator_snapshot_build_failed"); + + ControlPlaneProjectTick { + snapshot: None, + project_status: Some(operator_project_status_from_registration(project, 1)), + } + }, + } +} + +fn complete_project_status( + project: &ProjectRegistration, + mut status: OperatorProjectStatus, +) -> OperatorProjectStatus { + status.config_path = project.config_path().display().to_string(); + status.enabled = project.enabled(); + + status +} + +fn empty_control_plane_snapshot(limit: usize) -> OperatorStatusSnapshot { + OperatorStatusSnapshot { + project_id: String::from("all"), + run_limit: limit, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: Vec::new(), + recent_runs: Vec::new(), + history_lanes: Vec::new(), + queued_candidates: Vec::new(), + worktrees: Vec::new(), + post_review_lanes: Vec::new(), + } +} + +fn operator_project_status_from_registration( + project: &ProjectRegistration, + warning_count: usize, +) -> OperatorProjectStatus { + OperatorProjectStatus { + project_id: project.service_id().to_owned(), + config_path: project.config_path().display().to_string(), + repo_root: project.repo_root().display().to_string(), + enabled: project.enabled(), + active_run_count: 0, + queued_candidate_count: 0, + post_review_lane_count: 0, + retained_worktree_count: 0, + waiting_lane_count: 0, + attention_count: 0, + connector_state: if project.enabled() { + if warning_count == 0 { + String::from("ok") + } else { + String::from("degraded") + } + } else { + String::from("disabled") + }, + last_activity_at: None, + warning_count, + } +} diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs new file mode 100644 index 00000000..bed24c51 --- /dev/null +++ b/apps/decodex/src/orchestrator/execution.rs @@ -0,0 +1,1327 @@ +use git_credentials::GitSigningConfig; +use agent::CodexAccountPool; +use agent::CodexAccountProvider; + +struct AgentGitCredentialEnvironment { + process_env: AppServerProcessEnv, + askpass_path: PathBuf, +} +impl AgentGitCredentialEnvironment { + fn process_env(&self) -> &AppServerProcessEnv { + &self.process_env + } +} + +impl Drop for AgentGitCredentialEnvironment { + fn drop(&mut self) { + if let Err(error) = fs::remove_file(&self.askpass_path) + && error.kind() != ErrorKind::NotFound + { + tracing::warn!( + ?error, + askpass_path = %self.askpass_path.display(), + "Failed to remove agent Git askpass helper." + ); + } + } +} + +struct TerminalFailureLifecycle<'a> { + error_class: &'a str, + next_action: &'a str, + target_state: &'a str, + worktree_path: &'a str, + manual_attention_requested: bool, +} + +struct TerminalFailureWritebackRuntime<'a> { + service_id: &'a str, + state_store: Option<&'a StateStore>, +} + +fn execute_issue_run( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: IssueRunPlan, +) -> Result +where + T: IssueTracker, +{ + tracing::info!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + worktree_path = %relative_worktree_path(project, &issue_run.worktree), + "Starting issue run." + ); + + state_store.upsert_worktree( + project.service_id(), + &issue_run.issue.id, + &issue_run.worktree.branch_name, + &issue_run.worktree.path.display().to_string(), + )?; + + let result = ensure_automation_activity_label(tracker, &issue_run.issue, project.service_id(), true) + .and_then(|_| execute_issue_run_inner(tracker, project, workflow, state_store, &issue_run)); + + state_store.clear_lease(&issue_run.issue.id)?; + + match result { + Ok(summary) => { + persist_issue_run_outcome(state_store, &issue_run.run_id, &summary)?; + + tracing::info!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + worktree_path = %relative_worktree_path(project, &issue_run.worktree), + "Completed issue run." + ); + + Ok(summary) + }, + Err(error) => { + state_store.update_run_status(&issue_run.run_id, "failed")?; + + handle_failure(tracker, project, workflow, state_store, &issue_run, &error)?; + + Err(error) + }, + } +} + +fn persist_issue_run_outcome( + state_store: &StateStore, + run_id: &str, + summary: &RunSummary, +) -> Result<()> { + state_store.update_run_status( + run_id, + if summary.continuation_pending { CONTINUATION_PENDING_RUN_STATUS } else { "succeeded" }, + ) +} + +fn resolve_resume_thread_id( + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result> { + if let Some(run_attempt) = state_store.run_attempt(&issue_run.run_id)? + && run_attempt.attempt_number() == issue_run.attempt_number + && let Some(thread_id) = run_attempt.thread_id() + { + return Ok(Some(thread_id.to_owned())); + } + + let marker = state::read_run_activity_marker_snapshot(&issue_run.worktree.path)?; + + Ok(marker + .filter(|marker| { + marker.run_id() == issue_run.run_id + && marker.attempt_number() == issue_run.attempt_number + }) + .and_then(|marker| marker.thread_id().map(str::to_owned))) +} + +fn prepare_agent_git_credentials( + project: &ServiceConfig, + run_id: &str, + worktree_path: &Path, +) -> Result { + let github_token = project.github().resolve_token().map_err(|error| { + Report::new(AgentGitCredentialsUnavailable { + run_id: run_id.to_owned(), + token_env_var: project.github().token_env_var().to_owned(), + }) + .wrap_err(error) + })?; + let askpass_path = agent_git_askpass_path(project.worktree_root(), run_id); + let signing_config = GitSigningConfig::from_local_git_config(worktree_path)?; + + git_credentials::write_github_askpass_helper(&askpass_path)?; + + Ok(AgentGitCredentialEnvironment { + process_env: AppServerProcessEnv::with_github_credentials_and_signing_config( + project.github().token_env_var().to_owned(), + github_token, + askpass_path.clone(), + signing_config, + ), + askpass_path, + }) +} + +fn agent_git_askpass_path(worktree_root: &Path, run_id: &str) -> PathBuf { + let safe_run_id = sanitize_run_id_for_path(run_id); + + worktree_root.join(format!("{AGENT_GIT_ASKPASS_PREFIX}{safe_run_id}.sh")) +} + +fn sanitize_run_id_for_path(run_id: &str) -> String { + let sanitized = run_id + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' || character == '_' { + character + } else { + '_' + } + }) + .collect::(); + + if sanitized.is_empty() { String::from("run") } else { sanitized } +} + +fn lifecycle_event_identity<'a>( + project: &'a ServiceConfig, + issue_run: &'a IssueRunPlan, +) -> records::LinearExecutionEventIdentity<'a> { + records::LinearExecutionEventIdentity { + service_id: project.service_id(), + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + } +} + +fn write_lifecycle_event( + tracker: &T, + state_store: &StateStore, + issue_id: &str, + record: &records::LinearExecutionEventRecord, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let body = format!("Decodex execution event: {}", record.event_type); + + tracker::create_linear_execution_event_comment(tracker, issue_id, &body, record)?; + + state_store.record_linear_execution_event(record)?; + + Ok(()) +} + +fn write_prepare_lifecycle_events( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let worktree_path = relative_worktree_path(project, &issue_run.worktree); + let commit_sha = worktree_head_oid(&issue_run.worktree.path)?.ok_or_else(|| { + eyre::eyre!( + "Prepared worktree `{}` for issue `{}` did not expose a HEAD commit.", + issue_run.worktree.path.display(), + issue_run.issue.identifier + ) + })?; + + write_intake_lifecycle_event(tracker, project, state_store, issue_run, &worktree_path)?; + write_lease_lifecycle_event(tracker, project, state_store, issue_run, &worktree_path)?; + + write_worktree_prepared_lifecycle_event( + tracker, + project, + state_store, + issue_run, + &worktree_path, + &commit_sha, + ) +} + +fn write_intake_lifecycle_event( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + worktree_path: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let anchor = records::stable_event_anchor(&[issue_run.dispatch_mode.as_str(), "intake"]); + let mut record = records::LinearExecutionEventRecord::new( + lifecycle_event_identity(project, issue_run), + "intake", + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(worktree_path.to_owned()); + record.summary = Some(format!( + "Decodex selected the issue for {} dispatch.", + issue_run.dispatch_mode.as_str() + )); + + write_lifecycle_event(tracker, state_store, &issue_run.issue.id, &record) +} + +fn write_lease_lifecycle_event( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + worktree_path: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let anchor = records::stable_event_anchor(&[&issue_run.worktree.branch_name]); + let mut record = records::LinearExecutionEventRecord::new( + lifecycle_event_identity(project, issue_run), + "lease_acquired", + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(worktree_path.to_owned()); + record.summary = Some(String::from("Decodex acquired the local lane lease.")); + + write_lifecycle_event(tracker, state_store, &issue_run.issue.id, &record) +} + +fn write_worktree_prepared_lifecycle_event( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + worktree_path: &str, + commit_sha: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let anchor = records::stable_event_anchor(&[&issue_run.worktree.branch_name, commit_sha]); + let mut record = records::LinearExecutionEventRecord::new( + lifecycle_event_identity(project, issue_run), + "worktree_prepared", + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(worktree_path.to_owned()); + record.commit_sha = Some(commit_sha.to_owned()); + record.summary = Some(String::from("Decodex prepared the lane worktree.")); + + write_lifecycle_event(tracker, state_store, &issue_run.issue.id, &record) +} + +fn write_agent_started_lifecycle_event( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + transport: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let worktree_path = relative_worktree_path(project, &issue_run.worktree); + let anchor = records::stable_event_anchor(&[&issue_run.worktree.branch_name, transport]); + let mut record = records::LinearExecutionEventRecord::new( + lifecycle_event_identity(project, issue_run), + "agent_started", + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(worktree_path); + record.transport = Some(transport.to_owned()); + record.summary = Some(String::from("Decodex started the lane agent.")); + + write_lifecycle_event(tracker, state_store, &issue_run.issue.id, &record) +} + +fn terminal_failure_lifecycle_event( + service_id: &str, + issue_run: &IssueRunPlan, + failure: TerminalFailureLifecycle<'_>, +) -> records::LinearExecutionEventRecord { + let event_type = if failure.manual_attention_requested { + "needs_attention" + } else { + "terminal_failure" + }; + let anchor = records::stable_event_anchor(&[ + event_type, + failure.error_class, + failure.target_state, + ]); + let mut record = records::LinearExecutionEventRecord::new( + records::LinearExecutionEventIdentity { + service_id, + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + }, + event_type, + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(failure.worktree_path.to_owned()); + record.error_class = Some(failure.error_class.to_owned()); + record.next_action = Some(failure.next_action.to_owned()); + record.blockers = Some(vec![format!("Run failed with `{}`.", failure.error_class)]); + record.evidence = Some(vec![format!( + "Attempt {} reached terminal failure handling.", + issue_run.attempt_number + )]); + record.summary = Some(String::from("Decodex run failed and needs attention.")); + record.target_state = Some(failure.target_state.to_owned()); + + if failure.manual_attention_requested { + record.terminal_path = Some(String::from("manual_attention")); + } + + record +} + +fn write_cleanup_complete_lifecycle_event( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + pr_url: Option<&str>, + commit_sha: Option<&str>, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let worktree_path = relative_worktree_path(project, &issue_run.worktree); + let anchor = records::stable_event_anchor(&[ + &issue_run.worktree.branch_name, + commit_sha.unwrap_or_default(), + "cleanup_complete", + ]); + let mut record = records::LinearExecutionEventRecord::new( + lifecycle_event_identity(project, issue_run), + "cleanup_complete", + current_timestamp(), + &anchor, + ); + + record.branch = Some(issue_run.worktree.branch_name.clone()); + record.worktree_path = Some(worktree_path); + record.cleanup_status = Some(String::from("completed")); + record.summary = Some(String::from("Decodex cleaned up the retained lane worktree.")); + record.pr_url = pr_url.map(ToOwned::to_owned); + record.commit_sha = commit_sha.map(ToOwned::to_owned); + + write_lifecycle_event(tracker, state_store, &issue_run.issue.id, &record) +} + +fn execute_issue_run_inner( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result +where + T: IssueTracker, +{ + let transport = workflow.frontmatter().agent().transport().to_owned(); + let review_context = build_review_run_context(project, state_store, issue_run)?; + let tracker_tool_bridge = TrackerToolBridge::with_run_context_and_state_store( + tracker, + &issue_run.issue, + workflow, + review_context.clone(), + state_store, + ); + + if let Some(summary) = maybe_execute_deterministic_closeout( + tracker, + project, + workflow, + state_store, + issue_run, + &tracker_tool_bridge, + &review_context, + )? { + return Ok(summary); + } + + write_git_credentials_operation_marker(issue_run); + + let agent_git_credentials = + prepare_agent_git_credentials(project, &issue_run.run_id, &issue_run.worktree.path)?; + let codex_account_pool = + project.codex().accounts().map(CodexAccountPool::from_config).transpose()?; + let closeout_review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + let continuation_guard = IssueTurnContinuationGuard { + tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow, + service_id: project.service_id(), + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + initial_issue_state: &issue_run.initial_issue_state, + #[cfg(test)] + retry_project_slug: "", + dispatch_mode: issue_run.dispatch_mode, + review_state_inspector: Some(&closeout_review_state_inspector), + }; + let decodex_tool_bridge = + DecodexToolBridge::new(&tracker_tool_bridge, build_decodex_run_context(workflow, issue_run)); + + write_agent_started_lifecycle_event(tracker, project, state_store, issue_run, &transport)?; + + let run_result = agent::execute_app_server_run( + &AppServerRunRequest { + run_id: issue_run.run_id.clone(), + issue_id: issue_run.issue.id.clone(), + attempt_number: issue_run.attempt_number, + listen: transport, + cwd: issue_run.worktree.path.display().to_string(), + developer_instructions: build_run_developer_instructions( + tracker, + project, + workflow, + state_store, + issue_run, + &review_context, + )?, + user_input: build_run_user_input( + tracker, + project, + workflow, + state_store, + issue_run, + &review_context, + ), + max_turns: workflow.frontmatter().execution().max_turns(), + timeout: ACTIVE_RUN_IDLE_TIMEOUT, + process_env: agent_git_credentials.process_env().clone(), + continuation_user_input: Some(build_continuation_user_input( + &issue_run.issue, + workflow, + issue_run.dispatch_mode, + review_context.recorded_pr_url.as_deref(), + workflow.frontmatter().tracker().success_state(), + project.codex().internal_review_mode(), + )), + activity_marker_path: Some(issue_run.worktree.path.clone()), + resume_thread_id: resolve_resume_thread_id(state_store, issue_run)?, + command_exec_health_check: None, + dynamic_tool_handler: Some(&decodex_tool_bridge), + continuation_guard: Some(&continuation_guard), + codex_account_provider: codex_account_pool + .as_ref() + .map(|pool| pool as &dyn CodexAccountProvider), + }, + state_store, + ) + .map_err(|error| { + preserve_manual_attention_request( + tracker_tool_bridge.completion_disposition(), + issue_run, + workflow, + error, + ) + })?; + + if run_result.continuation_pending { + return Ok(continuation_boundary_summary(project, workflow, issue_run, &run_result)); + } + + apply_run_completion_disposition( + tracker, + project, + workflow, + state_store, + issue_run, + &tracker_tool_bridge, + )?; + + Ok(run_summary_from_issue_run(project.service_id(), issue_run)) +} + +fn maybe_execute_deterministic_closeout( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + tracker_tool_bridge: &TrackerToolBridge<'_>, + review_context: &ReviewHandoffContext, +) -> Result> +where + T: IssueTracker, +{ + if issue_run.dispatch_mode != IssueDispatchMode::Closeout { + return Ok(None); + } + + execute_deterministic_closeout( + tracker, + project, + workflow, + state_store, + issue_run, + tracker_tool_bridge, + review_context, + )?; + + Ok(Some(run_summary_from_issue_run(project.service_id(), issue_run))) +} + +fn build_run_developer_instructions( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + review_context: &ReviewHandoffContext, +) -> Result +where + T: IssueTracker, +{ + build_developer_instructions( + tracker, + project, + workflow, + issue_run, + state_store, + review_context.recorded_pr_url.as_deref(), + ) +} + +fn build_run_user_input( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + review_context: &ReviewHandoffContext, +) -> String +where + T: IssueTracker, +{ + build_user_input( + tracker, + project, + &issue_run.issue, + workflow, + issue_run, + state_store, + review_context.recorded_pr_url.as_deref(), + ) +} + +fn build_decodex_run_context( + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, +) -> DecodexRunContext { + let execution = workflow.frontmatter().execution(); + + DecodexRunContext { + run_id: issue_run.run_id.clone(), + attempt_number: issue_run.attempt_number, + issue_id: issue_run.issue.id.clone(), + issue_identifier: issue_run.issue.identifier.clone(), + branch: issue_run.worktree.branch_name.clone(), + worktree_path: issue_run.worktree.path.display().to_string(), + max_turns: execution.max_turns(), + default_canonicalize_commands: execution.canonicalize_commands().to_vec(), + default_verify_commands: execution.verify_commands().to_vec(), + } +} + +fn write_git_credentials_operation_marker(issue_run: &IssueRunPlan) { + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_GIT_CREDENTIALS, + ); +} + +fn execute_deterministic_closeout( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + tracker_tool_bridge: &TrackerToolBridge<'_>, + review_context: &ReviewHandoffContext, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REVIEW_WRITEBACK, + ); + + let pr_url = review_context.recorded_pr_url.as_deref().ok_or_else(|| { + eyre::eyre!( + "Retained closeout run `{}` for issue `{}` requires a recorded PR URL.", + issue_run.run_id, + issue_run.issue.identifier + ) + })?; + let pull_request = tracker_tool_bridge.validate_deterministic_closeout_pr(pr_url)?; + let cleanup_commit_sha = worktree_head_oid(&issue_run.worktree.path)?; + + ensure_closeout_issue_completed_state(tracker, workflow, issue_run)?; + + tracker_tool_bridge.apply_validated_deterministic_closeout(pull_request)?; + + cleanup_completed_post_review_lane(project, workflow, state_store, issue_run)?; + write_cleanup_complete_lifecycle_event( + tracker, + project, + state_store, + issue_run, + Some(pr_url), + cleanup_commit_sha.as_deref(), + )?; + + tracker_tool_bridge.clear_closeout_issue_scope()?; + + Ok(()) +} + +fn ensure_closeout_issue_completed_state( + tracker: &T, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let tracker_policy = workflow.frontmatter().tracker(); + let completed_state = tracker_policy.resolved_completed_state(); + let mut refreshed_issues = tracker.refresh_issues(slice::from_ref(&issue_run.issue.id))?; + let current_issue = refreshed_issues.pop().unwrap_or_else(|| issue_run.issue.clone()); + + if current_issue.state.name == completed_state { + return Ok(()); + } + if current_issue.state.name != tracker_policy.success_state() { + eyre::bail!( + "Retained closeout for issue `{}` requires tracker state `{}` or `{}`, but the refreshed issue is `{}`.", + current_issue.identifier, + tracker_policy.success_state(), + completed_state, + current_issue.state.name + ); + } + + let state_id = current_issue.state_id_for_name(completed_state).ok_or_else(|| { + eyre::eyre!( + "Issue `{}` does not expose tracker state `{}` on its team.", + current_issue.identifier, + completed_state + ) + })?; + + tracker.update_issue_state(¤t_issue.id, state_id)?; + + Ok(()) +} + +fn apply_run_completion_disposition( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + tracker_tool_bridge: &TrackerToolBridge<'_>, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + match tracker_tool_bridge.completion_disposition()? { + RunCompletionDisposition::ReviewHandoff => { + validate_review_handoff_runtime(project, false)?; + + let selected_repo_gate = + select_repo_gate_for_worktree(workflow.frontmatter().execution(), &issue_run.worktree.path); + + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REPO_GATE, + ); + run_repo_gate_commands( + selected_repo_gate.canonicalize_commands(), + selected_repo_gate.verify_commands(), + &issue_run.worktree.path, + )?; + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REVIEW_WRITEBACK, + ); + + tracker_tool_bridge.apply_review_handoff().map_err(|error| { + if let Some(writeback_error) = error.downcast_ref::() + { + Report::new(ReviewHandoffNeedsAttention { + issue_identifier: writeback_error.issue_identifier.clone(), + run_id: writeback_error.run_id.clone(), + }) + .wrap_err(error) + } else { + error + } + })?; + }, + RunCompletionDisposition::ManualAttention => { + return Err(Report::new(ManualAttentionRequested { + issue_identifier: issue_run.issue.identifier.clone(), + label: workflow.frontmatter().tracker().needs_attention_label().to_owned(), + run_id: issue_run.run_id.clone(), + })); + }, + RunCompletionDisposition::ReviewRepair => { + let selected_repo_gate = + select_repo_gate_for_worktree(workflow.frontmatter().execution(), &issue_run.worktree.path); + + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REPO_GATE, + ); + run_repo_gate_commands( + selected_repo_gate.canonicalize_commands(), + selected_repo_gate.verify_commands(), + &issue_run.worktree.path, + )?; + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REVIEW_WRITEBACK, + ); + + tracker_tool_bridge.apply_review_repair()?; + }, + RunCompletionDisposition::Closeout => { + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REVIEW_WRITEBACK, + ); + + let cleanup_commit_sha = worktree_head_oid(&issue_run.worktree.path)?; + + tracker_tool_bridge.apply_closeout()?; + + cleanup_completed_post_review_lane(project, workflow, state_store, issue_run)?; + write_cleanup_complete_lifecycle_event( + tracker, + project, + state_store, + issue_run, + tracker_tool_bridge + .review_context() + .and_then(|context| context.recorded_pr_url.as_deref()), + cleanup_commit_sha.as_deref(), + )?; + + tracker_tool_bridge.clear_closeout_issue_scope()?; + }, + } + + Ok(()) +} + +fn write_run_operation_marker_best_effort( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + current_operation: &str, +) { + if let Err(error) = state::write_run_operation_marker( + worktree_path, + run_id, + attempt_number, + current_operation, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + current_operation, + worktree_path = %worktree_path.display(), + "Run operation marker write failed; continuing completion flow." + ); + } +} + +fn run_summary_from_issue_run(project_id: &str, issue_run: &IssueRunPlan) -> RunSummary { + RunSummary { + project_id: project_id.to_owned(), + issue_id: issue_run.issue.id.clone(), + issue_identifier: issue_run.issue.identifier.clone(), + issue_state: issue_run.issue_state.clone(), + initial_issue_state: issue_run.initial_issue_state.clone(), + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode: issue_run.dispatch_mode, + branch_name: issue_run.worktree.branch_name.clone(), + worktree_path: issue_run.worktree.path.clone(), + attempt_number: issue_run.attempt_number, + run_id: issue_run.run_id.clone(), + continuation_pending: false, + } +} + +fn continuation_boundary_summary( + project: &ServiceConfig, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, + run_result: &AppServerRunResult, +) -> RunSummary { + tracing::info!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + thread_id = run_result.thread_id, + turn_count = run_result.turn_count, + max_turns = workflow.frontmatter().execution().max_turns(), + "Run reached a clean continuation boundary and will rely on the next bounded re-entry." + ); + + RunSummary { continuation_pending: true, ..run_summary_from_issue_run(project.service_id(), issue_run) } +} + +fn planned_issue_state_for_dispatch( + workflow: &WorkflowDocument, + issue: &TrackerIssue, + dispatch_mode: IssueDispatchMode, + preferred_issue_state: Option<&str>, +) -> String { + match dispatch_mode { + IssueDispatchMode::Normal => + workflow.frontmatter().tracker().in_progress_state().to_owned(), + IssueDispatchMode::Retry => preferred_issue_state + .filter(|state| { + *state == workflow.frontmatter().tracker().in_progress_state() + && workflow + .frontmatter() + .tracker() + .startable_states() + .iter() + .any(|candidate| candidate == &issue.state.name) + }) + .map(|_| { + preferred_issue_state + .expect("filtered preferred issue state should exist") + .to_owned() + }) + .unwrap_or_else(|| issue.state.name.clone()), + IssueDispatchMode::ReviewRepair | IssueDispatchMode::Closeout => + issue.state.name.clone(), + } +} + +fn preserve_manual_attention_request( + completion_disposition: Result, + issue_run: &IssueRunPlan, + workflow: &WorkflowDocument, + error: Report, +) -> Report { + if matches!(completion_disposition, Ok(RunCompletionDisposition::ManualAttention)) { + return Report::new(ManualAttentionRequested { + issue_identifier: issue_run.issue.identifier.clone(), + label: workflow.frontmatter().tracker().needs_attention_label().to_owned(), + run_id: issue_run.run_id.clone(), + }) + .wrap_err(error); + } + + error +} + +fn run_failure_requires_terminal_attention(error: &Report) -> bool { + error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error + .downcast_ref::() + .is_some_and(AppServerTurnFailure::requires_operator_attention) + || error.downcast_ref::().is_some() + || error + .downcast_ref::() + .is_some_and(|repo_gate_failure| { + repo_gate_failure.disposition() == RepoGateFailureDisposition::NeedsHumanAttention + }) +} + +fn handle_failure( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + error: &Report, +) -> Result<()> +where + T: IssueTracker, +{ + let max_attempts = i64::from(workflow.frontmatter().execution().max_attempts()); + let manual_attention_requested = error.downcast_ref::().is_some(); + let requires_terminal_attention = run_failure_requires_terminal_attention(error); + let worktree_path = relative_worktree_path(project, &issue_run.worktree); + let retry_budget_attempts = + retry_budget_attempts_for_current_failure(state_store, issue_run)?; + + if !requires_terminal_attention && retry_budget_attempts < max_attempts { + let (retry_error_class, retry_next_action) = retry_comment_details(error); + + write_retry_schedule_marker_for_runtime_retry( + error, + workflow, + issue_run, + retry_budget_attempts, + )?; + + tracing::warn!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + retry_budget_attempt = retry_budget_attempts, + max_attempts, + branch = issue_run.worktree.branch_name, + worktree_path = %worktree_path, + error_class = retry_error_class, + "Run failed and remains retryable." + ); + + tracker.create_comment( + &issue_run.issue.id, + &format_retry_comment(RetryComment { + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + retry_budget_attempt_number: retry_budget_attempts, + max_attempts, + worktree_path, + branch_name: &issue_run.worktree.branch_name, + error_class: retry_error_class, + next_action: &retry_next_action, + }), + )?; + + write_retry_budget_marker( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + retry_budget_attempts, + )?; + + return Ok(()); + } + + let retained_partial_progress = retained_partial_progress_error( + error, + issue_run, + &worktree_path, + ); + let terminal_error = retained_partial_progress.as_ref().unwrap_or(error); + let outcome = apply_terminal_failure_writeback( + tracker, + TerminalFailureWritebackRuntime { + service_id: project.service_id(), + state_store: Some(state_store), + }, + workflow, + issue_run, + &worktree_path, + manual_attention_requested, + terminal_error, + )?; + + if outcome.retry_guarded_by_state { + write_terminal_guard_marker( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + )?; + + state_store.update_run_status(&issue_run.run_id, TERMINAL_GUARDED_RUN_STATUS)?; + } + + write_retry_budget_marker( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + retry_budget_attempts, + )?; + + tracing::warn!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + worktree_path = %worktree_path, + error_class = outcome.error_class, + "Run failed and now requires operator attention." + ); + + Ok(()) +} + +fn retry_budget_attempts_for_current_failure( + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result { + let state_attempts = state_store.retry_budget_attempt_count(&issue_run.issue.id)?; + let current_attempt_counts = state_store + .run_attempt(&issue_run.run_id)? + .is_some_and(|attempt| { + attempt.issue_id() == issue_run.issue.id + && matches!( + attempt.status(), + "failed" | "interrupted" | "terminal_guarded" + ) + }); + let previous_state_attempts = + state_attempts.saturating_sub(i64::from(current_attempt_counts)); + + Ok(issue_run.retry_budget_base.max(previous_state_attempts) + + i64::from(current_attempt_counts)) +} + +fn retained_partial_progress_error( + error: &Report, + issue_run: &IssueRunPlan, + worktree_path: &str, +) -> Option { + if terminal_failure_has_specific_error_class(error) + || !worktree_has_tracked_changes(&issue_run.worktree.path) + { + return None; + } + + Some(Report::new(RetainedPartialProgress { + issue_identifier: issue_run.issue.identifier.clone(), + run_id: issue_run.run_id.clone(), + worktree_path: worktree_path.to_owned(), + })) +} + +fn terminal_failure_has_specific_error_class(error: &Report) -> bool { + error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() +} + +fn write_retry_schedule_marker_for_runtime_retry( + error: &Report, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, + retry_budget_attempts: i64, +) -> Result<()> { + let Some(repo_gate_failure) = error.downcast_ref::() else { + return Ok(()); + }; + let Some(retry_kind) = repo_gate_failure.retry_schedule_kind() else { + return Ok(()); + }; + let retry_attempt = u32::try_from(retry_budget_attempts).unwrap_or(u32::MAX).max(1); + let delay = retry_delay(RetryKind::Failure, retry_attempt, workflow); + let retry_ready_at_unix_epoch = OffsetDateTime::now_utc().unix_timestamp().saturating_add( + i64::try_from((delay.as_millis().saturating_add(999)) / 1_000).unwrap_or(i64::MAX), + ); + + state::write_run_retry_schedule( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + retry_kind, + retry_ready_at_unix_epoch, + ) +} + +fn apply_terminal_failure_writeback( + tracker: &T, + runtime: TerminalFailureWritebackRuntime<'_>, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, + worktree_path: &str, + manual_attention_requested: bool, + error: &Report, +) -> Result +where + T: IssueTracker, +{ + let tracker_policy = workflow.frontmatter().tracker(); + let needs_attention_label = tracker_policy.needs_attention_label(); + let needs_attention_label_id = tracker::issue_team_label_id_with_server_confirmation( + tracker, + &issue_run.issue, + needs_attention_label, + )?; + let failure_state_name = tracker_policy.failure_state(); + let failure_state_is_startable = + tracker_policy.startable_states().iter().any(|state| state == failure_state_name); + let guard_with_nonstartable_state = + needs_attention_label_id.is_none() && failure_state_is_startable; + let terminal_failure_state_name = if guard_with_nonstartable_state { + tracker_policy.in_progress_state() + } else { + failure_state_name + }; + let failure_state_id = + issue_run.issue.state_id_for_name(terminal_failure_state_name).ok_or_else(|| { + eyre::eyre!( + "State `{}` was not found for issue `{}`.", + terminal_failure_state_name, + issue_run.issue.identifier + ) + })?; + + tracker.update_issue_state(&issue_run.issue.id, failure_state_id)?; + + let needs_attention_label_available = apply_needs_attention_label( + tracker, + issue_run, + runtime.service_id, + needs_attention_label, + needs_attention_label_id, + terminal_failure_state_name, + )?; + let recovery_gate = terminal_failure_recovery_gate( + needs_attention_label, + needs_attention_label_available, + guard_with_nonstartable_state, + tracker_policy.in_progress_state(), + ); + let (error_class, next_action) = + terminal_failure_comment_details(manual_attention_requested, error, &recovery_gate); + let comment = format_terminal_failure_comment( + &issue_run.run_id, + issue_run.attempt_number, + worktree_path.to_owned(), + &issue_run.worktree.branch_name, + error_class, + &next_action, + ); + let event = terminal_failure_lifecycle_event( + runtime.service_id, + issue_run, + TerminalFailureLifecycle { + error_class, + next_action: &next_action, + target_state: terminal_failure_state_name, + worktree_path, + manual_attention_requested, + }, + ); + + tracker::create_linear_execution_event_comment( + tracker, + &issue_run.issue.id, + &comment, + &event, + )?; + + if let Some(state_store) = runtime.state_store { + state_store.record_linear_execution_event(&event)?; + } + + Ok(TerminalFailureOutcome { + error_class, + retry_guarded_by_state: guard_with_nonstartable_state, + }) +} + +fn apply_needs_attention_label( + tracker: &T, + issue_run: &IssueRunPlan, + service_id: &str, + needs_attention_label: &str, + needs_attention_label_id: Option, + terminal_failure_state_name: &str, +) -> Result +where + T: IssueTracker, +{ + if let Some(label_id) = needs_attention_label_id.as_deref() { + if !tracker::issue_has_label_with_server_confirmation( + tracker, + &issue_run.issue, + needs_attention_label, + )? { + tracker.add_issue_labels(&issue_run.issue.id, &[label_id.to_owned()])?; + } + } else { + tracing::warn!( + label = needs_attention_label, + issue = issue_run.issue.identifier, + guard_state = terminal_failure_state_name, + "Needs-attention label was not found in the issue team; using a non-startable state guard when needed." + ); + } + + ensure_automation_activity_label(tracker, &issue_run.issue, service_id, false)?; + + Ok(needs_attention_label_id.is_some()) +} + +fn ensure_automation_activity_label( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, + present: bool, +) -> Result<()> +where + T: IssueTracker, +{ + let mut refreshed_issues = tracker.refresh_issues(slice::from_ref(&issue.id))?; + let current_issue = refreshed_issues.pop().unwrap_or_else(|| issue.clone()); + let active_label = tracker::automation_active_label(service_id); + + tracker::set_issue_label_presence(tracker, ¤t_issue, &active_label, present)?; + + Ok(()) +} diff --git a/apps/decodex/src/orchestrator/git_ops.rs b/apps/decodex/src/orchestrator/git_ops.rs new file mode 100644 index 00000000..ff398ee9 --- /dev/null +++ b/apps/decodex/src/orchestrator/git_ops.rs @@ -0,0 +1,643 @@ +mod repo_gate_failure { + use std::fmt::Formatter; + use std::fmt::Display; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub(super) enum RepoGateFailureDisposition { + ContinueRepair, + RetryAfterBackoff, + NeedsHumanAttention, + } + + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub(super) enum RepoGateFailureKind { + CanonicalizeCommandFailed, + VerifyCommandFailed, + TrackedRewritesLeft, + GitLockContention, + CommandSpawnFailed, + CleanlinessCheckFailed, + } + impl RepoGateFailureKind { + fn error_class(self) -> &'static str { + match self { + Self::CanonicalizeCommandFailed => "repo_gate_canonicalize_failed", + Self::VerifyCommandFailed => "repo_gate_verify_failed", + Self::TrackedRewritesLeft => "repo_gate_tracked_rewrites_left", + Self::GitLockContention => "repo_gate_git_lock_contention", + Self::CommandSpawnFailed => "repo_gate_command_spawn_failed", + Self::CleanlinessCheckFailed => "repo_gate_cleanliness_check_failed", + } + } + + fn disposition(self) -> RepoGateFailureDisposition { + match self { + Self::CanonicalizeCommandFailed + | Self::VerifyCommandFailed + | Self::TrackedRewritesLeft => RepoGateFailureDisposition::ContinueRepair, + Self::GitLockContention => RepoGateFailureDisposition::RetryAfterBackoff, + Self::CommandSpawnFailed | Self::CleanlinessCheckFailed => { + RepoGateFailureDisposition::NeedsHumanAttention + }, + } + } + + fn retry_next_action(self) -> &'static str { + match self { + Self::CanonicalizeCommandFailed => { + "additional agent repair is required before repo canonicalization can pass; decodex will retry automatically" + }, + Self::VerifyCommandFailed => { + "additional agent repair is required before repo verification can pass; decodex will retry automatically" + }, + Self::TrackedRewritesLeft => { + "additional agent repair is required to reconcile repo-gate tracked rewrites before handoff; decodex will retry automatically" + }, + Self::GitLockContention => { + "another Git process appears to hold `.git/index.lock`; decodex will wait briefly, refresh lane state, and retry automatically" + }, + Self::CommandSpawnFailed => { + "manual repair is required to restore repo-gate command execution" + }, + Self::CleanlinessCheckFailed => { + "manual repair is required to restore repo-gate tracked-file inspection" + }, + } + } + + fn terminal_next_action(self, recovery_gate: &str) -> String { + match self { + Self::CanonicalizeCommandFailed => format!( + "inspect the worktree, repair the repo canonicalization failure manually, {recovery_gate}" + ), + Self::VerifyCommandFailed => format!( + "inspect the worktree, repair the repo verification failure manually, {recovery_gate}" + ), + Self::TrackedRewritesLeft => format!( + "inspect the worktree, reconcile the tracked rewrites left by the repo gate manually, {recovery_gate}" + ), + Self::GitLockContention => format!( + "inspect the worktree for an active or stale `.git/index.lock` holder, clear the Git lock contention manually, {recovery_gate}" + ), + Self::CommandSpawnFailed => format!( + "inspect the repo-gate runtime in the worktree, restore command execution manually, {recovery_gate}" + ), + Self::CleanlinessCheckFailed => format!( + "inspect the repo-gate runtime in the worktree, restore tracked-file cleanliness inspection manually, {recovery_gate}" + ), + } + } + } + + #[derive(Debug)] + pub(super) struct RepoGateFailure { + kind: RepoGateFailureKind, + message: String, + } + impl RepoGateFailure { + pub(super) fn new(kind: RepoGateFailureKind, message: String) -> Self { + Self { kind, message } + } + + pub(super) fn error_class(&self) -> &'static str { + self.kind.error_class() + } + + pub(super) fn disposition(&self) -> RepoGateFailureDisposition { + self.kind.disposition() + } + + pub(super) fn retry_next_action(&self) -> &'static str { + self.kind.retry_next_action() + } + + pub(super) fn retry_schedule_kind(&self) -> Option<&'static str> { + self.kind.retry_schedule_kind() + } + + pub(super) fn terminal_next_action(&self, recovery_gate: &str) -> String { + self.kind.terminal_next_action(recovery_gate) + } + } + impl std::error::Error for RepoGateFailure {} + + impl Display for RepoGateFailure { + fn fmt( + &self, + f: &mut Formatter<'_>, + ) -> std::result::Result<(), std::fmt::Error> { + write!(f, "{}", self.message) + } + } +} + +use std::{collections::BTreeSet, process::Output}; + +use repo_gate_failure::{RepoGateFailure, RepoGateFailureDisposition, RepoGateFailureKind}; +use crate::workflow::ResolvedRepoGate; + +impl RepoGateFailureKind { + fn retry_schedule_kind(self) -> Option<&'static str> { + match self { + Self::GitLockContention => Some("git_lock_contention"), + _ => None, + } + } +} + +pub(crate) fn delete_local_branch_if_present( + repo_root: &Path, + branch_name: &str, +) -> Result<()> { + let local_ref = format!("refs/heads/{branch_name}"); + let branch_check = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["show-ref", "--verify", "--quiet", local_ref.as_str()]) + .output()?; + + if !branch_check.status.success() { + if branch_check.status.code() == Some(1) { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&branch_check.stderr); + + eyre::bail!( + "Failed to inspect retained local branch `{branch_name}` in `{}`: {}", + repo_root.display(), + stderr.trim() + ); + } + + let delete_output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["branch", "-D", branch_name]) + .output()?; + + if delete_output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&delete_output.stderr); + + if stderr.contains("not found") || stderr.contains("branch not found") { + return Ok(()); + } + + eyre::bail!( + "Failed to delete retained local branch `{branch_name}` from `{}`: {}", + repo_root.display(), + stderr.trim() + ); +} + +pub(crate) fn detach_worktree_head_from_branch_if_checked_out( + worktree_path: &Path, + branch_name: &str, +) -> Result<()> { + let head_ref = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["symbolic-ref", "--quiet", "--short", "HEAD"]) + .output()?; + + if !head_ref.status.success() { + if head_ref.status.code() == Some(1) { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&head_ref.stderr); + + eyre::bail!( + "Failed to inspect retained worktree HEAD in `{}` before local branch cleanup: {}", + worktree_path.display(), + stderr.trim() + ); + } + + let current_branch = String::from_utf8(head_ref.stdout) + .map_err(|error| { + eyre::eyre!( + "Retained worktree HEAD in `{}` is not valid UTF-8: {error}", + worktree_path.display() + ) + })? + .trim() + .to_owned(); + + if current_branch != branch_name { + return Ok(()); + } + + let detach_output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["checkout", "--quiet", "--detach"]) + .output()?; + + if detach_output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&detach_output.stderr); + + eyre::bail!( + "Failed to detach retained worktree `{}` from branch `{branch_name}` before local branch cleanup: {}", + worktree_path.display(), + stderr.trim() + ); +} + +fn repo_gate_output_text(output: &Output) -> String { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = stderr.trim(); + let stdout = stdout.trim(); + + if !stderr.is_empty() { + return stderr.to_owned(); + } + if !stdout.is_empty() { + return stdout.to_owned(); + } + + String::from("(command produced no output)") +} + +fn repo_gate_git_output_lines(output: &Output) -> BTreeSet { + let stdout = String::from_utf8_lossy(&output.stdout); + + stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect() +} + +fn repo_gate_is_git_lock_contention(output_text: &str) -> bool { + let output_text = output_text.to_ascii_lowercase(); + + output_text.contains("index.lock") + && (output_text.contains("file exists") + || output_text.contains("already exists") + || output_text.contains("another git process seems to be running")) +} + +fn repo_gate_failure_kind_for_output( + default_kind: RepoGateFailureKind, + output_text: &str, +) -> RepoGateFailureKind { + if repo_gate_is_git_lock_contention(output_text) { + RepoGateFailureKind::GitLockContention + } else { + default_kind + } +} + +fn repo_gate_shell_from_env( + shell: Option, +) -> (std::ffi::OsString, &'static str) { + if let Some(shell) = shell + && !shell.is_empty() + { + let shell_path = Path::new(&shell); + let shell_name = shell_path + .file_name() + .and_then(std::ffi::OsStr::to_str); + + if shell_name == Some("sh") { + return (std::ffi::OsString::from("/bin/sh"), "-c"); + } + if !shell_path.is_absolute() || shell_path.is_file() { + return (shell, "-lc"); + } + } + + (std::ffi::OsString::from("/bin/sh"), "-c") +} + +fn repo_gate_shell() -> (std::ffi::OsString, &'static str) { + repo_gate_shell_from_env(env::var_os("SHELL")) +} + +fn run_repo_gate_shell_command(command: &str, cwd: &Path) -> Result { + let (shell, shell_flag) = repo_gate_shell(); + + Command::new(&shell) + .arg(shell_flag) + .arg(command) + .current_dir(cwd) + .output() + .map_err(|error| { + Report::new(RepoGateFailure::new( + RepoGateFailureKind::CommandSpawnFailed, + format!( + "Failed to spawn repo gate command `{}` in `{}` via `{}` `{}`: {}", + command, + cwd.display(), + shell.to_string_lossy(), + shell_flag, + error + ), + )) + }) +} + +fn run_repo_gate_cleanliness_check_with_git( + git_binary: &std::ffi::OsStr, + cwd: &Path, +) -> Result { + Command::new(git_binary) + .arg("-C") + .arg(cwd) + .args(["status", "--porcelain", "--untracked-files=no"]) + .output() + .map_err(|error| { + Report::new(RepoGateFailure::new( + RepoGateFailureKind::CommandSpawnFailed, + format!( + "Failed to spawn repo gate tracked-file cleanliness check in `{}` via `{}`: {}", + cwd.display(), + git_binary.to_string_lossy(), + error + ), + )) + }) +} + +fn run_repo_gate_git_command( + args: &[&str], + cwd: &Path, +) -> Result { + Command::new("git") + .arg("-C") + .arg(cwd) + .args(args) + .output() + .map_err(|error| { + eyre::eyre!( + "Failed to inspect repo-gate changed-file classification in `{}` via `git {}`: {}", + cwd.display(), + args.join(" "), + error + ) + }) +} + +fn repo_gate_remote_head_ref(cwd: &Path) -> Result { + let output = + run_repo_gate_git_command(&["symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD"], cwd)?; + + if output.status.success() { + let remote_head = repo_gate_output_text(&output); + + if !remote_head.is_empty() && remote_head != "(command produced no output)" { + return Ok(remote_head); + } + } + + let remote_probe = run_repo_gate_git_command(&["ls-remote", "--symref", "origin", "HEAD"], cwd)?; + + if !remote_probe.status.success() { + eyre::bail!( + "Failed to resolve `origin/HEAD` for repo-gate changed-file classification in `{}`: {}", + cwd.display(), + repo_gate_output_text(&remote_probe) + ); + } + + let stdout = String::from_utf8_lossy(&remote_probe.stdout); + let Some(remote_head) = stdout.lines().find_map(|line| { + let line = line.trim(); + + line + .strip_prefix("ref: refs/heads/") + .and_then(|remainder| remainder.strip_suffix("\tHEAD")) + .map(|branch_name| format!("origin/{branch_name}")) + }) else { + eyre::bail!( + "Remote `origin` did not advertise a default HEAD branch for repo-gate changed-file classification in `{}`.", + cwd.display() + ); + }; + + Ok(remote_head) +} + +fn repo_gate_merge_base( + cwd: &Path, + base_ref: &str, +) -> Result { + let output = run_repo_gate_git_command(&["merge-base", "HEAD", base_ref], cwd)?; + + if !output.status.success() { + eyre::bail!( + "Failed to resolve merge-base for repo-gate changed-file classification in `{}` against `{}`: {}", + cwd.display(), + base_ref, + repo_gate_output_text(&output) + ); + } + + let merge_base = repo_gate_output_text(&output); + + if merge_base.is_empty() || merge_base == "(command produced no output)" { + eyre::bail!( + "`git merge-base` returned no revision for repo-gate changed-file classification in `{}` against `{}`.", + cwd.display(), + base_ref + ); + } + + Ok(merge_base) +} + +fn repo_gate_changed_files_for_diff_spec( + cwd: &Path, + diff_spec: &str, +) -> Result> { + let output = run_repo_gate_git_command( + &["diff", "--name-only", "--diff-filter=ACDMRTUXB", diff_spec], + cwd, + )?; + + if !output.status.success() { + eyre::bail!( + "Failed to compute repo-gate changed-file classification in `{}` for diff `{}`: {}", + cwd.display(), + diff_spec, + repo_gate_output_text(&output) + ); + } + + Ok(repo_gate_git_output_lines(&output)) +} + +fn repo_gate_changed_tracked_files(cwd: &Path) -> Result> { + let base_ref = repo_gate_remote_head_ref(cwd)?; + let merge_base = repo_gate_merge_base(cwd, &base_ref)?; + let committed_range = format!("{merge_base}..HEAD"); + let mut changed_files = repo_gate_changed_files_for_diff_spec(cwd, &committed_range)?; + + changed_files.extend(repo_gate_changed_files_for_diff_spec(cwd, "HEAD")?); + + Ok(changed_files) +} + +fn select_repo_gate_for_worktree<'a>( + execution: &'a WorkflowExecution, + cwd: &Path, +) -> ResolvedRepoGate<'a> { + if execution.gate_profiles().is_empty() { + return execution.default_repo_gate(); + } + + let changed_files = match repo_gate_changed_tracked_files(cwd) { + Ok(changed_files) => changed_files, + Err(error) => { + tracing::warn!( + repo_root = %cwd.display(), + error = %error, + "Falling back to the default full repo gate because changed-file classification was unavailable." + ); + + return execution.default_repo_gate(); + }, + }; + let selected_gate = execution.select_repo_gate_for_changed_files(&changed_files); + + if let Some(profile_name) = selected_gate.profile_name() { + tracing::info!( + repo_root = %cwd.display(), + profile_name, + changed_file_count = changed_files.len(), + "Selected a narrowed repo gate profile from changed tracked files." + ); + } + + selected_gate +} + +fn run_canonicalize_commands(commands: &[String], cwd: &Path) -> Result<()> { + for command in commands { + let output = run_repo_gate_shell_command(command, cwd)?; + + if !output.status.success() { + let output_text = repo_gate_output_text(&output); + + return Err(Report::new(RepoGateFailure::new( + repo_gate_failure_kind_for_output( + RepoGateFailureKind::CanonicalizeCommandFailed, + &output_text, + ), + format!( + "Repo canonicalize command `{}` failed in `{}`: {}", + command, + cwd.display(), + output_text + ), + ))); + } + } + + Ok(()) +} + +fn run_verify_commands(commands: &[String], cwd: &Path) -> Result<()> { + for command in commands { + let output = run_repo_gate_shell_command(command, cwd)?; + + if !output.status.success() { + let output_text = repo_gate_output_text(&output); + + return Err(Report::new(RepoGateFailure::new( + repo_gate_failure_kind_for_output( + RepoGateFailureKind::VerifyCommandFailed, + &output_text, + ), + format!( + "Repo verify command `{}` failed in `{}`: {}", + command, + cwd.display(), + output_text + ), + ))); + } + } + + Ok(()) +} + +fn ensure_repo_gate_left_no_tracked_changes(cwd: &Path, phase: &str) -> Result<()> { + let output = run_repo_gate_cleanliness_check_with_git(std::ffi::OsStr::new("git"), cwd)?; + + if !output.status.success() { + let output_text = repo_gate_output_text(&output); + + return Err(Report::new(RepoGateFailure::new( + repo_gate_failure_kind_for_output( + RepoGateFailureKind::CleanlinessCheckFailed, + &output_text, + ), + format!( + "Failed to inspect tracked-file cleanliness after repo gate {phase} in `{}`: {}", + cwd.display(), + output_text + ), + ))); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let dirty_entries = stdout.trim(); + + if !dirty_entries.is_empty() { + return Err(Report::new(RepoGateFailure::new( + RepoGateFailureKind::TrackedRewritesLeft, + format!( + "Repo gate {phase} rewrote tracked files in `{}`; commit or revert these changes before continuing:\n{}", + cwd.display(), + dirty_entries + ), + ))); + } + + Ok(()) +} + +fn run_repo_gate_commands( + canonicalize_commands: &[String], + verify_commands: &[String], + cwd: &Path, +) -> Result<()> { + run_canonicalize_commands(canonicalize_commands, cwd)?; + run_verify_commands(verify_commands, cwd)?; + ensure_repo_gate_left_no_tracked_changes(cwd, "verification")?; + + Ok(()) +} + +fn relative_worktree_path(project: &ServiceConfig, worktree: &WorktreeSpec) -> String { + relative_worktree_path_for_path(project, &worktree.path) +} + +fn relative_worktree_path_for_path(project: &ServiceConfig, worktree_path: &Path) -> String { + if let Ok(relative_path) = worktree_path.strip_prefix(project.repo_root()) { + if relative_path.as_os_str().is_empty() { + return String::from("."); + } + + return relative_path.display().to_string(); + } + if let Some(root_name) = project.worktree_root().file_name() + && let Ok(relative_path) = worktree_path.strip_prefix(project.worktree_root()) + { + return Path::new(root_name).join(relative_path).display().to_string(); + } + + worktree_path.file_name().map_or_else( + || worktree_path.display().to_string(), + |path| path.to_string_lossy().into_owned(), + ) +} diff --git a/apps/decodex/src/orchestrator/operator_dashboard.html b/apps/decodex/src/orchestrator/operator_dashboard.html new file mode 100644 index 00000000..85fe45a0 --- /dev/null +++ b/apps/decodex/src/orchestrator/operator_dashboard.html @@ -0,0 +1,8167 @@ + + + + + + + Decodex + + + + +

+
+
+
+
+

Decodex

+
+
+
+
+
+
+ + + +
+
+
+
+ +
+
+
+ Intake + 0 issues +
+
+ Running + 0 lanes +
+
+ Review + 0 PRs +
+
+ Landing + 0 PRs +
+
+
+
+ +
+
+
+ Control Plane +

Accounts · Projects

+
+
+
+ + +
+
+
+
+
+ +
+
+
+

Projects

+
+

+
+
+
+
+
+ +
+ Execution +

Running · Intake

+
+
+
+
+

Running Lanes

+
+

+
+
+
+
+
+ +
+
+
+

Intake Queue

+
+

+
+
+
+
+
+ +
+ Closeout +

Review · Recovery · History

+
+
+
+
+

Review & Landing

+
+

Waiting for snapshot

+
+
+
+
+
+
+ +
+ +
+

Recovery Worktrees

+
+
+

+ +
+
+
+
+
+
+ +
+ +
+

Run History

+
+
+

+ +
+
+
+
+
+
+
+
+ + + + + + diff --git a/apps/decodex/src/orchestrator/operator_http.rs b/apps/decodex/src/orchestrator/operator_http.rs new file mode 100644 index 00000000..0f94a341 --- /dev/null +++ b/apps/decodex/src/orchestrator/operator_http.rs @@ -0,0 +1,1288 @@ +use base64::Engine as _; +use sha1::{Digest as _, Sha1}; +use base64::engine::general_purpose::STANDARD; + +#[cfg(test)] +type DashboardRetryLauncherForTest = fn(&Path, &str) -> Result; + +const OPERATOR_DASHBOARD_HTML: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/orchestrator/operator_dashboard.html")); + +#[cfg(test)] +static DASHBOARD_RETRY_LAUNCHER_FOR_TEST: Mutex> = + Mutex::new(None); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OperatorSnapshotReadiness { + Ready, + SnapshotUnavailable, + SnapshotStale, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OperatorRequestRoute { + Dashboard, + DashboardWs, + Live, + Ready, + State, +} + +enum DashboardClientFrame { + Text(Vec), + Close, + Ping(Vec), + Pong, +} + +#[derive(Clone, Default)] +struct DashboardEventHub { + clients: Arc>>>, +} +impl DashboardEventHub { + fn subscribe(&self) -> Result> { + let (event_tx, event_rx) = mpsc::channel(); + let mut clients = self + .clients + .lock() + .map_err(|error| eyre::eyre!("Dashboard event client lock poisoned: {error}"))?; + + clients.push(event_tx); + + Ok(event_rx) + } + + fn broadcast(&self, event_type: &'static str, payload: Value) { + let Ok(mut clients) = self.clients.lock() else { + tracing::warn!("Skipped dashboard event broadcast because the client list lock is poisoned."); + + return; + }; + let event = DashboardBroadcastEvent { event_type, payload }; + + clients.retain(|client| client.send(event.clone()).is_ok()); + } + + #[cfg(test)] + fn close_clients_for_test(&self) { + if let Ok(mut clients) = self.clients.lock() { + clients.clear(); + } + } +} + +#[derive(Clone, Debug)] +struct DashboardBroadcastEvent { + event_type: &'static str, + payload: Value, +} + +#[derive(Clone, Debug, Default)] +struct DashboardClientSubscription { + project_id: Option, + issue_id: Option, + run_id: Option, +} + +#[derive(Default)] +struct DashboardWebSocketSession { + subscription: DashboardClientSubscription, +} + +#[derive(Debug, Deserialize)] +struct DashboardClientMessage { + #[serde(rename = "type")] + message_type: String, + #[serde(rename = "requestId")] + request_id: Option, + + action: Option, + #[serde(rename = "projectId")] + project_id: Option, + #[serde(rename = "issueId")] + issue_id: Option, + #[serde(rename = "runId")] + run_id: Option, +} + +struct DashboardControlAck<'a> { + request_id: Option<&'a str>, + action: &'a str, + accepted: bool, + status: &'a str, + message: &'a str, + project_id: Option<&'a str>, + issue_id: Option<&'a str>, + run_id: Option<&'a str>, + subscription: Option<&'a DashboardClientSubscription>, +} + +struct DashboardRunActivityEvent { + fingerprint: Vec, + event: DashboardBroadcastEvent, +} + +#[cfg(test)] +struct DashboardRetryLauncherGuardForTest { + previous: Option, +} +#[cfg(test)] +impl Drop for DashboardRetryLauncherGuardForTest { + fn drop(&mut self) { + let mut slot = DASHBOARD_RETRY_LAUNCHER_FOR_TEST + .lock() + .expect("dashboard retry launcher test hook should not be poisoned"); + + *slot = self.previous.take(); + } +} + +fn run_operator_state_endpoint( + listener: TcpListener, + snapshot: Arc>, + dashboard_events: DashboardEventHub, + state_store: Arc, + ready_stale_after: Duration, + shutdown_rx: Receiver<()>, +) { + loop { + if shutdown_rx.try_recv().is_ok() { + return; + } + + match listener.accept() { + Ok((stream, _peer_addr)) => { + let connection_snapshot = Arc::clone(&snapshot); + let connection_dashboard_events = dashboard_events.clone(); + let connection_state_store = Arc::clone(&state_store); + + thread::spawn(move || { + if let Err(error) = handle_operator_state_endpoint_connection( + stream, + &connection_snapshot, + &connection_dashboard_events, + &connection_state_store, + ready_stale_after, + ) { + tracing::warn!(?error, "Operator state endpoint request failed."); + } + }); + }, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(20)); + }, + Err(error) => { + tracing::warn!(?error, "Operator state endpoint accept failed."); + + thread::sleep(Duration::from_millis(50)); + }, + } + } +} + +fn handle_operator_state_endpoint_connection( + mut stream: TcpStream, + snapshot: &Arc>, + dashboard_events: &DashboardEventHub, + state_store: &Arc, + ready_stale_after: Duration, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_millis(250)))?; + stream.set_write_timeout(Some(Duration::from_millis(250)))?; + + let request = read_operator_state_request_headers(&mut stream)?; + let route = match parse_operator_state_request_route(&request) { + Ok(route) => route, + Err(response) => { + stream.write_all(&response)?; + + return Ok(()); + }, + }; + + if route == OperatorRequestRoute::DashboardWs { + handle_operator_dashboard_websocket_connection( + stream, + &request, + dashboard_events, + state_store, + )?; + + return Ok(()); + } + + let response = match route { + OperatorRequestRoute::Dashboard => { + build_operator_state_http_response_for_route( + route, + None, + None, + OperatorSnapshotReadiness::Ready, + ) + }, + OperatorRequestRoute::Live => { + build_operator_state_http_response_for_route( + route, + None, + None, + OperatorSnapshotReadiness::Ready, + ) + }, + OperatorRequestRoute::Ready => { + let last_publish_unix_epoch = snapshot + .lock() + .map_err(|error| eyre::eyre!("Operator state snapshot lock poisoned: {error}"))? + .last_publish_unix_epoch; + + build_operator_state_http_response_for_route( + route, + None, + None, + operator_snapshot_readiness( + last_publish_unix_epoch, + OffsetDateTime::now_utc().unix_timestamp(), + ready_stale_after, + ), + ) + }, + OperatorRequestRoute::State => { + let published_snapshot = snapshot + .lock() + .map_err(|error| eyre::eyre!("Operator state snapshot lock poisoned: {error}"))? + .clone(); + + build_operator_state_http_response_for_route( + route, + published_snapshot.snapshot_json.as_deref(), + published_snapshot.last_publish_unix_epoch, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + }, + OperatorRequestRoute::DashboardWs => unreachable!( + "dashboard websocket route is handled before building one-shot responses" + ), + }; + + stream.write_all(&response)?; + + Ok(()) +} + +fn handle_operator_dashboard_websocket_connection( + mut stream: TcpStream, + request: &[u8], + dashboard_events: &DashboardEventHub, + state_store: &Arc, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_millis(20)))?; + stream.set_write_timeout(Some(Duration::from_secs(2)))?; + + let response = match operator_dashboard_websocket_response_headers(request) { + Ok(response) => response, + Err(response) => { + stream.write_all(&response)?; + + return Ok(()); + }, + }; + let events = dashboard_events.subscribe()?; + let mut session = DashboardWebSocketSession::default(); + let mut client_frame_buffer = Vec::new(); + let mut last_heartbeat = Instant::now(); + + stream.write_all(&response)?; + + write_dashboard_websocket_event( + &mut stream, + "controlReady", + &dashboard_control_ready_payload(&session.subscription), + )?; + + loop { + for frame in read_dashboard_websocket_client_frames(&mut stream, &mut client_frame_buffer)? { + match frame { + DashboardClientFrame::Text(payload) => { + let response = + handle_dashboard_client_message(&mut session, state_store, &payload); + + write_dashboard_websocket_event(&mut stream, "controlAck", &response)?; + }, + DashboardClientFrame::Close => return Ok(()), + DashboardClientFrame::Ping(payload) => { + stream.write_all(&websocket_frame(0xA, &payload)?)?; + }, + DashboardClientFrame::Pong => {}, + } + } + + match events.recv_timeout(Duration::from_millis(100)) { + Ok(event) => { + if let Some(event) = + dashboard_event_for_subscription(&event, &session.subscription) + { + write_dashboard_websocket_event(&mut stream, event.event_type, &event.payload)?; + } + }, + Err(RecvTimeoutError::Timeout) => {}, + Err(RecvTimeoutError::Disconnected) => return Ok(()), + } + + if last_heartbeat.elapsed() >= OPERATOR_DASHBOARD_WS_HEARTBEAT_INTERVAL { + stream.write_all(&websocket_ping_frame())?; + + last_heartbeat = Instant::now(); + } + } +} + +fn run_operator_run_activity_websocket_broadcasts( + state_store: Arc, + dashboard_events: DashboardEventHub, + shutdown_rx: Receiver<()>, +) { + let mut last_fingerprint: Option> = None; + + loop { + match shutdown_rx.recv_timeout(OPERATOR_RUN_ACTIVITY_STREAM_INTERVAL) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => return, + Err(RecvTimeoutError::Timeout) => {}, + } + match build_operator_run_activity_event(&state_store) { + Ok(event) => { + if last_fingerprint.as_deref() == Some(event.fingerprint.as_slice()) { + continue; + } + + dashboard_events.broadcast(event.event.event_type, event.event.payload); + + last_fingerprint = Some(event.fingerprint); + }, + Err(error) => { + tracing::warn!(?error, "Skipped dashboard run activity event."); + }, + } + } +} + +fn build_operator_run_activity_event(state_store: &StateStore) -> Result { + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut active_runs = Vec::new(); + + for registration in state_store.list_projects()? { + if !registration.enabled() { + continue; + } + + let project = match ServiceConfig::from_path(registration.config_path()) { + Ok(project) => project, + Err(error) => { + tracing::debug!( + project_id = registration.service_id(), + config_path = %registration.config_path().display(), + ?error, + "Skipped dashboard run activity for an unreadable registered project." + ); + + continue; + }, + }; + + for run in state_store.list_active_runs(project.service_id())? { + let run_status = operator_run_status(&project, state_store, run, now_unix_epoch)?; + + if operator_run_counts_as_active(&run_status) { + active_runs.push(run_status); + } + } + } + + let fingerprint = serde_json::to_vec(&active_runs)?; + let payload = json!({ + "emittedAtUnixEpoch": now_unix_epoch, + "activeRuns": active_runs, + }); + + Ok(DashboardRunActivityEvent { + fingerprint, + event: DashboardBroadcastEvent { event_type: "runActivity", payload }, + }) +} + +fn write_dashboard_websocket_event( + stream: &mut TcpStream, + event_type: &'static str, + payload: &Value, +) -> Result<()> { + stream.write_all(&dashboard_websocket_message(event_type, payload)?)?; + + Ok(()) +} + +fn dashboard_websocket_message(event_type: &str, payload: &Value) -> Result> { + let message = serde_json::to_vec(&json!({ + "type": event_type, + "payload": payload, + }))?; + + websocket_frame(0x1, &message) +} + +fn websocket_frame(opcode: u8, payload: &[u8]) -> Result> { + let mut frame = Vec::with_capacity(payload.len().saturating_add(10)); + + frame.push(0x80 | opcode); + + match payload.len() { + length @ 0..=125 => frame.push(length as u8), + length @ 126..=65_535 => { + frame.push(126); + frame.extend_from_slice(&(length as u16).to_be_bytes()); + }, + length => { + frame.push(127); + + let length = u64::try_from(length) + .map_err(|error| eyre::eyre!("WebSocket frame payload length overflow: {error}"))?; + + frame.extend_from_slice(&length.to_be_bytes()); + }, + } + + frame.extend_from_slice(payload); + + Ok(frame) +} + +fn websocket_ping_frame() -> Vec { + vec![0x89, 0] +} + +fn read_dashboard_websocket_client_frames( + stream: &mut TcpStream, + buffer: &mut Vec, +) -> Result> { + let mut frames = Vec::new(); + let mut chunk = [0_u8; 2_048]; + + loop { + match stream.read(&mut chunk) { + Ok(0) => { + frames.push(DashboardClientFrame::Close); + + break; + }, + Ok(bytes_read) => { + buffer.extend_from_slice(&chunk[..bytes_read]); + + while let Some(frame) = parse_dashboard_websocket_client_frame(buffer)? { + frames.push(frame); + } + }, + Err(error) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => + { + break; + }, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error.into()), + } + } + + Ok(frames) +} + +fn parse_dashboard_websocket_client_frame( + buffer: &mut Vec, +) -> Result> { + if buffer.len() < 2 { + return Ok(None); + } + + let fin = buffer[0] & 0x80 != 0; + let opcode = buffer[0] & 0x0f; + let masked = buffer[1] & 0x80 != 0; + let payload_length_marker = buffer[1] & 0x7f; + let mut offset = 2_usize; + let payload_length = match payload_length_marker { + length @ 0..=125 => usize::from(length), + 126 => { + if buffer.len() < offset + 2 { + return Ok(None); + } + + let length = usize::from(u16::from_be_bytes([buffer[offset], buffer[offset + 1]])); + + offset += 2; + + length + }, + 127 => { + if buffer.len() < offset + 8 { + return Ok(None); + } + + let length = u64::from_be_bytes([ + buffer[offset], + buffer[offset + 1], + buffer[offset + 2], + buffer[offset + 3], + buffer[offset + 4], + buffer[offset + 5], + buffer[offset + 6], + buffer[offset + 7], + ]); + + offset += 8; + + usize::try_from(length) + .map_err(|error| eyre::eyre!("WebSocket client frame length overflow: {error}"))? + }, + _ => unreachable!("websocket payload length marker is masked to seven bits"), + }; + + if payload_length > OPERATOR_DASHBOARD_WS_CLIENT_MESSAGE_MAX_BYTES { + eyre::bail!("WebSocket client frame exceeded the dashboard message limit."); + } + if !fin { + eyre::bail!("Fragmented dashboard WebSocket messages are not supported."); + } + if !masked { + eyre::bail!("Dashboard WebSocket client frame was not masked."); + } + if buffer.len() < offset + 4 { + return Ok(None); + } + + let mask = [buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]]; + + offset += 4; + + let frame_end = offset + .checked_add(payload_length) + .ok_or_else(|| eyre::eyre!("WebSocket client frame length overflow."))?; + + if buffer.len() < frame_end { + return Ok(None); + } + + let mut payload = buffer[offset..frame_end].to_vec(); + + for (index, byte) in payload.iter_mut().enumerate() { + *byte ^= mask[index % mask.len()]; + } + + buffer.drain(..frame_end); + + let frame = match opcode { + 0x1 => DashboardClientFrame::Text(payload), + 0x8 => DashboardClientFrame::Close, + 0x9 => DashboardClientFrame::Ping(payload), + 0xA => DashboardClientFrame::Pong, + _ => return Ok(None), + }; + + Ok(Some(frame)) +} + +fn dashboard_control_ready_payload(subscription: &DashboardClientSubscription) -> Value { + json!({ + "supportedActions": [ + "subscribe", + "focus", + "clearFocus", + "pauseProject", + "resumeProject", + "retryRun", + "ack" + ], + "subscription": dashboard_subscription_payload(subscription), + "retryTransport": "local-decodex-run", + }) +} + +fn handle_dashboard_client_message( + session: &mut DashboardWebSocketSession, + state_store: &StateStore, + payload: &[u8], +) -> Value { + let message = match serde_json::from_slice::(payload) { + Ok(message) => message, + Err(error) => { + return dashboard_control_ack_value(DashboardControlAck { + request_id: None, + action: "parse", + accepted: false, + status: "invalid_message", + message: &format!("Dashboard control message was not valid JSON: {error}"), + project_id: None, + issue_id: None, + run_id: None, + subscription: Some(&session.subscription), + }); + }, + }; + let action = message + .action + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(message.message_type.as_str()) + .to_owned(); + + match message.message_type.as_str() { + "subscribe" => { + session.subscription = dashboard_subscription_from_message(&message); + + dashboard_control_ack_for_message( + session, + &message, + "subscribe", + true, + "subscribed", + "Dashboard stream subscription updated.", + ) + }, + "control" => handle_dashboard_control_action(session, state_store, &message, &action), + _ => dashboard_control_ack_for_message( + session, + &message, + &action, + false, + "unsupported_message", + "Unsupported dashboard WebSocket message type.", + ), + } +} + +fn handle_dashboard_control_action( + session: &mut DashboardWebSocketSession, + state_store: &StateStore, + message: &DashboardClientMessage, + action: &str, +) -> Value { + match action { + "focus" => dashboard_focus_control_ack(session, message, action), + "clearFocus" | "clearSubscription" => + dashboard_clear_focus_control_ack(session, message, action), + "pause" | "pauseProject" => + dashboard_project_enabled_control_ack(session, state_store, message, action, false), + "resume" | "resumeProject" => + dashboard_project_enabled_control_ack(session, state_store, message, action, true), + "retry" | "retryRun" => + dashboard_retry_control_ack(session, state_store, message, action), + "ack" | "ackNotice" => dashboard_control_ack_for_message( + session, + message, + action, + true, + "acknowledged", + "Dashboard acknowledgement recorded for this browser session only.", + ), + _ => dashboard_unsupported_control_ack(session, message, action), + } +} + +fn dashboard_focus_control_ack( + session: &mut DashboardWebSocketSession, + message: &DashboardClientMessage, + action: &str, +) -> Value { + session.subscription = dashboard_subscription_from_message(message); + + dashboard_control_ack_for_message( + session, + message, + action, + true, + "focused", + "Dashboard focus updated for this WebSocket session.", + ) +} + +fn dashboard_clear_focus_control_ack( + session: &mut DashboardWebSocketSession, + message: &DashboardClientMessage, + action: &str, +) -> Value { + session.subscription = DashboardClientSubscription::default(); + + dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted: true, + status: "focused", + message: "Dashboard focus cleared for this WebSocket session.", + project_id: None, + issue_id: None, + run_id: None, + subscription: Some(&session.subscription), + }) +} + +fn dashboard_project_enabled_control_ack( + session: &DashboardWebSocketSession, + state_store: &StateStore, + message: &DashboardClientMessage, + action: &str, + enabled: bool, +) -> Value { + let Some(project_id) = dashboard_required_project_id(message) else { + return dashboard_missing_project_control_ack(session, message, action); + }; + let result = state_store.set_project_enabled(project_id, enabled); + let (accepted, status, copy) = match (enabled, result) { + (true, Ok(())) => (true, "resumed", String::from("Project dispatch resumed.")), + (false, Ok(())) => ( + true, + "paused", + String::from("Project dispatch paused; active lanes are not killed."), + ), + (_, Err(error)) => (false, "failed", error.to_string()), + }; + + dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted, + status, + message: ©, + project_id: Some(project_id), + issue_id: message.issue_id.as_deref(), + run_id: message.run_id.as_deref(), + subscription: Some(&session.subscription), + }) +} + +fn dashboard_retry_control_ack( + session: &DashboardWebSocketSession, + state_store: &StateStore, + message: &DashboardClientMessage, + action: &str, +) -> Value { + let Some(project_id) = dashboard_required_project_id(message) else { + return dashboard_missing_project_control_ack(session, message, action); + }; + let Some(issue_id) = dashboard_required_issue_id(message) else { + return dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted: false, + status: "missing_issue", + message: "Retry requires an issue id.", + project_id: Some(project_id), + issue_id: message.issue_id.as_deref(), + run_id: message.run_id.as_deref(), + subscription: Some(&session.subscription), + }); + }; + + match spawn_dashboard_retry_run(state_store, project_id, issue_id) { + Ok(child_id) => dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted: true, + status: "retry_started", + message: &format!("Started `decodex run {issue_id}` as process {child_id}."), + project_id: Some(project_id), + issue_id: Some(issue_id), + run_id: message.run_id.as_deref(), + subscription: Some(&session.subscription), + }), + Err(error) => dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted: false, + status: "failed", + message: &error.to_string(), + project_id: Some(project_id), + issue_id: Some(issue_id), + run_id: message.run_id.as_deref(), + subscription: Some(&session.subscription), + }), + } +} + +fn dashboard_missing_project_control_ack( + session: &DashboardWebSocketSession, + message: &DashboardClientMessage, + action: &str, +) -> Value { + dashboard_control_ack_for_message( + session, + message, + action, + false, + "missing_project", + "Control action requires a project id.", + ) +} + +fn dashboard_unsupported_control_ack( + session: &DashboardWebSocketSession, + message: &DashboardClientMessage, + action: &str, +) -> Value { + dashboard_control_ack_for_message( + session, + message, + action, + false, + "unsupported_action", + "Unsupported dashboard control action.", + ) +} + +fn dashboard_control_ack_for_message( + session: &DashboardWebSocketSession, + message: &DashboardClientMessage, + action: &str, + accepted: bool, + status: &str, + copy: &str, +) -> Value { + dashboard_control_ack_value(DashboardControlAck { + request_id: message.request_id.as_deref(), + action, + accepted, + status, + message: copy, + project_id: message.project_id.as_deref(), + issue_id: message.issue_id.as_deref(), + run_id: message.run_id.as_deref(), + subscription: Some(&session.subscription), + }) +} + +fn dashboard_control_ack_value(ack: DashboardControlAck<'_>) -> Value { + json!({ + "requestId": ack.request_id, + "action": ack.action, + "accepted": ack.accepted, + "status": ack.status, + "message": ack.message, + "projectId": ack.project_id, + "issueId": ack.issue_id, + "runId": ack.run_id, + "subscription": ack.subscription.map(dashboard_subscription_payload), + }) +} + +fn dashboard_subscription_from_message( + message: &DashboardClientMessage, +) -> DashboardClientSubscription { + DashboardClientSubscription { + project_id: dashboard_clean_scope_value(message.project_id.as_deref()), + issue_id: dashboard_clean_scope_value(message.issue_id.as_deref()), + run_id: dashboard_clean_scope_value(message.run_id.as_deref()), + } +} + +fn dashboard_subscription_payload(subscription: &DashboardClientSubscription) -> Value { + json!({ + "projectId": subscription.project_id, + "issueId": subscription.issue_id, + "runId": subscription.run_id, + }) +} + +fn dashboard_required_project_id(message: &DashboardClientMessage) -> Option<&str> { + message.project_id.as_deref().map(str::trim).filter(|value| !value.is_empty()) +} + +fn dashboard_required_issue_id(message: &DashboardClientMessage) -> Option<&str> { + message.issue_id.as_deref().map(str::trim).filter(|value| !value.is_empty()) +} + +fn dashboard_clean_scope_value(value: Option<&str>) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +fn spawn_dashboard_retry_run( + state_store: &StateStore, + project_id: &str, + issue_id: &str, +) -> Result { + let registration = state_store + .list_projects()? + .into_iter() + .find(|registration| registration.service_id() == project_id) + .ok_or_else(|| eyre::eyre!("Decodex project `{project_id}` is not registered."))?; + + #[cfg(test)] + if let Some(launcher) = *DASHBOARD_RETRY_LAUNCHER_FOR_TEST + .lock() + .expect("dashboard retry launcher test hook should not be poisoned") + { + return launcher(registration.config_path(), issue_id); + } + + let mut child = Command::new(env::current_exe()?) + .arg("-c") + .arg(registration.config_path()) + .arg("run") + .arg(issue_id) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| eyre::eyre!("Failed to start dashboard retry run: {error}"))?; + let child_id = child.id(); + + thread::spawn(move || { + if let Err(error) = child.wait() { + tracing::warn!(?error, "Dashboard retry child wait failed."); + } + }); + + Ok(child_id) +} + +#[cfg(test)] +fn install_dashboard_retry_launcher_for_test( + launcher: DashboardRetryLauncherForTest, +) -> DashboardRetryLauncherGuardForTest { + let mut slot = DASHBOARD_RETRY_LAUNCHER_FOR_TEST + .lock() + .expect("dashboard retry launcher test hook should not be poisoned"); + let previous = slot.replace(launcher); + + DashboardRetryLauncherGuardForTest { previous } +} + +fn dashboard_event_for_subscription( + event: &DashboardBroadcastEvent, + subscription: &DashboardClientSubscription, +) -> Option { + if event.event_type != "runActivity" || dashboard_subscription_is_empty(subscription) { + return Some(event.clone()); + } + + let active_runs = event + .payload + .get("activeRuns") + .and_then(Value::as_array) + .map(|runs| { + runs + .iter() + .filter(|run| dashboard_run_matches_subscription(run, subscription)) + .cloned() + .collect::>() + })?; + let mut payload = event.payload.clone(); + + payload["activeRuns"] = Value::Array(active_runs); + + Some(DashboardBroadcastEvent { event_type: event.event_type, payload }) +} + +fn dashboard_subscription_is_empty(subscription: &DashboardClientSubscription) -> bool { + subscription.project_id.is_none() && subscription.issue_id.is_none() && subscription.run_id.is_none() +} + +fn dashboard_run_matches_subscription( + run: &Value, + subscription: &DashboardClientSubscription, +) -> bool { + if let Some(project_id) = subscription.project_id.as_deref() + && run.get("project_id").and_then(Value::as_str) != Some(project_id) + { + return false; + } + if let Some(issue_id) = subscription.issue_id.as_deref() + && run.get("issue_id").and_then(Value::as_str) != Some(issue_id) + { + return false; + } + if let Some(run_id) = subscription.run_id.as_deref() + && run.get("run_id").and_then(Value::as_str) != Some(run_id) + { + return false; + } + + true +} + +fn operator_dashboard_websocket_response_headers( + request: &[u8], +) -> std::result::Result, Vec> { + let request = String::from_utf8_lossy(request); + let Some(upgrade) = operator_http_header_value(&request, "Upgrade") else { + return Err(websocket_upgrade_required_response()); + }; + let Some(connection) = operator_http_header_value(&request, "Connection") else { + return Err(websocket_upgrade_required_response()); + }; + let Some(version) = operator_http_header_value(&request, "Sec-WebSocket-Version") else { + return Err(websocket_upgrade_required_response()); + }; + let Some(key) = operator_http_header_value(&request, "Sec-WebSocket-Key") else { + return Err(websocket_upgrade_required_response()); + }; + + if !upgrade.eq_ignore_ascii_case("websocket") + || !operator_http_header_contains_token(connection, "upgrade") + || version != "13" + { + return Err(websocket_upgrade_required_response()); + } + + let accept_key = websocket_accept_key(key); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept_key}\r\n\r\n" + ); + + Ok(response.into_bytes()) +} + +fn websocket_upgrade_required_response() -> Vec { + http_response_bytes_with_headers( + "426 Upgrade Required", + "text/plain; charset=utf-8", + &[("Upgrade", String::from("websocket"))], + b"websocket upgrade required", + ) +} + +fn websocket_accept_key(key: &str) -> String { + const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + + let mut hasher = Sha1::new(); + + hasher.update(key.as_bytes()); + hasher.update(WEBSOCKET_GUID.as_bytes()); + + STANDARD.encode(hasher.finalize()) +} + +fn operator_http_header_value<'a>(request: &'a str, header_name: &str) -> Option<&'a str> { + request + .lines() + .skip(1) + .take_while(|line| !line.trim().is_empty()) + .find_map(|line| { + let (name, value) = line.split_once(':')?; + + name.trim() + .eq_ignore_ascii_case(header_name) + .then(|| value.trim()) + }) +} + +fn operator_http_header_contains_token(value: &str, token: &str) -> bool { + value + .split(',') + .any(|candidate| candidate.trim().eq_ignore_ascii_case(token)) +} + +fn operator_snapshot_readiness( + last_publish_unix_epoch: Option, + now_unix_epoch: i64, + ready_stale_after: Duration, +) -> OperatorSnapshotReadiness { + let Some(last_publish_unix_epoch) = last_publish_unix_epoch else { + return OperatorSnapshotReadiness::SnapshotUnavailable; + }; + + if last_publish_unix_epoch > now_unix_epoch { + return OperatorSnapshotReadiness::SnapshotStale; + } + + let Some(snapshot_age_seconds) = now_unix_epoch.checked_sub(last_publish_unix_epoch) else { + return OperatorSnapshotReadiness::SnapshotStale; + }; + let ready_stale_after_seconds = i64::try_from(ready_stale_after.as_secs()).unwrap_or(i64::MAX); + + if snapshot_age_seconds <= ready_stale_after_seconds { + OperatorSnapshotReadiness::Ready + } else { + OperatorSnapshotReadiness::SnapshotStale + } +} + +fn read_operator_state_request_headers(stream: &mut TcpStream) -> Result> { + let mut request = Vec::with_capacity(1_024); + + loop { + if request + .windows(OPERATOR_STATE_HEADER_TERMINATOR.len()) + .any(|window| window == OPERATOR_STATE_HEADER_TERMINATOR) + { + return Ok(request); + } + if request.len() >= OPERATOR_STATE_MAX_REQUEST_BYTES { + eyre::bail!("Operator state endpoint request headers exceeded the size limit."); + } + + let mut buffer = [0_u8; 1_024]; + + match stream.read(&mut buffer) { + Ok(0) => return Ok(request), + Ok(bytes_read) => request.extend_from_slice(&buffer[..bytes_read]), + Err(error) if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => { + eyre::bail!("Timed out while reading operator state endpoint request headers."); + }, + Err(error) => return Err(error.into()), + } + } +} + +#[cfg(test)] +fn build_operator_state_http_response( + request: &[u8], + snapshot_json: Option<&[u8]>, + readiness: OperatorSnapshotReadiness, +) -> Result> { + let route = match parse_operator_state_request_route(request) { + Ok(route) => route, + Err(response) => return Ok(response), + }; + + Ok(build_operator_state_http_response_for_route( + route, + snapshot_json, + None, + readiness, + )) +} + +fn build_operator_state_http_response_for_route( + route: OperatorRequestRoute, + snapshot_json: Option<&[u8]>, + snapshot_last_publish_unix_epoch: Option, + readiness: OperatorSnapshotReadiness, +) -> Vec { + match route { + OperatorRequestRoute::Dashboard => { + http_response_bytes("200 OK", "text/html; charset=utf-8", OPERATOR_DASHBOARD_HTML.as_bytes()) + }, + OperatorRequestRoute::DashboardWs => websocket_upgrade_required_response(), + OperatorRequestRoute::Live => { + http_response_bytes("200 OK", "text/plain; charset=utf-8", b"ok") + }, + OperatorRequestRoute::Ready => match readiness { + OperatorSnapshotReadiness::Ready => { + http_response_bytes("200 OK", "text/plain; charset=utf-8", b"ready") + }, + OperatorSnapshotReadiness::SnapshotUnavailable => http_response_bytes( + "503 Service Unavailable", + "text/plain; charset=utf-8", + b"snapshot_unavailable", + ), + OperatorSnapshotReadiness::SnapshotStale => http_response_bytes( + "503 Service Unavailable", + "text/plain; charset=utf-8", + b"snapshot_stale", + ), + }, + OperatorRequestRoute::State => match snapshot_json { + Some(snapshot_json) => { + let headers = snapshot_response_headers(snapshot_last_publish_unix_epoch); + + http_response_bytes_with_headers("200 OK", "application/json", &headers, snapshot_json) + }, + None => http_response_bytes( + "503 Service Unavailable", + "text/plain; charset=utf-8", + b"operator snapshot unavailable", + ), + }, + } +} + +fn parse_operator_state_request_route( + request: &[u8], +) -> std::result::Result> { + let request = String::from_utf8_lossy(request); + let mut request_line = request.lines(); + let Some(request_line) = request_line.next() else { + return Err(http_response_bytes( + "400 Bad Request", + "text/plain; charset=utf-8", + b"missing request line", + )); + }; + let mut parts = request_line.split_whitespace(); + let Some(method) = parts.next() else { + return Err(http_response_bytes( + "400 Bad Request", + "text/plain; charset=utf-8", + b"missing method", + )); + }; + let Some(path) = parts.next() else { + return Err(http_response_bytes( + "400 Bad Request", + "text/plain; charset=utf-8", + b"missing path", + )); + }; + + if method != "GET" { + return Err(http_response_bytes( + "405 Method Not Allowed", + "text/plain; charset=utf-8", + b"method not allowed", + )); + } + + let path_without_query = path + .split_once('?') + .map_or(path, |(path_without_query, _)| path_without_query); + let normalized_path = path_without_query + .split_once('#') + .map_or(path_without_query, |(path_without_fragment, _)| path_without_fragment); + + match normalized_path { + OPERATOR_DASHBOARD_ENDPOINT_PATH | OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH => + Ok(OperatorRequestRoute::Dashboard), + OPERATOR_DASHBOARD_WS_ENDPOINT_PATH => Ok(OperatorRequestRoute::DashboardWs), + OPERATOR_LIVE_ENDPOINT_PATH => Ok(OperatorRequestRoute::Live), + OPERATOR_READY_ENDPOINT_PATH => Ok(OperatorRequestRoute::Ready), + OPERATOR_STATE_ENDPOINT_PATH => Ok(OperatorRequestRoute::State), + _ => Err(http_response_bytes( + "404 Not Found", + "text/plain; charset=utf-8", + b"not found", + )), + } +} + +fn snapshot_response_headers( + last_publish_unix_epoch: Option, +) -> Vec<(&'static str, String)> { + last_publish_unix_epoch + .map(|last_publish_unix_epoch| { + vec![("X-Decodex-Snapshot-Unix-Epoch", last_publish_unix_epoch.to_string())] + }) + .unwrap_or_default() +} + +fn http_response_bytes(status_line: &str, content_type: &str, body: &[u8]) -> Vec { + http_response_bytes_with_headers(status_line, content_type, &[], body) +} + +fn http_response_bytes_with_headers( + status_line: &str, + content_type: &str, + extra_headers: &[(&str, String)], + body: &[u8], +) -> Vec { + let mut response = format!( + "HTTP/1.1 {status_line}\r\nContent-Type: {content_type}\r\n" + ) + .into_bytes(); + + for (header, value) in extra_headers { + response.extend_from_slice(format!("{header}: {value}\r\n").as_bytes()); + } + + response.extend_from_slice( + format!("Content-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes(), + ); + response.extend_from_slice(body); + + response +} diff --git a/apps/decodex/src/orchestrator/prompting.rs b/apps/decodex/src/orchestrator/prompting.rs new file mode 100644 index 00000000..1233d8ba --- /dev/null +++ b/apps/decodex/src/orchestrator/prompting.rs @@ -0,0 +1,540 @@ +const PROMPT_ONLY_INTERNAL_REVIEW_INSTRUCTION: &str = + "Review your work repeatedly and fix any logic bugs until no new issues are found."; + +fn review_pull_request_title(issue: &TrackerIssue) -> String { + let title = issue.title.trim(); + let prefix = format!("{}:", issue.identifier); + + if let Some(candidate_prefix) = title.get(..prefix.len()) + && candidate_prefix.eq_ignore_ascii_case(&prefix) + { + let summary = title.get(prefix.len()..).unwrap_or_default().trim(); + + if summary.is_empty() { + return issue.identifier.clone(); + } + + return format!("{prefix} {summary}"); + } + + format!("{prefix} {title}") +} + +fn build_developer_instructions( + _tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, + state_store: &StateStore, + recorded_pr_url: Option<&str>, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let continuation_guidance = if allows_clean_continuation(workflow, issue_run.dispatch_mode) { + "\n- If more implementation work still remains at the current turn boundary, you may end the turn without `{terminal_finalize_tool}` and `decodex` may continue the same lane in a later turn." + } else { + "" + }; + let mut sections = Vec::new(); + + if !workflow.body().trim().is_empty() { + sections.push(format!("Workflow policy\n{}", workflow.body())); + } + + for relative_path in workflow.frontmatter().context().read_first() { + let absolute_path = project.repo_root().join(relative_path); + let contents = fs::read_to_string(&absolute_path)?; + + sections.push(format!("File: {relative_path}\n{contents}")); + } + + sections.push(String::from( + "Execution discipline\n- Keep pre-edit discovery bounded to the smallest code surface that can satisfy the current issue.\n- Start with the implementation files directly implicated by the issue before reading broader docs or repo-wide guidance.\n- Do not browse upstream references or general repository documentation unless a concrete ambiguity blocks the change.\n- Once the relevant change surface is identified, patch code and run validation instead of continuing broad searches.", + )); + sections.push(String::from( + "Commit contract\n- When you create a local commit for this lane, use a single-line `decodex/commit/1` JSON commit message.\n- Required fields: `schema`, `summary`, and `authority`.\n- `authority` must be the authoritative Linear issue identifier for this lane.\n- Optional fields: `related` and `breaking`.\n- Do not encode landing mode, CI status, closeout state, or other process-state fields in the commit message.", + )); + + let repair_architecture_guidance = + build_external_repair_architecture_guidance(project, state_store, issue_run); + let completed_state = workflow + .frontmatter() + .tracker() + .resolved_completed_state(); + let internal_review_mode = project.codex().internal_review_mode(); + let tracker_contract = match issue_run.dispatch_mode { + IssueDispatchMode::ReviewRepair => format!( + "Tracker tool contract\n- You own issue-scoped tracker writes for `{issue}` on retained PR `{pr_url}`.\n- This run resumes an existing `{success}` lane. Do not move the issue back to `{in_progress}` and do not call `{review_handoff_tool}`.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n{internal_review_guidance}- For each actionable review item on `{pr_url}`, including non-thread review summaries, validate the claim against the codebase, tests, and requirements before changing code, and keep pushback or clarification threads open until the repaired head is ready.\n- If this run was triggered by retained landing fallback, handle only the implementation-shaped blocker such as branch sync, conflict resolution, ambiguous mergeability, or repository-specific recovery. Do not merge or land the PR yourself.\n{repair_architecture_guidance}- Repair the current PR head on branch `{branch}`, run the repository validation needed to justify the repaired head, and push the repaired head.\n- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- Do not request fresh external review yourself. `decodex` will post the next runtime-owned external review request after `{review_repair_tool}` succeeds.\n- After the repaired head is pushed, reply in-thread for every addressed comment and resolve only the GitHub review threads whose fixes landed and verified on the repaired head.\n{completion_guidance}- If you determine the issue needs human attention, add label `{needs_attention}` with `{label_tool}`, explain the exact observed blocker in a comment, including the failed command and raw error when available, and then call `{terminal_finalize_tool}` with path `manual_attention`. Do not speculate about capabilities you did not directly verify.\n- Keep the tracker issue in `{success}`. `decodex` will handle the later external review request or clean-path runtime landing, closeout, and cleanup lifecycle.\n- Do not report the run as complete or treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}\n- Never write to any other issue.", + issue = issue_run.issue.identifier, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + review_handoff_tool = ISSUE_REVIEW_HANDOFF_TOOL_NAME, + review_repair_tool = ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + in_progress = workflow.frontmatter().tracker().in_progress_state(), + success = workflow.frontmatter().tracker().success_state(), + branch = issue_run.worktree.branch_name, + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + continuation_guidance = continuation_guidance, + repair_architecture_guidance = repair_architecture_guidance, + internal_review_guidance = build_repair_internal_review_guidance(internal_review_mode), + completion_guidance = build_repair_completion_guidance(internal_review_mode), + ), + IssueDispatchMode::Closeout => format!( + "Tracker tool contract\n- You own issue-scoped tracker writes for `{issue}` on retained PR `{pr_url}`.\n- This run resumes a merged post-review lane for the same PR lineage. The tracker issue may still be in `{success}` or may already be in `{completed}` while deterministic closeout tail work remains. Do not move the issue back to `{in_progress}` and do not call `{review_handoff_tool}` or `{review_repair_tool}`.\n- Treat retained closeout as a short deterministic tail. Reuse the existing merged PR evidence instead of restarting broad discovery, and only rerun the minimum validation needed to justify `Done` plus cleanup.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n- If you call `{progress_checkpoint_tool}` during closeout, either omit `head_sha` and let `decodex` record the exact current lane HEAD automatically, or pass the exact full current `HEAD` SHA. Do not send an abbreviated SHA that differs from the live lane head.\n- Merge is already authoritative for `{pr_url}` before this run starts. Do not land, merge, or request review from this closeout run.\n- If the issue is still in `{success}`, transition it once to `{completed}` with `{transition_tool}` before `{closeout_tool}`. If it is already in `{completed}`, leave it there.\n- Finish the remaining Linear closeout tail work for this same merged PR lineage, then call `{closeout_tool}` with PR `{pr_url}` and a short result summary, then call `{terminal_finalize_tool}` with path `closeout`.\n- Do not end the turn without either `{closeout_tool}` plus `{terminal_finalize_tool}`, or the manual-attention path.\n- If you determine the issue needs human attention, add label `{needs_attention}` with `{label_tool}`, explain the exact observed blocker in a comment, including the failed command and raw error when available, and then call `{terminal_finalize_tool}` with path `manual_attention`. Do not speculate about capabilities you did not directly verify.\n- Keep all tracker and PR writes scoped to this retained lane. `decodex` will validate the merged PR lineage, the resolved completed state, and the later cleanup boundary.\n- Do not report the run as complete or treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}\n- Never write to any other issue.", + issue = issue_run.issue.identifier, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + review_handoff_tool = ISSUE_REVIEW_HANDOFF_TOOL_NAME, + review_repair_tool = ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + closeout_tool = ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + in_progress = workflow.frontmatter().tracker().in_progress_state(), + success = workflow.frontmatter().tracker().success_state(), + completed = completed_state, + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + continuation_guidance = continuation_guidance, + ), + _ => format!( + "Tracker tool contract\n- You own issue-scoped tracker writes for `{issue}`.\n- At the start of execution, call `{transition_tool}` to move the issue to `{in_progress}` and add a brief `{comment_tool}` comment that you started work on run `{run_id}` attempt `{attempt}`.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n{internal_review_guidance}- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- When the implementation is ready, commit the lane, push branch `{branch}`, and create or update a non-draft PR titled `{pr_title}` for that branch.\n{completion_guidance}- If you determine the issue needs human attention, add label `{needs_attention}` with `{label_tool}`, explain the exact observed blocker in a comment, including the failed command and raw error when available, and then call `{terminal_finalize_tool}` with path `manual_attention`. Do not speculate about capabilities you did not directly verify. Do not call `{review_handoff_tool}` in that case; `decodex` will stop the lane as a human-required failure without automatic retry.\n- Do not move the issue directly to `{success}` with `{transition_tool}`. `decodex` will complete the success writeback only after its own validation passes.\n- Do not report the run as complete or treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}\n- Never write to any other issue.", + issue = issue_run.issue.identifier, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + comment_tool = ISSUE_COMMENT_TOOL_NAME, + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + review_handoff_tool = ISSUE_REVIEW_HANDOFF_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + in_progress = workflow.frontmatter().tracker().in_progress_state(), + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + success = workflow.frontmatter().tracker().success_state(), + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + continuation_guidance = continuation_guidance, + pr_title = review_pull_request_title(&issue_run.issue), + internal_review_guidance = build_handoff_internal_review_guidance( + internal_review_mode + ), + completion_guidance = build_handoff_completion_guidance(internal_review_mode), + ), + }; + + sections.push(tracker_contract); + + Ok(sections.join("\n\n")) +} + +fn build_user_input( + _tracker: &T, + project: &ServiceConfig, + issue: &TrackerIssue, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, + state_store: &StateStore, + recorded_pr_url: Option<&str>, +) -> String +where + T: IssueTracker + ?Sized, +{ + let continuation_guidance = if allows_clean_continuation(workflow, issue_run.dispatch_mode) { + "\n- If more work still remains at the current turn boundary, you may end the turn without `{terminal_finalize_tool}` and `decodex` will decide whether to continue the lane." + } else { + "" + }; + let description = render_issue_description_for_prompt(issue); + let repair_architecture_guidance = + build_external_repair_architecture_guidance(project, state_store, issue_run); + let completed_state = workflow + .frontmatter() + .tracker() + .resolved_completed_state(); + let internal_review_mode = project.codex().internal_review_mode(); + + match issue_run.dispatch_mode { + IssueDispatchMode::ReviewRepair => format!( + "Continue retained review repair for Linear issue {identifier}: {title}\n\nDescription:\n{description}\n\nCurrent PR:\n- `{pr_url}`\n\nExecution checklist:\n- Resume from the current branch and PR state in this worktree. Do not move the issue back to `{in_progress}`.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n{internal_review_guidance}- Read the current review feedback on `{pr_url}`, including non-thread review summaries, validate each actionable claim against the codebase, tests, and requirements, fix only the verified issues on branch `{branch}`, and keep scope limited to the outstanding retained repair.\n- If the lane is here because retained landing was not a deterministic clean path, handle only the branch sync, conflict resolution, ambiguous mergeability, or repository-specific recovery needed to make the PR clean again. Do not merge or land the PR yourself.\n- Leave pushback or clarification threads open until the repaired head is ready.\n{repair_architecture_guidance}- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- Run the repository validation needed to justify the repaired head.\n- Commit the repair and push the same branch. Do not request fresh external review yourself; `decodex` will post the next runtime-owned external review request after `{review_repair_tool}` succeeds.\n- After the repaired head is pushed, reply in-thread for every addressed comment and resolve only the GitHub review threads whose fixes landed and verified on the repaired head.\n{completion_guidance}- If the issue needs manual attention, add label `{needs_attention}` with `{label_tool}`, explain why in a comment, and then call `{terminal_finalize_tool}` with path `manual_attention`.\n- Keep the issue in `{success}` and do not treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}", + identifier = issue.identifier, + title = issue.title, + description = description, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + in_progress = workflow.frontmatter().tracker().in_progress_state(), + branch = issue_run.worktree.branch_name, + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + review_repair_tool = ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + success = workflow.frontmatter().tracker().success_state(), + continuation_guidance = continuation_guidance, + repair_architecture_guidance = repair_architecture_guidance, + internal_review_guidance = build_repair_internal_review_guidance(internal_review_mode), + completion_guidance = build_repair_completion_guidance(internal_review_mode), + ), + IssueDispatchMode::Closeout => format!( + "Continue retained closeout for Linear issue {identifier}: {title}\n\nDescription:\n{description}\n\nCurrent PR:\n- `{pr_url}`\n\nExecution checklist:\n- Resume from the current branch and merged PR lineage in this worktree. Do not move the issue back to `{in_progress}`.\n- Treat retained closeout as a short deterministic tail. Reuse the existing merged PR evidence instead of restarting broad discovery, and only rerun the minimum validation needed to justify `Done` plus cleanup.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n- If you call `{progress_checkpoint_tool}` during closeout, either omit `head_sha` and let `decodex` record the exact current lane HEAD automatically, or pass the exact full current `HEAD` SHA.\n- Merge is already authoritative for `{pr_url}` before this run starts. Do not land, merge, or request review from this closeout run.\n- The tracker issue may already be in `{completed}` while this deterministic tail work remains pending.\n- If the issue is still in `{success}`, move it once to `{completed}` with `{transition_tool}` before `{closeout_tool}`.\n- Call `{closeout_tool}` with `{pr_url}` and a short result summary, then call `{terminal_finalize_tool}` with path `closeout`.\n- Do not end the turn without either `{closeout_tool}` plus `{terminal_finalize_tool}`, or the manual-attention path.\n- If the issue needs manual attention, add label `{needs_attention}` with `{label_tool}`, explain why in a comment, and then call `{terminal_finalize_tool}` with path `manual_attention`.\n- Keep the lane scoped to this retained post-review work and do not treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}", + identifier = issue.identifier, + title = issue.title, + description = description, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + in_progress = workflow.frontmatter().tracker().in_progress_state(), + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + closeout_tool = ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + success = workflow.frontmatter().tracker().success_state(), + completed = completed_state, + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + continuation_guidance = continuation_guidance, + ), + _ => format!( + "Resolve Linear issue {identifier}: {title}\n\nDescription:\n{description}\n\nExecution checklist:\n- Move the issue to `{in_progress}` with `{transition_tool}` and leave a short `{comment_tool}` comment that includes run `{run_id}` attempt `{attempt}`.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n- Keep discovery bounded to the minimal implementation files needed for this issue; defer broader docs or upstream reading unless a concrete ambiguity blocks the change.\n- Implement the fix in the current worktree.\n{internal_review_guidance}- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- Run the repository validation needed to justify a reviewable PR.\n- Commit the lane, push branch `{branch}`, and create or update a non-draft PR titled `{pr_title}` for that branch.\n{completion_guidance}- If the issue needs manual attention, add label `{needs_attention}` with `{label_tool}`, explain why in a comment, and then call `{terminal_finalize_tool}` with path `manual_attention`. Do not call `{review_handoff_tool}` in that case; `decodex` will stop the lane as a human-required failure without automatic retry.\n- Do not move the issue directly to `{success}` with `{transition_tool}`; `decodex` will finish that writeback after its own validation passes.\n- Do not report the run as complete or treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}", + identifier = issue.identifier, + title = issue.title, + description = description, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + comment_tool = ISSUE_COMMENT_TOOL_NAME, + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + review_handoff_tool = ISSUE_REVIEW_HANDOFF_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + in_progress = workflow.frontmatter().tracker().in_progress_state(), + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + success = workflow.frontmatter().tracker().success_state(), + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + continuation_guidance = continuation_guidance, + pr_title = review_pull_request_title(issue), + internal_review_guidance = build_handoff_internal_review_guidance( + internal_review_mode + ), + completion_guidance = build_handoff_completion_guidance(internal_review_mode), + ), + } +} + +fn build_continuation_user_input( + issue: &TrackerIssue, + workflow: &WorkflowDocument, + dispatch_mode: IssueDispatchMode, + recorded_pr_url: Option<&str>, + success_state: &str, + internal_review_mode: InternalReviewMode, +) -> String { + let completed_state = workflow + .frontmatter() + .tracker() + .resolved_completed_state(); + + match dispatch_mode { + IssueDispatchMode::ReviewRepair => format!( + "Continue retained review repair for Linear issue {identifier} in the current thread and worktree.\n\nContinuation checklist:\n- Resume from the current repository state and outstanding review feedback or retained landing fallback on `{pr_url}`.\n- Keep changes scoped to the same retained review lane and do not move the issue out of `{success}`.\n{internal_review_guidance}- Validate each actionable review claim against the codebase, tests, and requirements before changing code, and keep pushback or clarification threads open until the repaired head is ready.\n- If the blocker is landing fallback, repair only the branch sync, conflict, ambiguous mergeability, or repository-specific recovery issue; do not merge or land the PR yourself.\n- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- If the repaired head is ready, push it. Do not request fresh external review yourself; Decodex will post the next runtime-owned external review request after `{review_repair_tool}` succeeds.\n- After the repaired head is pushed, reply in-thread for every addressed comment and resolve only the GitHub review threads whose fixes landed and verified on the repaired head.\n{completion_guidance}- If the issue requires manual attention, record the manual-attention tracker path before ending the turn.\n- If more work still remains after this turn, you may end the turn without terminal finalization and Decodex will decide whether to continue.", + identifier = issue.identifier, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + success = success_state, + review_repair_tool = ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + internal_review_guidance = build_repair_continuation_review_guidance( + internal_review_mode + ), + completion_guidance = build_repair_continuation_completion_guidance( + internal_review_mode + ), + ), + IssueDispatchMode::Closeout => format!( + "Continue retained closeout for Linear issue {identifier} in the current thread and worktree.\n\nContinuation checklist:\n- Resume from the current repository state and merged PR lineage on `{pr_url}`.\n- Keep changes scoped to the same retained post-review lane. Do not move the issue back to implementation; the tracker may already be in `{completed}` while closeout or cleanup remains pending.\n- Treat this resumed closeout as a short deterministic tail. Reuse the existing merged PR evidence instead of restarting broad discovery, and only rerun the minimum validation needed to justify `Done` plus cleanup.\n- If you record `{progress_checkpoint_tool}` during closeout, either omit `head_sha` or pass the exact full current `HEAD` SHA.\n- Merge is already authoritative for `{pr_url}` before this run starts. Do not land, merge, or request review from this closeout run.\n- If the issue is still in `{success}`, transition it once to `{completed}` with `{transition_tool}` before `{closeout_tool}`.\n- If Linear closeout is complete, call `{closeout_tool}` and then call `{terminal_finalize_tool}` with path `closeout`.\n- Do not end the turn without either `{closeout_tool}` plus `{terminal_finalize_tool}`, or the manual-attention path.\n- If the issue requires manual attention, record the manual-attention tracker path before ending the turn.", + identifier = issue.identifier, + pr_url = recorded_pr_url.unwrap_or("(missing review handoff marker)"), + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + success = success_state, + completed = completed_state, + closeout_tool = ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + ), + _ => format!( + "Continue working on Linear issue {identifier} in the current thread and worktree.\n\nContinuation checklist:\n- Resume from the current repository state instead of restarting broad discovery.\n- Keep changes scoped to the same issue lane.\n{internal_review_guidance}- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n{completion_guidance}- If the issue requires manual attention, record the manual-attention tracker path before ending the turn.\n- If more work still remains after this turn, you may end the turn without terminal finalization and Decodex will decide whether to continue.", + identifier = issue.identifier, + internal_review_guidance = build_handoff_continuation_review_guidance( + internal_review_mode + ), + completion_guidance = build_handoff_continuation_completion_guidance( + internal_review_mode, + &review_pull_request_title(issue), + ), + ), + } +} + +fn build_handoff_internal_review_guidance(internal_review_mode: InternalReviewMode) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Follow the repo-native bounded review method from `WORKFLOW.md`: review the actual current diff and branch state, run both the requirements pass and the adversarial reviewer pass, fix only the smallest coherent owned batch, rerun verification, and re-read `HEAD` before deciding the next normalized review status.\n- Every time the repo-native bounded review method produces a result for the current head, call `{}` with that normalized status, the exact current `HEAD` SHA, and any concise evidence items.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + InternalReviewMode::Prompt => format!("- {PROMPT_ONLY_INTERNAL_REVIEW_INSTRUCTION}\n"), + InternalReviewMode::Off => format!( + "- `codex.internal_review_mode = \"off\"` for this project, so skip internal self-review and do not call `{}`.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + } +} + +fn build_repair_internal_review_guidance(internal_review_mode: InternalReviewMode) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Follow the repo-native bounded review method from `WORKFLOW.md`: review the actual repaired branch state, run both the requirements pass and the adversarial reviewer pass, fix only the smallest coherent owned batch, rerun verification, and re-read `HEAD` before deciding the next normalized review status.\n- Every time the repo-native bounded review method produces a result for the current repaired head, call `{}` with that normalized status, the exact current `HEAD` SHA, and any concise evidence items.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + InternalReviewMode::Prompt => format!("- {PROMPT_ONLY_INTERNAL_REVIEW_INSTRUCTION}\n"), + InternalReviewMode::Off => format!( + "- `codex.internal_review_mode = \"off\"` for this project, so skip internal self-review and do not call `{}`.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + } +} + +fn build_handoff_completion_guidance(internal_review_mode: InternalReviewMode) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Call `{}` only after the latest `{}` for this handoff phase and current `HEAD` is `clean`. Then call `{}` with path `review_handoff`.\n", + ISSUE_REVIEW_HANDOFF_TOOL_NAME, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + InternalReviewMode::Prompt | InternalReviewMode::Off => format!( + "- Call `{}` after the branch is pushed, the non-draft PR is ready, and required validation has passed. Then call `{}` with path `review_handoff`.\n", + ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + } +} + +fn build_repair_completion_guidance(internal_review_mode: InternalReviewMode) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Call `{}` only after the latest `{}` for this repair phase and current `HEAD` is `clean`. Then call `{}` with path `review_repair`.\n", + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + InternalReviewMode::Prompt | InternalReviewMode::Off => format!( + "- Call `{}` after the repaired head is pushed and required validation has passed. Then call `{}` with path `review_repair`.\n", + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + } +} + +fn build_handoff_continuation_review_guidance( + internal_review_mode: InternalReviewMode, +) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Resume the repo-native bounded review method from `WORKFLOW.md`: review the actual current diff and branch state, run both the requirements pass and the adversarial reviewer pass, fix only the smallest coherent owned batch, rerun verification, and re-read `HEAD` before deciding the next normalized review status.\n- After each bounded review result for the current head, call `{}` with the normalized status and current `HEAD` SHA.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + InternalReviewMode::Prompt => format!("- {PROMPT_ONLY_INTERNAL_REVIEW_INSTRUCTION}\n"), + InternalReviewMode::Off => format!( + "- `codex.internal_review_mode = \"off\"` for this project, so continue without internal self-review and do not call `{}`.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + } +} + +fn build_repair_continuation_review_guidance(internal_review_mode: InternalReviewMode) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Resume the repo-native bounded review method from `WORKFLOW.md`: review the actual repaired branch state, run both the requirements pass and the adversarial reviewer pass, fix only the smallest coherent owned batch, rerun verification, and re-read `HEAD` before deciding the next normalized review status.\n- After each bounded review result for the repaired head, call `{}` with the normalized status and current `HEAD` SHA.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + InternalReviewMode::Prompt => format!("- {PROMPT_ONLY_INTERNAL_REVIEW_INSTRUCTION}\n"), + InternalReviewMode::Off => format!( + "- `codex.internal_review_mode = \"off\"` for this project, so continue without internal self-review and do not call `{}`.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME + ), + } +} + +fn build_handoff_continuation_completion_guidance( + internal_review_mode: InternalReviewMode, + pr_title: &str, +) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- If the implementation is review-ready, ensure the non-draft PR title is `{pr_title}` and finish the PR-backed tracker handoff only after the latest `{}` for the current `HEAD` is `clean`.\n", + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + ), + InternalReviewMode::Prompt | InternalReviewMode::Off => format!( + "- If the implementation is review-ready, ensure the non-draft PR title is `{pr_title}` and finish the PR-backed tracker handoff after required validation has passed.\n", + ), + } +} + +fn build_repair_continuation_completion_guidance( + internal_review_mode: InternalReviewMode, +) -> String { + match internal_review_mode { + InternalReviewMode::Loop => format!( + "- Call `{}` only after the latest `{}` for the current `HEAD` is `clean`, and then call `{}` with path `review_repair`.\n", + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, + ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + InternalReviewMode::Prompt | InternalReviewMode::Off => format!( + "- Call `{}` after the repaired head is pushed and required validation has passed, and then call `{}` with path `review_repair`.\n", + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME + ), + } +} + +fn allows_clean_continuation( + workflow: &WorkflowDocument, + dispatch_mode: IssueDispatchMode, +) -> bool { + workflow.frontmatter().execution().max_turns() > 1 + && dispatch_mode != IssueDispatchMode::Closeout +} + +fn build_external_repair_architecture_guidance( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> String +{ + let review_handoff = match state_store.review_handoff_marker( + project.service_id(), + &issue_run.issue.id, + &issue_run.worktree.branch_name, + ) { + Ok(Some(review_handoff)) => review_handoff, + Ok(None) => return String::new(), + Err(error) => { + tracing::warn!( + ?error, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + worktree_path = %issue_run.worktree.path.display(), + "Retained review prompt could not read the runtime handoff; omitting architecture guidance." + ); + + return String::new(); + }, + }; + let marker = match state_store.review_orchestration_marker( + project.service_id(), + &issue_run.issue.id, + &review_handoff, + ) { + Ok(Some(marker)) => marker, + Ok(None) => return String::new(), + Err(error) => { + tracing::warn!( + ?error, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + worktree_path = %issue_run.worktree.path.display(), + "Retained review prompt could not read runtime orchestration state; omitting architecture guidance." + ); + + return String::new(); + }, + }; + + if marker.external_round_count() < 4 { + return String::new(); + } + + format!( + "- This retained repair is external review round {}. Before another patch-only cycle, decide whether the repeated churn points to an architectural or root-cause defect that local patching will not converge.\n- If it is architectural, take the manual-attention path instead of continuing patch-on-patch repair.\n- If it is not architectural and the findings are still normal retained review work, continue this repair normally; a successful `{}` will reset the external review-round budget.\n", + marker.external_round_count(), + ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME + ) +} + +fn build_review_run_context( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result +{ + match issue_run.dispatch_mode { + IssueDispatchMode::ReviewRepair => { + validate_review_repair_runtime(project, false)?; + + let review_handoff = read_retained_review_handoff(project, state_store, issue_run)? + .ok_or_else(|| { + eyre::eyre!( + "Retained review-repair run `{}` for issue `{}` requires an existing runtime review handoff.", + issue_run.run_id, + issue_run.issue.identifier + ) + })?; + + Ok(ReviewHandoffContext { + attempt_number: issue_run.attempt_number, + branch_name: issue_run.worktree.branch_name.clone(), + run_id: issue_run.run_id.clone(), + service_id: project.service_id().to_owned(), + worktree_path: relative_worktree_path(project, &issue_run.worktree), + cwd: issue_run.worktree.path.clone(), + github_token_env_var: Some(project.github().token_env_var().to_owned()), + internal_review_mode: project.codex().internal_review_mode(), + mode: ReviewExecutionMode::Repair, + recorded_pr_url: Some(review_handoff.pr_url().to_owned()), + }) + }, + IssueDispatchMode::Closeout => { + validate_closeout_runtime(project, false)?; + + let review_handoff = read_retained_review_handoff(project, state_store, issue_run)? + .ok_or_else(|| { + eyre::eyre!( + "Retained closeout run `{}` for issue `{}` requires an existing runtime review handoff.", + issue_run.run_id, + issue_run.issue.identifier + ) + })?; + + Ok(ReviewHandoffContext { + attempt_number: issue_run.attempt_number, + branch_name: issue_run.worktree.branch_name.clone(), + run_id: issue_run.run_id.clone(), + service_id: project.service_id().to_owned(), + worktree_path: relative_worktree_path(project, &issue_run.worktree), + cwd: issue_run.worktree.path.clone(), + github_token_env_var: Some(project.github().token_env_var().to_owned()), + internal_review_mode: project.codex().internal_review_mode(), + mode: ReviewExecutionMode::Closeout, + recorded_pr_url: Some(review_handoff.pr_url().to_owned()), + }) + }, + _ => Ok(ReviewHandoffContext { + attempt_number: issue_run.attempt_number, + branch_name: issue_run.worktree.branch_name.clone(), + run_id: issue_run.run_id.clone(), + service_id: project.service_id().to_owned(), + worktree_path: relative_worktree_path(project, &issue_run.worktree), + cwd: issue_run.worktree.path.clone(), + github_token_env_var: Some(project.github().token_env_var().to_owned()), + internal_review_mode: project.codex().internal_review_mode(), + mode: ReviewExecutionMode::Handoff, + recorded_pr_url: None, + }), + } +} + +fn read_retained_review_handoff( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result> +{ + state_store.review_handoff_marker( + project.service_id(), + &issue_run.issue.id, + &issue_run.worktree.branch_name, + ) +} diff --git a/apps/decodex/src/orchestrator/pull_request_review.rs b/apps/decodex/src/orchestrator/pull_request_review.rs new file mode 100644 index 00000000..dd204343 --- /dev/null +++ b/apps/decodex/src/orchestrator/pull_request_review.rs @@ -0,0 +1,332 @@ +fn query_pull_request_review_state_page( + cwd: &Path, + owner: &str, + repo: &str, + number: u64, + review_threads_after: Option<&str>, + pr_url: &str, + github_token: &str, +) -> Result { + let mut command = Command::new("gh"); + + command.args(["api", "graphql", "-f", &format!("query={PULL_REQUEST_REVIEW_STATE_QUERY}")]); + command.args(["-F", &format!("owner={owner}")]); + command.args(["-F", &format!("name={repo}")]); + command.args(["-F", &format!("number={number}")]); + + if let Some(review_threads_after) = review_threads_after { + command.args(["-F", &format!("reviewThreadsAfter={review_threads_after}")]); + } + + command.current_dir(cwd); + + github::configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to inspect pull request review state `{pr_url}`: {}", stderr.trim()); + } + + let response = serde_json::from_slice::(&output.stdout)?; + let Some(repository) = response.data.repository else { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a repository."); + }; + + if repository.pull_request.is_none() { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a pull request."); + } + + Ok(repository) +} + +fn query_pull_request_issue_comments_page( + cwd: &Path, + owner: &str, + repo: &str, + number: u64, + comments_after: &str, + pr_url: &str, + github_token: &str, +) -> Result { + let mut command = Command::new("gh"); + + command.args(["api", "graphql", "-f", &format!("query={PULL_REQUEST_ISSUE_COMMENTS_QUERY}")]); + command.args(["-F", &format!("owner={owner}")]); + command.args(["-F", &format!("name={repo}")]); + command.args(["-F", &format!("number={number}")]); + command.args(["-F", &format!("commentsAfter={comments_after}")]); + command.current_dir(cwd); + + github::configure_gh_command(&mut command, github_token); + + let output = command.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect pull request issue comments for `{pr_url}`: {}", + stderr.trim() + ); + } + + let response = serde_json::from_slice::(&output.stdout)?; + let Some(repository) = response.data.repository else { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a repository."); + }; + let Some(pull_request) = repository.pull_request else { + eyre::bail!("GitHub GraphQL response for `{pr_url}` did not include a pull request."); + }; + + Ok(pull_request) +} + +fn pull_request_review_state_from_page( + repository: &PullRequestReviewStateRepository, + pull_request: &PullRequestReviewStateNode, +) -> PullRequestReviewState { + PullRequestReviewState { + url: pull_request.url.clone(), + state: pull_request.state.clone(), + is_draft: pull_request.is_draft, + review_decision: pull_request.review_decision.clone(), + merge_commit_allowed: repository.merge_commit_allowed, + pending_review_requests: pull_request.review_requests.total_count, + mergeable: pull_request.mergeable.clone(), + merge_state_status: pull_request.merge_state_status.clone(), + head_ref_name: pull_request.head_ref_name.clone(), + head_ref_oid: pull_request.head_ref_oid.clone(), + merge_commit_oid: pull_request.merge_commit.as_ref().map(|commit| commit.oid.clone()), + head_repository_name: pull_request + .head_repository + .as_ref() + .map(|repository| repository.name.clone()), + head_repository_owner: pull_request + .head_repository_owner + .as_ref() + .map(|owner| owner.login.clone()), + status_check_rollup_state: pull_request_status_check_rollup_state(pull_request), + unresolved_review_threads: count_unresolved_review_threads(&pull_request.review_threads), + issue_description_external_review_thumbs_up_count: reaction_group_actor_count( + &pull_request.reaction_groups, + "THUMBS_UP", + EXTERNAL_REVIEW_ACTOR_LOGIN, + ), + issue_comments: pull_request + .comments + .nodes + .iter() + .map(issue_comment_state_from_node) + .collect(), + reviews: pull_request + .reviews + .nodes + .iter() + .filter_map(review_summary_state_from_node) + .collect(), + } +} + +fn merge_pull_request_review_state_page( + review_state: &mut PullRequestReviewState, + repository: &PullRequestReviewStateRepository, + pull_request: &PullRequestReviewStateNode, +) -> Result> { + if review_state.url != pull_request.url + || review_state.state != pull_request.state + || review_state.is_draft != pull_request.is_draft + || review_state.review_decision != pull_request.review_decision + || review_state.merge_commit_allowed != repository.merge_commit_allowed + || review_state.pending_review_requests != pull_request.review_requests.total_count + || review_state.mergeable != pull_request.mergeable + || review_state.merge_state_status != pull_request.merge_state_status + || review_state.head_ref_name != pull_request.head_ref_name + || review_state.head_ref_oid != pull_request.head_ref_oid + || review_state.merge_commit_oid + != pull_request.merge_commit.as_ref().map(|commit| commit.oid.clone()) + || review_state.head_repository_name + != pull_request.head_repository.as_ref().map(|repository| repository.name.clone()) + || review_state.head_repository_owner + != pull_request.head_repository_owner.as_ref().map(|owner| owner.login.clone()) + || review_state.status_check_rollup_state + != pull_request_status_check_rollup_state(pull_request) + || review_state.issue_description_external_review_thumbs_up_count + != reaction_group_actor_count( + &pull_request.reaction_groups, + "THUMBS_UP", + EXTERNAL_REVIEW_ACTOR_LOGIN, + ) + || review_state.reviews + != pull_request + .reviews + .nodes + .iter() + .filter_map(review_summary_state_from_node) + .collect::>() + { + eyre::bail!("Pull request review state changed while paginating `{}`.", review_state.url); + } + + review_state.unresolved_review_threads += + count_unresolved_review_threads(&pull_request.review_threads); + + next_pull_request_review_threads_cursor(pull_request) +} + +fn merge_pull_request_issue_comment_page( + review_state: &mut PullRequestReviewState, + pull_request: &PullRequestIssueCommentsNode, +) -> Result> { + if review_state.url != pull_request.url { + eyre::bail!("Pull request issue comment state changed while paginating `{}`.", review_state.url); + } + + for comment in pull_request.comments.nodes.iter().map(issue_comment_state_from_node) { + if review_state.issue_comments.iter().any(|existing| existing.database_id == comment.database_id) + { + eyre::bail!( + "Pull request issue comments repeated while paginating `{}`.", + review_state.url + ); + } + + review_state.issue_comments.push(comment); + } + + next_pull_request_issue_comments_cursor(&pull_request.comments, pull_request.url.as_str()) +} + +fn count_unresolved_review_threads(review_threads: &PullRequestReviewThreadConnection) -> usize { + review_threads.nodes.iter().filter(|thread| !thread.is_resolved && !thread.is_outdated).count() +} + +fn pull_request_status_check_rollup_state( + pull_request: &PullRequestReviewStateNode, +) -> Option { + pull_request + .commits + .nodes + .first() + .and_then(|node| node.commit.status_check_rollup.as_ref()) + .map(|rollup| rollup.state.clone()) +} + +fn issue_comment_state_from_node( + comment: &PullRequestIssueCommentNode, +) -> PullRequestIssueCommentState { + PullRequestIssueCommentState { + database_id: comment.database_id, + author_login: comment.author.as_ref().map(|author| author.login.clone()), + body: comment.body.clone(), + created_at_unix_epoch: parse_github_timestamp_to_unix_epoch(&comment.created_at) + .expect("pull request issue comment timestamp should parse"), + external_review_eyes_reaction_count: reaction_group_actor_count( + &comment.reaction_groups, + "EYES", + EXTERNAL_REVIEW_ACTOR_LOGIN, + ), + } +} + +fn review_summary_state_from_node( + review: &PullRequestReviewNode, +) -> Option { + let submitted_at_unix_epoch = + parse_github_timestamp_to_unix_epoch(review.submitted_at.as_deref()?).ok()?; + + Some(PullRequestReviewSummaryState { + author_login: review.author.as_ref().map(|author| author.login.clone()), + body: review.body.clone(), + state: review.state.clone(), + submitted_at_unix_epoch, + }) +} + +fn reaction_group_actor_count( + groups: &[PullRequestReactionGroup], + content: &str, + actor_login: &str, +) -> usize { + groups + .iter() + .find(|group| group.content == content) + .map_or(0, |group| { + group + .users + .nodes + .iter() + .filter(|actor| actor.login.eq_ignore_ascii_case(actor_login)) + .count() + }) +} + +fn parse_github_timestamp_to_unix_epoch(timestamp: &str) -> Result { + Ok( + OffsetDateTime::parse(timestamp, &Rfc3339) + .map_err(|error| eyre::eyre!("Failed to parse GitHub timestamp `{timestamp}`: {error}"))? + .unix_timestamp(), + ) +} + +fn next_pull_request_review_threads_cursor( + pull_request: &PullRequestReviewStateNode, +) -> Result> { + if !pull_request.review_threads.page_info.has_next_page { + return Ok(None); + } + + pull_request.review_threads.page_info.end_cursor.clone().map(Some).ok_or_else(|| { + eyre::eyre!( + "GitHub GraphQL response for `{}` reported additional review thread pages without an end cursor.", + pull_request.url + ) + }) +} + +fn next_pull_request_issue_comments_cursor( + comments: &PullRequestIssueCommentConnection, + pr_url: &str, +) -> Result> { + if !comments.page_info.has_next_page { + return Ok(None); + } + + comments.page_info.end_cursor.clone().map(Some).ok_or_else(|| { + eyre::eyre!( + "GitHub GraphQL response for `{pr_url}` reported additional issue comment pages without an end cursor.", + ) + }) +} + +fn format_run_once_summary(summary: &RunSummary, dry_run: bool) -> String { + if dry_run { + return format!( + "dry run: project={} issue={} branch={} worktree={} attempt={}", + summary.project_id, + summary.issue_identifier, + summary.branch_name, + summary.worktree_path.display(), + summary.attempt_number + ); + } + if summary.continuation_pending { + return format!( + "run paused at continuation boundary: project={} issue={} run_id={} worktree={} next_action=rerun_or_use_daemon", + summary.project_id, + summary.issue_identifier, + summary.run_id, + summary.worktree_path.display() + ); + } + + format!( + "run complete: project={} issue={} run_id={} worktree={}", + summary.project_id, + summary.issue_identifier, + summary.run_id, + summary.worktree_path.display() + ) +} diff --git a/apps/decodex/src/orchestrator/reconciliation.rs b/apps/decodex/src/orchestrator/reconciliation.rs new file mode 100644 index 00000000..5261f028 --- /dev/null +++ b/apps/decodex/src/orchestrator/reconciliation.rs @@ -0,0 +1,500 @@ +#[cfg(test)] +fn inspect_active_run_reconciliation_at( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + active_workflow_override: Option>, + now_unix_epoch: i64, +) -> Result> +where + T: IssueTracker, +{ + let leases = state_store.list_leases(project.service_id())?; + + if leases.is_empty() { + return Ok(Vec::new()); + } + + let issue_ids = leases.iter().map(|lease| lease.issue_id().to_owned()).collect::>(); + let issues = tracker.refresh_issues(&issue_ids)?; + let issues_by_id = + issues.into_iter().map(|issue| (issue.id.clone(), issue)).collect::>(); + let mut actions = Vec::new(); + + for lease in leases { + let Some(issue) = issues_by_id.get(lease.issue_id()).cloned() else { + continue; + }; + let Some(run_attempt) = state_store.run_attempt(lease.run_id())? else { + continue; + }; + let worktree_mapping = state_store.worktree_for_issue(&issue.id)?; + let action_workflow = active_reconciliation_workflow_for_lease( + workflow, + active_workflow_override, + &issue, + &run_attempt, + ); + let retained_closeout = + terminal_issue_keeps_retained_closeout( + tracker, + &issue, + project, + action_workflow, + state_store, + )?; + let disposition = if !retained_closeout && is_terminal_issue(&issue, action_workflow) { + Some(ActiveRunDisposition::Terminal) + } else if !retained_closeout + && is_issue_nonactive_for_run(&issue, action_workflow) + { + Some(ActiveRunDisposition::NonActive) + } else if let Some(idle_for) = stalled_idle_duration( + state_store, + &run_attempt, + worktree_mapping.as_ref(), + now_unix_epoch, + )? { + if retained_review_handoff_matches_run( + state_store, + &run_attempt, + worktree_mapping.as_ref(), + )? { + Some(ActiveRunDisposition::RetainedReviewComplete) + } else { + Some(ActiveRunDisposition::Stalled { idle_for }) + } + } else { + None + }; + + if let Some(disposition) = disposition { + actions.push(ActiveRunReconciliation { + issue: issue.clone(), + run_attempt, + worktree_mapping, + disposition, + workflow: action_workflow.clone(), + }); + } + } + + Ok(actions) +} + +fn inspect_exited_daemon_child_reconciliation( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_id: &str, + run_id: &str, +) -> Result> +where + T: IssueTracker, +{ + inspect_exited_daemon_child_reconciliation_at( + tracker, + project, + workflow, + state_store, + issue_id, + run_id, + OffsetDateTime::now_utc().unix_timestamp(), + ) +} + +fn inspect_exited_daemon_child_reconciliation_at( + tracker: &T, + _project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_id: &str, + run_id: &str, + now_unix_epoch: i64, +) -> Result> +where + T: IssueTracker, +{ + let Some(issue) = refresh_issue(tracker, issue_id)? else { + return Ok(Vec::new()); + }; + let Some(run_attempt) = state_store.run_attempt(run_id)? else { + return Ok(Vec::new()); + }; + let worktree_mapping = state_store.worktree_for_issue(issue_id)?; + + if run_attempt.status() != "failed" || !is_issue_active_for_run(&issue, workflow) { + return Ok(Vec::new()); + } + + let Some(idle_for) = stalled_protocol_idle_duration( + state_store, + &run_attempt, + worktree_mapping.as_ref(), + now_unix_epoch, + )? + else { + return Ok(Vec::new()); + }; + + Ok(vec![ActiveRunReconciliation { + issue, + run_attempt, + worktree_mapping, + disposition: ActiveRunDisposition::Stalled { idle_for }, + workflow: workflow.clone(), + }]) +} + +fn active_reconciliation_workflow_for_lease<'a>( + current_workflow: &'a WorkflowDocument, + active_workflow_override: Option>, + issue: &TrackerIssue, + run_attempt: &RunAttempt, +) -> &'a WorkflowDocument { + match active_workflow_override { + Some(override_context) + if override_context.child.issue_id == issue.id + && override_context.child.run_id == run_attempt.run_id() => + override_context.workflow, + _ => current_workflow, + } +} + +fn apply_active_run_reconciliation( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + worktree_manager: &WorktreeManager, + actions: Vec, +) -> Result<()> +where + T: IssueTracker, +{ + for action in actions { + match &action.disposition { + ActiveRunDisposition::RetainedReviewComplete => { + reconcile_retained_review_complete_active_run(project, state_store, &action)?; + }, + ActiveRunDisposition::Terminal => { + tracing::info!( + project_id = project.service_id(), + issue_id = action.issue.id, + issue = action.issue.identifier, + run_id = action.run_attempt.run_id(), + disposition = "terminal", + "Reconciling terminal active run." + ); + + mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "terminated")?; + + tracker::clear_automation_lane_labels(tracker, &action.issue, project.service_id())?; + + state_store.clear_lease(&action.issue.id)?; + + if let Some(mapping) = &action.worktree_mapping { + cleanup_worktree_mapping( + state_store, + worktree_manager, + &action.workflow, + &action.issue.identifier, + mapping, + )?; + } + }, + ActiveRunDisposition::NonActive => { + reconcile_nonactive_active_run(project, state_store, worktree_manager, &action)?; + }, + ActiveRunDisposition::Stalled { idle_for } => { + reconcile_stalled_active_run( + tracker, + project, + state_store, + worktree_manager, + &action, + *idle_for, + )?; + }, + } + } + + Ok(()) +} + +fn reconcile_retained_review_complete_active_run( + project: &ServiceConfig, + state_store: &StateStore, + action: &ActiveRunReconciliation, +) -> Result<()> { + tracing::info!( + project_id = project.service_id(), + issue_id = action.issue.id, + issue = action.issue.identifier, + run_id = action.run_attempt.run_id(), + disposition = "retained_review_complete", + "Reconciling completed retained review run." + ); + + mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "succeeded")?; + + state_store.clear_lease(&action.issue.id)?; + + Ok(()) +} + +fn reconcile_nonactive_active_run( + project: &ServiceConfig, + state_store: &StateStore, + worktree_manager: &WorktreeManager, + action: &ActiveRunReconciliation, +) -> Result<()> { + tracing::info!( + project_id = project.service_id(), + issue_id = action.issue.id, + issue = action.issue.identifier, + run_id = action.run_attempt.run_id(), + disposition = "non_active", + "Reconciling non-active run." + ); + + mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "interrupted")?; + + let worktree_path = action.worktree_mapping.as_ref().map_or_else( + || worktree_manager.plan_for_issue(&action.issue.identifier).path, + |mapping| mapping.worktree_path().to_path_buf(), + ); + + if worktree_path.exists() { + write_retry_budget_marker( + &worktree_path, + action.run_attempt.run_id(), + action.run_attempt.attempt_number(), + retry_budget_base_for_issue_worktree(state_store, &action.issue.id, &worktree_path)?, + )?; + } + + state_store.clear_lease(&action.issue.id)?; + + Ok(()) +} + +fn reconcile_stalled_active_run( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + worktree_manager: &WorktreeManager, + action: &ActiveRunReconciliation, + idle_for: Duration, +) -> Result<()> +where + T: IssueTracker, +{ + tracing::warn!( + project_id = project.service_id(), + issue_id = action.issue.id, + issue = action.issue.identifier, + run_id = action.run_attempt.run_id(), + disposition = "stalled", + idle_for_s = idle_for.as_secs(), + "Reconciling stalled run." + ); + + state_store.update_run_status(action.run_attempt.run_id(), "stalled")?; + state_store.clear_lease(&action.issue.id)?; + + let worktree = action.worktree_mapping.as_ref().map_or_else( + || worktree_manager.plan_for_issue(&action.issue.identifier), + |mapping| WorktreeSpec { + branch_name: mapping.branch_name().to_owned(), + issue_identifier: action.issue.identifier.clone(), + path: mapping.worktree_path().to_path_buf(), + reused_existing: true, + }, + ); + let retry_budget_base = + retry_budget_base_for_issue_worktree(state_store, &action.issue.id, &worktree.path)?; + let issue_run = IssueRunPlan { + issue: action.issue.clone(), + issue_state: planned_issue_state_for_dispatch( + &action.workflow, + &action.issue, + IssueDispatchMode::Retry, + None, + ), + initial_issue_state: action.issue.state.name.clone(), + worktree, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode: IssueDispatchMode::Retry, + attempt_number: action.run_attempt.attempt_number(), + run_id: action.run_attempt.run_id().to_owned(), + retry_budget_base, + }; + + write_reconciliation_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_RECONCILIATION, + ); + handle_failure( + tracker, + project, + &action.workflow, + state_store, + &issue_run, + &Report::new(StalledRunNeedsAttention { + issue_identifier: action.issue.identifier.clone(), + run_id: action.run_attempt.run_id().to_owned(), + idle_for, + }), + )?; + + Ok(()) +} + +fn write_reconciliation_operation_marker_best_effort( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + current_operation: &str, +) { + if let Err(error) = state::write_run_operation_marker_preserving_activity( + worktree_path, + run_id, + attempt_number, + current_operation, + ) { + tracing::warn!( + ?error, + run_id, + attempt_number, + current_operation, + worktree_path = %worktree_path.display(), + "Run operation marker write failed; continuing stalled-run reconciliation." + ); + } +} + +fn retained_review_handoff_matches_run( + state_store: &StateStore, + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, +) -> Result { + let Some(worktree_mapping) = worktree_mapping else { + return Ok(false); + }; + let Some(marker) = state_store.review_handoff_marker( + worktree_mapping.project_id(), + run_attempt.issue_id(), + worktree_mapping.branch_name(), + )? else { + return Ok(false); + }; + + Ok(marker.run_id() == run_attempt.run_id() + && marker.attempt_number() == run_attempt.attempt_number() + && marker.branch_name() == worktree_mapping.branch_name()) +} + +fn stalled_idle_duration( + state_store: &StateStore, + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, + now_unix_epoch: i64, +) -> Result> { + if !matches!(run_attempt.status(), "starting" | "running") { + return Ok(None); + } + + let Some(last_activity) = + last_observed_run_activity_unix_epoch(state_store, run_attempt, worktree_mapping)? + else { + return Ok(None); + }; + let Some(idle_for) = observed_idle_duration(last_activity, now_unix_epoch) else { + return Ok(None); + }; + + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT { + return Ok(Some(idle_for)); + } + + Ok(None) +} + +fn last_observed_run_activity_unix_epoch( + state_store: &StateStore, + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, +) -> Result> { + let state_store_activity = state_store.last_run_activity_unix_epoch(run_attempt.run_id())?; + let worktree_activity = match worktree_mapping { + Some(mapping) => state::read_run_activity_marker( + mapping.worktree_path(), + run_attempt.run_id(), + run_attempt.attempt_number(), + )?, + None => None, + }; + + Ok(match (state_store_activity, worktree_activity) { + (Some(left), Some(right)) => Some(left.max(right)), + (Some(activity), None) | (None, Some(activity)) => Some(activity), + (None, None) => None, + }) +} + +fn stalled_protocol_idle_duration( + state_store: &StateStore, + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, + now_unix_epoch: i64, +) -> Result> { + let Some(last_activity) = + last_observed_protocol_activity_unix_epoch(state_store, run_attempt, worktree_mapping)? + else { + return Ok(None); + }; + let Some(idle_for) = observed_idle_duration(last_activity, now_unix_epoch) else { + return Ok(None); + }; + + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT { + return Ok(Some(idle_for)); + } + + Ok(None) +} + +fn last_observed_protocol_activity_unix_epoch( + state_store: &StateStore, + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, +) -> Result> { + let state_store_activity = + state_store.last_protocol_activity_unix_epoch(run_attempt.run_id())?; + let worktree_activity = match worktree_mapping { + Some(mapping) => state::read_run_protocol_activity_marker( + mapping.worktree_path(), + run_attempt.run_id(), + run_attempt.attempt_number(), + )?, + None => None, + }; + + Ok(match (state_store_activity, worktree_activity) { + (Some(left), Some(right)) => Some(left.max(right)), + (Some(activity), None) | (None, Some(activity)) => Some(activity), + (None, None) => None, + }) +} + +fn observed_idle_duration(last_activity_unix_epoch: i64, now_unix_epoch: i64) -> Option { + now_unix_epoch + .checked_sub(last_activity_unix_epoch) + .and_then(|idle_seconds| u64::try_from(idle_seconds).ok()) + .map(Duration::from_secs) +} diff --git a/apps/decodex/src/orchestrator/run_cycle.rs b/apps/decodex/src/orchestrator/run_cycle.rs new file mode 100644 index 00000000..1d37dd09 --- /dev/null +++ b/apps/decodex/src/orchestrator/run_cycle.rs @@ -0,0 +1,2603 @@ +use state::IssueLease; +use state::PreacquiredLeaseGuards; + +use crate::commit_message; + +const INTERNAL_RETAINED_DRAIN_MAX_PASSES: usize = 2; + +struct RetainedReviewLane { + snapshot: PostReviewLaneSnapshot, + review_state: PullRequestReviewState, + orchestration_marker: ReviewOrchestrationMarker, +} + +struct RetainedReviewRuntime<'a, T> { + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, + github_token: &'a mut Option, + now_unix_epoch: i64, +} + +#[derive(Clone, Copy)] +struct RetainedReviewOrchestrationMarkerFields { + request_comment_database_id: Option, + request_created_at_unix_epoch: Option, + request_retry_count: i64, + external_round_count: i64, + auto_merge_enabled_at_unix_epoch: Option, +} +impl RetainedReviewOrchestrationMarkerFields { + fn from_marker(marker: &ReviewOrchestrationMarker) -> Self { + Self { + request_comment_database_id: marker.request_comment_database_id(), + request_created_at_unix_epoch: marker.request_created_at_unix_epoch(), + request_retry_count: marker.request_retry_count(), + external_round_count: marker.external_round_count(), + auto_merge_enabled_at_unix_epoch: marker.auto_merge_enabled_at_unix_epoch(), + } + } +} + +#[derive(Clone, Copy)] +struct RetainedAdminMergeReasons { + admin_merge_unavailable: &'static str, + admin_merge_failed: &'static str, +} + +enum RetainedReviewLaneReviewLoad { + Skip, + Blocked(String), + ReviewState(Box), +} + +fn run_configured_cycle( + request: RunCycleRequest<'_>, +) -> Result> { + let config = ServiceConfig::from_path(request.config_path)?; + let workflow = load_configured_cycle_workflow(&config, request.preferred_workflow_snapshot)?; + let api_key = config.tracker().resolve_api_key()?; + let tracker = LinearClient::new(api_key)?; + + if let Some(issue_id) = request.preferred_issue_id { + let target_context = TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: request.state_store, + issue_id, + preferred_issue_state: request.preferred_issue_state, + preferred_initial_issue_state: request.preferred_initial_issue_state, + dry_run: request.dry_run, + lease_preacquired: request.preferred_lease_acquired, + preferred_issue_claim_fd: request.preferred_issue_claim_fd, + preferred_dispatch_slot_fd: request.preferred_dispatch_slot_fd, + preferred_dispatch_slot_index: request.preferred_dispatch_slot_index, + dispatch_mode: request.preferred_dispatch_mode.unwrap_or(IssueDispatchMode::Normal), + preferred_run_identity: request.preferred_run_identity, + preferred_retry_budget_base: request.preferred_retry_budget_base, + }; + + return match request.preferred_dispatch_mode { + Some(_) => run_target_issue_once(target_context), + None => run_target_issue_once_with_inferred_dispatch(target_context), + }; + } + + run_project_once(&tracker, &config, &workflow, request.state_store, request.dry_run) +} + +fn load_configured_cycle_workflow( + config: &ServiceConfig, + preferred_workflow_snapshot: Option<&str>, +) -> Result { + let workflow_path = config.workflow_path().to_path_buf(); + + match preferred_workflow_snapshot { + Some(snapshot) => WorkflowDocument::parse_markdown(snapshot), + None => WorkflowDocument::from_path(&workflow_path), + } +} + +fn run_project_once( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + dry_run: bool, +) -> Result> +where + T: IssueTracker, +{ + run_project_once_with_exclusions(tracker, project, workflow, state_store, dry_run, &[]) +} + +fn run_project_once_with_exclusions( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + dry_run: bool, + excluded_issue_ids: &[&str], +) -> Result> +where + T: IssueTracker, +{ + let Some(issue_run) = plan_project_issue_run_with_exclusions( + tracker, + project, + workflow, + state_store, + dry_run, + excluded_issue_ids, + )? + else { + return Ok(None); + }; + + complete_issue_run(tracker, project, workflow, state_store, issue_run, dry_run) +} + +fn reconcile_post_review_orchestration( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result<()> +where + T: IssueTracker, +{ + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + + reconcile_post_review_orchestration_with_inspector( + tracker, + project, + workflow, + state_store, + &review_state_inspector, + ) +} + +fn reconcile_post_review_orchestration_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> Result<()> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let worktrees = state_store.list_worktrees(project.service_id())?; + + if worktrees.is_empty() { + return Ok(()); + } + + let issue_ids = + worktrees.iter().map(|mapping| mapping.issue_id().to_owned()).collect::>(); + let issues = tracker.refresh_issues(&issue_ids)?; + let issues_by_id = + issues.into_iter().map(|issue| (issue.id.clone(), issue)).collect::>(); + let tracker_policy = workflow.frontmatter().tracker(); + let success_state = tracker_policy.success_state(); + let opt_out_label = tracker_policy.opt_out_label(); + let needs_attention_label = tracker_policy.needs_attention_label(); + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut github_token: Option = None; + + for worktree in worktrees { + let Some(issue) = issues_by_id.get(worktree.issue_id()).cloned() else { + continue; + }; + + if !eligible_post_review_orchestration_issue( + tracker, + &issue, + project.service_id(), + success_state, + opt_out_label, + needs_attention_label, + )? { + continue; + } + if state_store.lease_for_issue(&issue.id)?.is_some() { + continue; + } + + let lane = match load_retained_review_lane( + project.service_id(), + state_store, + issue, + worktree, + review_state_inspector, + )? { + RetainedReviewLaneLoad::Skip => continue, + RetainedReviewLaneLoad::Blocked(blocked) => { + apply_passive_retained_manual_attention_with_run_identity( + tracker, + project, + workflow, + &blocked.issue, + &blocked.worktree, + &blocked.run_identity, + &blocked.reason, + )?; + + continue; + }, + RetainedReviewLaneLoad::Ready(lane) => *lane, + }; + + if let Some(reason) = validate_review_orchestration_marker( + &lane.snapshot, + &lane.review_state, + &lane.orchestration_marker, + ) { + apply_passive_retained_manual_attention( + tracker, + project, + workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + reason, + )?; + + continue; + } + + reconcile_retained_review_lane( + tracker, + project, + workflow, + state_store, + &lane, + &mut github_token, + now_unix_epoch, + )?; + } + + Ok(()) +} + +fn eligible_post_review_orchestration_issue( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, + success_state: &str, + opt_out_label: &str, + needs_attention_label: &str, +) -> Result +where + T: IssueTracker + ?Sized, +{ + Ok(tracker::issue_has_label_with_server_confirmation( + tracker, + issue, + &tracker::automation_active_label(service_id), + )? && issue.state.name == success_state + && !issue.has_label(opt_out_label) + && !issue.has_label(needs_attention_label)) +} + +fn load_retained_review_lane( + project_id: &str, + state_store: &StateStore, + issue: TrackerIssue, + worktree: WorktreeMapping, + review_state_inspector: &I, +) -> Result +where + I: PullRequestReviewStateInspector, +{ + let review_handoff = + state_store.review_handoff_marker(project_id, &issue.id, worktree.branch_name())?; + let Some(review_handoff) = review_handoff else { + return Ok(blocked_retained_review_lane( + issue, + worktree, + None, + "missing_review_handoff_record", + )); + }; + let local_branch_name = match worktree_checkout_branch_name(worktree.worktree_path()) { + Ok(local_branch_name) => local_branch_name, + Err(_error) => { + return Ok(blocked_retained_review_lane( + issue, + worktree, + Some(&review_handoff), + "worktree_checkout_branch_read_failed", + )); + }, + }; + let Some(local_branch_name) = local_branch_name else { + return Ok(blocked_retained_review_lane( + issue, + worktree, + Some(&review_handoff), + "worktree_checkout_branch_missing", + )); + }; + let local_head_oid = match worktree_head_oid(worktree.worktree_path()) { + Ok(local_head_oid) => local_head_oid, + Err(_error) => { + return Ok(blocked_retained_review_lane( + issue, + worktree, + Some(&review_handoff), + "worktree_head_read_failed", + )); + }, + }; + let Some(local_head_oid) = local_head_oid else { + return Ok(blocked_retained_review_lane( + issue, + worktree, + Some(&review_handoff), + "worktree_head_missing", + )); + }; + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(review_handoff.clone()), + local_branch_name: Some(local_branch_name), + local_head_oid: Some(local_head_oid.clone()), + }; + let review_state = match load_retained_review_lane_review_state( + &snapshot, + review_state_inspector, + )? { + RetainedReviewLaneReviewLoad::Skip => return Ok(RetainedReviewLaneLoad::Skip), + RetainedReviewLaneReviewLoad::Blocked(reason) => + return Ok(blocked_retained_review_lane( + snapshot.issue, + snapshot.worktree, + Some(&review_handoff), + &reason, + )), + RetainedReviewLaneReviewLoad::ReviewState(review_state) => *review_state, + }; + let orchestration_marker = ensure_review_orchestration_marker( + project_id, + state_store, + &snapshot.issue, + &review_handoff, + &local_head_oid, + )?; + + Ok(RetainedReviewLaneLoad::Ready(Box::new(RetainedReviewLane { + snapshot, + review_state, + orchestration_marker, + }))) +} + +fn load_retained_review_lane_review_state( + snapshot: &PostReviewLaneSnapshot, + review_state_inspector: &I, +) -> Result +where + I: PullRequestReviewStateInspector, +{ + let review_state = match load_post_review_lane_review_state(snapshot, review_state_inspector)? { + PostReviewLaneStateLoad::Classification(classification) => + return Ok(RetainedReviewLaneReviewLoad::Blocked(classification.reason)), + PostReviewLaneStateLoad::ReviewState(review_state) => Box::new(review_state), + }; + + if review_state.state == "MERGED" { + return Ok(RetainedReviewLaneReviewLoad::Skip); + } + if review_state.state != "OPEN" { + return Ok(RetainedReviewLaneReviewLoad::Blocked(String::from( + "pull_request_not_open", + ))); + } + if review_state.is_draft { + return Ok(RetainedReviewLaneReviewLoad::Blocked(String::from( + "pull_request_is_draft", + ))); + } + + Ok(RetainedReviewLaneReviewLoad::ReviewState(review_state)) +} + +fn blocked_retained_review_lane( + issue: TrackerIssue, + worktree: WorktreeMapping, + review_handoff: Option<&ReviewHandoffMarker>, + reason: &str, +) -> RetainedReviewLaneLoad { + let (run_id, attempt_number) = + retained_review_run_identity(worktree.worktree_path(), review_handoff); + + RetainedReviewLaneLoad::Blocked(Box::new(RetainedReviewLaneBlocked { + issue, + worktree, + run_identity: RetainedReviewRunIdentity { run_id, attempt_number }, + reason: reason.to_owned(), + })) +} + +fn retained_review_run_identity( + worktree_path: &Path, + review_handoff: Option<&ReviewHandoffMarker>, +) -> (String, i64) { + if let Some(review_handoff) = review_handoff { + return (review_handoff.run_id().to_owned(), review_handoff.attempt_number()); + } + if let Ok(Some(marker)) = state::read_run_activity_marker_snapshot(worktree_path) { + return (marker.run_id().to_owned(), marker.attempt_number()); + } + + (String::from("retained-review-orchestration"), 1) +} + +fn reconcile_retained_review_lane( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + lane: &RetainedReviewLane, + github_token: &mut Option, + now_unix_epoch: i64, +) -> Result<()> +where + T: IssueTracker, +{ + if !project.codex().external_review_enabled() { + return handle_internal_review_only_lane( + tracker, + project, + workflow, + state_store, + lane, + github_token, + now_unix_epoch, + ); + } + + let phase = ReviewOrchestrationPhase::parse(lane.orchestration_marker.phase()) + .map_err(|error| eyre::eyre!("Failed to parse retained review orchestration phase: {error}"))?; + + match phase { + ReviewOrchestrationPhase::RequestPending => handle_request_pending_phase( + tracker, + project, + workflow, + state_store, + lane, + github_token, + ), + ReviewOrchestrationPhase::WaitingForAck => handle_waiting_for_ack_phase( + tracker, + project, + workflow, + state_store, + lane, + github_token, + now_unix_epoch, + ), + ReviewOrchestrationPhase::WaitingForResult + | ReviewOrchestrationPhase::PassWaitingForGates => { + let mut runtime = RetainedReviewRuntime { + tracker, + project, + workflow, + state_store, + github_token, + now_unix_epoch, + }; + + handle_waiting_for_result_phase(&mut runtime, lane, phase) + }, + ReviewOrchestrationPhase::RepairRequired => Ok(()), + ReviewOrchestrationPhase::WaitingForMerge => handle_waiting_for_merge_phase( + tracker, + project, + workflow, + lane, + now_unix_epoch, + "external_review_merge_visibility_timeout", + ), + } +} + +fn handle_internal_review_only_lane( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + lane: &RetainedReviewLane, + github_token: &mut Option, + now_unix_epoch: i64, +) -> Result<()> +where + T: IssueTracker, +{ + let phase = ReviewOrchestrationPhase::parse(lane.orchestration_marker.phase()) + .map_err(|error| eyre::eyre!("Failed to parse retained review orchestration phase: {error}"))?; + + if phase == ReviewOrchestrationPhase::WaitingForMerge { + return handle_waiting_for_merge_phase( + tracker, + project, + workflow, + lane, + now_unix_epoch, + "internal_review_only_merge_visibility_timeout", + ); + } + if external_review_requires_repair(&lane.review_state, &lane.orchestration_marker) + || failed_checks_require_repair( + lane.review_state.status_check_rollup_state.as_deref(), + &lane.review_state.merge_state_status, + ) + || merge_state_requires_review_repair( + &lane.review_state.mergeable, + &lane.review_state.merge_state_status, + ) + .is_some() + { + return write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + } + if review_state_landing_requires_agent_fallback(&lane.review_state) { + return write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + } + if !review_state_landing_gates_satisfied(&lane.review_state) { + return Ok(()); + } + + let mut runtime = RetainedReviewRuntime { + tracker, + project, + workflow, + state_store, + now_unix_epoch, + github_token, + }; + + start_retained_admin_merge( + &mut runtime, + lane, + RetainedAdminMergeReasons { + admin_merge_unavailable: "internal_review_only_admin_merge_unavailable", + admin_merge_failed: "internal_review_only_admin_merge_failed", + }, + ) +} + +fn handle_request_pending_phase( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + lane: &RetainedReviewLane, + github_token: &mut Option, +) -> Result<()> +where + T: IssueTracker, +{ + match external_review_request_ci_gate(&lane.review_state) { + ExternalReviewRequestCiGate::Ready => {}, + ExternalReviewRequestCiGate::WaitForGreenChecks => return Ok(()), + ExternalReviewRequestCiGate::RepairRequired => { + return write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + }, + ExternalReviewRequestCiGate::ManualAttention(reason) => { + return apply_passive_retained_manual_attention( + tracker, + project, + workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + reason, + ); + }, + } + + let github_token = retained_review_github_token(project, github_token)?; + let (comment_id, created_at_unix_epoch) = github::post_pull_request_issue_comment( + lane.snapshot.worktree.worktree_path(), + lane.review_state.url.as_str(), + EXTERNAL_REVIEW_REQUEST_BODY, + github_token, + )?; + + write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::WaitingForAck, + RetainedReviewOrchestrationMarkerFields { + request_comment_database_id: Some(comment_id), + request_created_at_unix_epoch: Some(created_at_unix_epoch), + request_retry_count: 0, + ..RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker) + }, + ) +} + +fn handle_waiting_for_ack_phase( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + lane: &RetainedReviewLane, + github_token: &mut Option, + now_unix_epoch: i64, +) -> Result<()> +where + T: IssueTracker, +{ + if request_comment_has_eyes(&lane.review_state, &lane.orchestration_marker).unwrap_or(false) { + return write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::WaitingForResult, + RetainedReviewOrchestrationMarkerFields { + ..RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker) + }, + ); + } + + let Some(request_created_at_unix_epoch) = + lane.orchestration_marker.request_created_at_unix_epoch() + else { + return Ok(()); + }; + + if now_unix_epoch - request_created_at_unix_epoch <= EXTERNAL_REVIEW_ACK_TIMEOUT_SECS { + return Ok(()); + } + if lane.orchestration_marker.request_retry_count() == 0 { + let github_token = retained_review_github_token(project, github_token)?; + let (comment_id, created_at_unix_epoch) = github::post_pull_request_issue_comment( + lane.snapshot.worktree.worktree_path(), + lane.review_state.url.as_str(), + EXTERNAL_REVIEW_REQUEST_BODY, + github_token, + )?; + + return write_retained_review_orchestration_marker( + state_store, + lane, + ReviewOrchestrationPhase::WaitingForAck, + RetainedReviewOrchestrationMarkerFields { + request_comment_database_id: Some(comment_id), + request_created_at_unix_epoch: Some(created_at_unix_epoch), + request_retry_count: 1, + ..RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker) + }, + ); + } + + apply_passive_retained_manual_attention( + tracker, + project, + workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + "external_review_ack_timeout", + ) +} + +fn handle_waiting_for_result_phase( + runtime: &mut RetainedReviewRuntime<'_, T>, + lane: &RetainedReviewLane, + phase: ReviewOrchestrationPhase, +) -> Result<()> +where + T: IssueTracker, +{ + if external_review_requires_repair(&lane.review_state, &lane.orchestration_marker) { + return write_retained_review_orchestration_marker( + runtime.state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields { + external_round_count: lane + .orchestration_marker + .external_round_count() + .saturating_add(1), + ..RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker) + }, + ); + } + if failed_checks_require_repair( + lane.review_state.status_check_rollup_state.as_deref(), + &lane.review_state.merge_state_status, + ) || merge_state_requires_review_repair( + &lane.review_state.mergeable, + &lane.review_state.merge_state_status, + ) + .is_some() + { + return write_retained_review_orchestration_marker( + runtime.state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + } + if external_review_has_strict_pass_signals(&lane.review_state, &lane.orchestration_marker) { + if review_state_clean_path_landing_gates_satisfied(&lane.review_state) { + return start_retained_admin_merge( + runtime, + lane, + RetainedAdminMergeReasons { + admin_merge_unavailable: "external_review_admin_merge_unavailable", + admin_merge_failed: "external_review_admin_merge_failed", + }, + ); + } + if review_state_landing_requires_agent_fallback(&lane.review_state) { + return write_retained_review_orchestration_marker( + runtime.state_store, + lane, + ReviewOrchestrationPhase::RepairRequired, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + } + if phase == ReviewOrchestrationPhase::WaitingForResult { + return write_retained_review_orchestration_marker( + runtime.state_store, + lane, + ReviewOrchestrationPhase::PassWaitingForGates, + RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker), + ); + } + + return Ok(()); + } + if external_review_result_arrived(&lane.review_state, &lane.orchestration_marker) { + return apply_passive_retained_manual_attention( + runtime.tracker, + runtime.project, + runtime.workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + "external_review_pass_signal_missing", + ); + } + + Ok(()) +} + +fn external_review_requires_repair( + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> bool { + review_state.unresolved_review_threads > 0 + || matches!(review_state.review_decision.as_deref(), Some("CHANGES_REQUESTED")) + || external_review_has_actionable_feedback(review_state, marker) +} + +fn handle_waiting_for_merge_phase( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + lane: &RetainedReviewLane, + now_unix_epoch: i64, + timeout_reason: &str, +) -> Result<()> +where + T: IssueTracker, +{ + let Some(auto_merge_enabled_at_unix_epoch) = + lane.orchestration_marker.auto_merge_enabled_at_unix_epoch() + else { + return Ok(()); + }; + + if now_unix_epoch - auto_merge_enabled_at_unix_epoch + <= EXTERNAL_REVIEW_MERGE_VISIBILITY_TIMEOUT_SECS + { + return Ok(()); + } + + apply_passive_retained_manual_attention( + tracker, + project, + workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + timeout_reason, + ) +} + +fn retained_review_merge_subject(lane: &RetainedReviewLane) -> Result { + let review_handoff = lane.snapshot.review_handoff.as_ref().ok_or_else(|| { + eyre::eyre!( + "Retained admin merge for `{}` requires a matching runtime review handoff on branch `{}`.", + lane.snapshot.issue.identifier, + lane.snapshot.worktree.branch_name(), + ) + })?; + + if review_handoff.pr_head_oid() != lane.orchestration_marker.head_sha() { + eyre::bail!( + "Retained admin merge for `{}` requires review handoff head `{}` to match orchestration head `{}`.", + lane.snapshot.issue.identifier, + review_handoff.pr_head_oid(), + lane.orchestration_marker.head_sha(), + ); + } + + let head_subject = retained_review_head_commit_subject( + lane.snapshot.worktree.worktree_path(), + lane.orchestration_marker.head_sha(), + )?; + + commit_message::build_landed_merge_commit_message( + &head_subject, + &lane.snapshot.issue.identifier, + ) +} + +fn retained_review_head_commit_subject(worktree_path: &Path, head_sha: &str) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["log", "-1", "--format=%s"]) + .arg(head_sha) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to read retained review head commit subject `{}` in `{}`: {}", + head_sha, + worktree_path.display(), + stderr.trim() + ); + } + + String::from_utf8(output.stdout) + .map(|stdout| stdout.trim_end_matches(['\n', '\r']).to_owned()) + .map_err(Into::into) +} + +fn start_retained_admin_merge( + runtime: &mut RetainedReviewRuntime<'_, T>, + lane: &RetainedReviewLane, + reasons: RetainedAdminMergeReasons, +) -> Result<()> +where + T: IssueTracker, +{ + if !lane.review_state.merge_commit_allowed { + return apply_passive_retained_manual_attention( + runtime.tracker, + runtime.project, + runtime.workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + reasons.admin_merge_unavailable, + ); + } + + let merge_subject = match retained_review_merge_subject(lane) { + Ok(subject) => subject, + Err(error) => { + tracing::warn!( + issue_id = lane.snapshot.issue.id, + issue = lane.snapshot.issue.identifier, + branch = lane.snapshot.worktree.branch_name(), + ?error, + "Retained admin merge could not derive a compliant landed change record." + ); + + return apply_passive_retained_manual_attention( + runtime.tracker, + runtime.project, + runtime.workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + "retained_admin_merge_subject_unavailable", + ); + }, + }; + let github_token = retained_review_github_token(runtime.project, &mut *runtime.github_token)?; + let merge_succeeded = match github::admin_merge_pull_request( + lane.snapshot.worktree.worktree_path(), + lane.review_state.url.as_str(), + lane.orchestration_marker.head_sha(), + Some(merge_subject.as_str()), + github_token, + ) { + Ok(()) => true, + Err(_error) => + matches!( + github::pull_request_is_merged_at_head( + lane.snapshot.worktree.worktree_path(), + lane.review_state.url.as_str(), + lane.orchestration_marker.head_sha(), + github_token, + ), + Ok(true) + ), + }; + + if merge_succeeded { + return write_retained_review_orchestration_marker( + runtime.state_store, + lane, + ReviewOrchestrationPhase::WaitingForMerge, + RetainedReviewOrchestrationMarkerFields { + auto_merge_enabled_at_unix_epoch: Some(runtime.now_unix_epoch), + ..RetainedReviewOrchestrationMarkerFields::from_marker(&lane.orchestration_marker) + }, + ); + } + + apply_passive_retained_manual_attention( + runtime.tracker, + runtime.project, + runtime.workflow, + &lane.snapshot.issue, + &lane.snapshot.worktree, + &lane.orchestration_marker, + reasons.admin_merge_failed, + ) +} + +fn retained_review_github_token<'a>( + project: &ServiceConfig, + github_token: &'a mut Option, +) -> Result<&'a str> { + if github_token.is_none() { + *github_token = Some(resolve_configured_env_var( + "github.token_env_var", + Some(project.github().token_env_var()), + )?); + } + + github_token + .as_deref() + .ok_or_else(|| eyre::eyre!("Retained review orchestration requires a configured GitHub token.")) +} + +fn write_retained_review_orchestration_marker( + state_store: &StateStore, + lane: &RetainedReviewLane, + phase: ReviewOrchestrationPhase, + fields: RetainedReviewOrchestrationMarkerFields, +) -> Result<()> +{ + let local_head_oid = lane + .snapshot + .local_head_oid + .as_deref() + .ok_or_else(|| eyre::eyre!("Retained review orchestration requires a local lane HEAD."))?; + let marker = ReviewOrchestrationMarker::new( + lane.orchestration_marker.run_id().to_owned(), + lane.orchestration_marker.attempt_number(), + lane.snapshot.worktree.branch_name().to_owned(), + lane.review_state.url.clone(), + local_head_oid.to_owned(), + phase.as_str(), + fields.request_comment_database_id, + fields.request_created_at_unix_epoch, + None, + fields.request_retry_count, + fields.external_round_count, + fields.auto_merge_enabled_at_unix_epoch, + ); + + state_store.upsert_review_orchestration_marker( + lane.snapshot.worktree.project_id(), + &lane.snapshot.issue.id, + &marker, + )?; + + Ok(()) +} + +fn ensure_review_orchestration_marker( + project_id: &str, + state_store: &StateStore, + issue: &TrackerIssue, + review_handoff: &ReviewHandoffMarker, + local_head_oid: &str, +) -> Result +{ + if let Some(marker) = + state_store.review_orchestration_marker(project_id, &issue.id, review_handoff)? + { + return Ok(marker); + } + + let marker = ReviewOrchestrationMarker::new( + review_handoff.run_id().to_owned(), + review_handoff.attempt_number(), + review_handoff.branch_name().to_owned(), + review_handoff.pr_url().to_owned(), + local_head_oid.to_owned(), + ReviewOrchestrationPhase::RequestPending.as_str(), + None, + None, + None, + 0, + 0, + None, + ); + + state_store.upsert_review_orchestration_marker(project_id, &issue.id, &marker)?; + + Ok(marker) +} + +fn apply_passive_retained_manual_attention( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + issue: &TrackerIssue, + worktree: &WorktreeMapping, + orchestration_marker: &ReviewOrchestrationMarker, + reason: &str, +) -> Result<()> +where + T: IssueTracker, +{ + apply_passive_retained_manual_attention_with_run_identity( + tracker, + project, + workflow, + issue, + worktree, + &RetainedReviewRunIdentity { + run_id: orchestration_marker.run_id().to_owned(), + attempt_number: orchestration_marker.attempt_number(), + }, + reason, + ) +} + +fn apply_passive_retained_manual_attention_with_run_identity( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + issue: &TrackerIssue, + worktree: &WorktreeMapping, + run_identity: &RetainedReviewRunIdentity, + reason: &str, +) -> Result<()> +where + T: IssueTracker, +{ + let synthetic_issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: worktree.branch_name().to_owned(), + issue_identifier: issue.identifier.clone(), + path: worktree.worktree_path().to_path_buf(), + reused_existing: true, + }, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: run_identity.attempt_number, + run_id: run_identity.run_id.clone(), + retry_budget_base: 0, + }; + let worktree_path = + relative_worktree_path_for_path(project, synthetic_issue_run.worktree.path.as_path()); + let _ = apply_terminal_failure_writeback( + tracker, + TerminalFailureWritebackRuntime { service_id: project.service_id(), state_store: None }, + workflow, + &synthetic_issue_run, + &worktree_path, + true, + &Report::new(RetainedReviewNeedsAttention { reason: reason.to_owned() }), + )?; + + Ok(()) +} + +fn plan_project_issue_run_with_exclusions( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + dry_run: bool, + excluded_issue_ids: &[&str], +) -> Result> +where + T: IssueTracker, +{ + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + + state_store.configure_dispatch_slot_root( + project.service_id(), + project.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + )?; + + let recovered_state = + recover_runtime_state_from_tracker_and_worktrees(tracker, project, workflow, state_store)?; + + if !dry_run { + reconcile_project_state(tracker, project, workflow, state_store, &worktree_manager)?; + reconcile_post_review_orchestration(tracker, project, workflow, state_store)?; + } + + let issues = queued_issues_for_dispatch(tracker, project, workflow, dry_run)?; + let Some(selected_issue) = select_project_issue_run_candidate( + tracker, + project, + workflow, + state_store, + recovered_state, + issues, + excluded_issue_ids, + )? else { + return Ok(None); + }; + let mut refreshed_issues = tracker.refresh_issues(slice::from_ref(&selected_issue.issue.id))?; + let Some(issue) = refreshed_issues.pop() else { + return Ok(None); + }; + let dispatch_mode = selected_issue.dispatch_mode; + let preferred_run_identity = selected_issue.preferred_run_identity; + let concurrency = ConcurrencySnapshot::new(project.service_id(), state_store)?; + + if !dry_run && dispatch_mode != IssueDispatchMode::Closeout { + ensure_project_has_no_merged_worktree_cleanup_debt(project)?; + } + if !concurrency.has_global_capacity(workflow.frontmatter().execution()) { + return Ok(None); + } + if !dispatch_mode.allows_issue( + tracker, + &issue, + project, + workflow, + state_store, + RetryIssueStateHint::default(), + )? { + if dispatch_mode == IssueDispatchMode::Closeout + && let Some(reason) = + closeout_dispatch_block_reason(tracker, &issue, project, workflow, state_store)? + { + if !dry_run { + eyre::bail!("retained closeout dispatch blocked: {reason}"); + } + + return Ok(None); + } + + return replan_project_issue_run_after_excluding( + tracker, + project, + workflow, + state_store, + dry_run, + excluded_issue_ids, + issue.id.as_str(), + ); + } + + let Some(issue_run) = prepare_issue_run( + PrepareIssueRunContext { + tracker, + project, + workflow, + state_store, + worktree_manager: &worktree_manager, + dry_run, + lease_preacquired: false, + dispatch_mode, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: preferred_run_identity.as_ref().map(|identity| { + PreferredRunIdentity { + run_id: identity.run_id.as_str(), + attempt_number: identity.attempt_number, + } + }), + preferred_retry_budget_base: None, + }, + issue, + )? + else { + return Ok(None); + }; + + Ok(Some(issue_run)) +} + +fn select_project_issue_run_candidate( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + recovered_state: RecoveredRuntimeState, + issues: Vec, + excluded_issue_ids: &[&str], +) -> Result> +where + T: IssueTracker, +{ + let selected_retry_issue = + select_recovered_retry_issue_candidate(project, state_store, recovered_state, excluded_issue_ids)?; + let selected_post_review_issue = select_post_review_issue_candidate( + tracker, + project, + workflow, + state_store, + excluded_issue_ids, + )?; + + if let Some(candidate) = selected_retry_issue.or(selected_post_review_issue) { + return Ok(Some(candidate)); + } + + Ok(select_issue_candidate_with_exclusions( + tracker, + issues, + workflow, + state_store, + project.service_id(), + excluded_issue_ids, + )? + .map(|issue| SelectedIssueRunCandidate::new(issue, IssueDispatchMode::Normal))) +} + +fn select_recovered_retry_issue_candidate( + project: &ServiceConfig, + state_store: &StateStore, + recovered_state: RecoveredRuntimeState, + excluded_issue_ids: &[&str], +) -> Result> { + for issue in recovered_state.active_issues { + if excluded_issue_ids.contains(&issue.id.as_str()) { + continue; + } + if state_store.issue_has_active_shared_claim(project.service_id(), &issue.id)? { + continue; + } + + return Ok(Some(SelectedIssueRunCandidate::new( + issue, + IssueDispatchMode::Retry, + ))); + } + + Ok(None) +} + +fn queued_issues_for_dispatch( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + dry_run: bool, +) -> Result> +where + T: IssueTracker, +{ + let queue_label = tracker::automation_queue_label(project.service_id()); + + clear_terminal_queued_lane_labels( + tracker, + project, + workflow, + tracker.list_issues_with_label(&queue_label)?, + dry_run, + ) +} + +fn clear_terminal_queued_lane_labels( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + issues: Vec, + dry_run: bool, +) -> Result> +where + T: IssueTracker, +{ + let mut nonterminal_issues = Vec::with_capacity(issues.len()); + + for issue in issues { + if is_terminal_issue(&issue, workflow) { + if !dry_run { + tracker::clear_automation_lane_labels(tracker, &issue, project.service_id())?; + + tracing::info!( + project_id = project.service_id(), + issue_id = issue.id, + issue = issue.identifier, + "Cleared automation lane labels from terminal queued issue." + ); + } + + continue; + } + + nonterminal_issues.push(issue); + } + + Ok(nonterminal_issues) +} + +fn replan_project_issue_run_after_excluding( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + dry_run: bool, + excluded_issue_ids: &[&str], + issue_id: &str, +) -> Result> +where + T: IssueTracker, +{ + let mut next_excluded_issue_ids = excluded_issue_ids.to_vec(); + + next_excluded_issue_ids.push(issue_id); + + plan_project_issue_run_with_exclusions( + tracker, + project, + workflow, + state_store, + dry_run, + &next_excluded_issue_ids, + ) +} + +fn select_post_review_issue_candidate( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + excluded_issue_ids: &[&str], +) -> Result> +where + T: IssueTracker, +{ + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + + select_post_review_issue_candidate_with_inspector( + tracker, + project, + workflow, + state_store, + excluded_issue_ids, + &review_state_inspector, + ) +} + +fn select_post_review_issue_candidate_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + excluded_issue_ids: &[&str], + review_state_inspector: &I, +) -> Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + if let Some(issue) = select_post_review_repair_issue_candidate_with_inspector( + tracker, + project, + workflow, + state_store, + excluded_issue_ids, + review_state_inspector, + )? { + return Ok(Some(SelectedIssueRunCandidate::new( + issue, + IssueDispatchMode::ReviewRepair, + ))); + } + + select_post_review_closeout_issue_candidate_with_inspector( + tracker, + project, + workflow, + state_store, + excluded_issue_ids, + review_state_inspector, + ) +} + +fn select_post_review_repair_issue_candidate_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + excluded_issue_ids: &[&str], + review_state_inspector: &I, +) -> Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let lanes = build_post_review_lane_statuses( + tracker, + project, + workflow, + state_store, + review_state_inspector, + )?; + let candidate_issue_ids = lanes + .iter() + .filter(|lane| lane.classification == "needs_review_repair") + .filter(|lane| !excluded_issue_ids.contains(&lane.issue_id.as_str())) + .map(|lane| lane.issue_id.clone()) + .collect::>(); + + if candidate_issue_ids.is_empty() { + return Ok(None); + } + + let issues = tracker.refresh_issues(&candidate_issue_ids)?; + let mut issues_by_id = + issues.into_iter().map(|issue| (issue.id.clone(), issue)).collect::>(); + + for lane in lanes { + if lane.classification != "needs_review_repair" { + continue; + } + if excluded_issue_ids.contains(&lane.issue_id.as_str()) { + continue; + } + if state_store.issue_has_active_shared_claim(project.service_id(), &lane.issue_id)? { + continue; + } + + if let Some(issue) = issues_by_id.remove(&lane.issue_id) { + return Ok(Some(issue)); + } + } + + Ok(None) +} + +fn select_post_review_closeout_issue_candidate_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + excluded_issue_ids: &[&str], + review_state_inspector: &I, +) -> Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let completed_state = workflow.frontmatter().tracker().resolved_completed_state(); + let lanes = build_post_review_lane_statuses( + tracker, + project, + workflow, + state_store, + review_state_inspector, + )?; + let candidate_issue_ids = lanes + .iter() + .filter(|lane| post_review_lane_is_closeout_candidate(lane, completed_state)) + .filter(|lane| !excluded_issue_ids.contains(&lane.issue_id.as_str())) + .map(|lane| lane.issue_id.clone()) + .collect::>(); + + if candidate_issue_ids.is_empty() { + return Ok(None); + } + + let issues = tracker.refresh_issues(&candidate_issue_ids)?; + let mut issues_by_id = + issues.into_iter().map(|issue| (issue.id.clone(), issue)).collect::>(); + + for lane in lanes { + let is_closeout_candidate = post_review_lane_is_closeout_candidate(&lane, completed_state); + + if !is_closeout_candidate { + continue; + } + if excluded_issue_ids.contains(&lane.issue_id.as_str()) { + continue; + } + if state_store.issue_has_active_shared_claim(project.service_id(), &lane.issue_id)? { + continue; + } + + if let Some(issue) = issues_by_id.remove(&lane.issue_id) { + let preferred_run_identity = + retained_closeout_preferred_run_identity(state_store, project.service_id(), &issue)?; + + return Ok(Some(SelectedIssueRunCandidate { + issue, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity, + })); + } + } + + Ok(None) +} + +fn retained_closeout_preferred_run_identity( + state_store: &StateStore, + project_id: &str, + issue: &TrackerIssue, +) -> Result> +{ + let Some(worktree) = state_store.worktree_for_issue(&issue.id)? else { + return Ok(None); + }; + let Some(review_handoff) = + state_store.review_handoff_marker(project_id, &issue.id, worktree.branch_name())? + else { + return Ok(None); + }; + let identity = RetainedReviewRunIdentity { + run_id: review_handoff.run_id().to_owned(), + attempt_number: review_handoff.attempt_number(), + }; + + if retained_closeout_run_identity_is_reusable(state_store, &issue.id, &identity)? + || retained_closeout_handoff_identity_is_reusable_after_parent_reconciliation( + state_store, + &issue.id, + &identity, + &worktree, + )? + { + return Ok(Some(identity)); + } + + Ok(None) +} + +fn retained_closeout_run_identity_is_reusable( + state_store: &StateStore, + issue_id: &str, + identity: &RetainedReviewRunIdentity, +) -> Result { + if state_store.issue_has_retry_budget_attempt_after(issue_id, identity.attempt_number)? { + return Ok(false); + } + + let Some(existing_attempt) = state_store.run_attempt(&identity.run_id)? else { + return Ok(true); + }; + + if existing_attempt.issue_id() != issue_id + || existing_attempt.attempt_number() != identity.attempt_number + { + return Ok(false); + } + + Ok(!matches!( + existing_attempt.status(), + "failed" | "interrupted" | TERMINAL_GUARDED_RUN_STATUS + )) +} + +fn retained_closeout_handoff_identity_is_reusable_after_parent_reconciliation( + state_store: &StateStore, + issue_id: &str, + identity: &RetainedReviewRunIdentity, + worktree: &WorktreeMapping, +) -> Result { + if state_store.issue_has_retry_budget_attempt_after(issue_id, identity.attempt_number)? { + return Ok(false); + } + + let Some(existing_attempt) = state_store.run_attempt(&identity.run_id)? else { + return Ok(false); + }; + + if existing_attempt.issue_id() != issue_id + || existing_attempt.attempt_number() != identity.attempt_number + { + return Ok(false); + } + if !matches!(existing_attempt.status(), "failed" | "interrupted") { + return Ok(false); + } + if worktree_has_retry_schedule_for_run(worktree.worktree_path(), identity)? { + return Ok(false); + } + + Ok(true) +} + +fn worktree_has_retry_schedule_for_run( + worktree_path: &Path, + identity: &RetainedReviewRunIdentity, +) -> Result { + let Some(marker) = state::read_run_activity_marker_snapshot(worktree_path)? else { + return Ok(false); + }; + + Ok(marker.run_id() == identity.run_id + && marker.attempt_number() == identity.attempt_number + && marker.retry_kind().is_some()) +} + +fn post_review_lane_is_closeout_candidate( + lane: &OperatorPostReviewLaneStatus, + _completed_state: &str, +) -> bool { + lane.classification == "continue" && lane.reason == "pull_request_merged_closeout_pending" +} + +fn run_target_issue_once( + context: TargetIssueRunContext<'_, T>, +) -> Result> +where + T: IssueTracker, +{ + let worktree_manager = WorktreeManager::new( + context.project.service_id(), + context.project.repo_root(), + context.project.worktree_root(), + ); + + context.state_store.configure_dispatch_slot_root( + context.project.service_id(), + context.project.worktree_root(), + context.workflow.frontmatter().execution().max_concurrent_agents(), + )?; + + let issue_id = resolve_target_issue_id(context.tracker, context.issue_id)?; + + if !context.dry_run { + context.state_store.canonicalize_issue_identity(context.issue_id, &issue_id)?; + } + if context.lease_preacquired && !context.dry_run { + adopt_preacquired_target_issue_lease(&context, &issue_id)?; + } + if !context.lease_preacquired { + recover_runtime_state_from_tracker_and_worktrees( + context.tracker, + context.project, + context.workflow, + context.state_store, + )?; + + if !context.dry_run { + reconcile_project_state( + context.tracker, + context.project, + context.workflow, + context.state_store, + &worktree_manager, + )?; + } + } + + let Some(issue) = refresh_issue(context.tracker, &issue_id)? else { + return Ok(None); + }; + let closeout_preferred_run_identity = + target_closeout_preferred_run_identity(&context, &issue)?; + let preferred_run_identity = preferred_run_identity_with_closeout_fallback( + context.preferred_run_identity, + closeout_preferred_run_identity.as_ref(), + ); + let retry_state_hint = RetryIssueStateHint { + preferred_issue_state: context.preferred_issue_state, + preferred_initial_issue_state: context.preferred_initial_issue_state, + }; + + if !context.dispatch_mode.allows_issue( + context.tracker, + &issue, + context.project, + context.workflow, + context.state_store, + retry_state_hint, + )? { + ensure_target_closeout_dispatch_is_unblocked(&context, &issue)?; + + return Ok(None); + } + if !context.lease_preacquired + && context + .state_store + .issue_has_active_shared_claim(context.project.service_id(), &issue_id)? + { + return Ok(None); + } + if !context.lease_preacquired { + let concurrency = ConcurrencySnapshot::new(context.project.service_id(), context.state_store)?; + + if !concurrency.has_global_capacity(context.workflow.frontmatter().execution()) { + return Ok(None); + } + } + if !context.dry_run && context.dispatch_mode != IssueDispatchMode::Closeout { + ensure_project_has_no_merged_worktree_cleanup_debt(context.project)?; + } + + let Some(issue_run) = prepare_issue_run( + PrepareIssueRunContext { + tracker: context.tracker, + project: context.project, + workflow: context.workflow, + state_store: context.state_store, + worktree_manager: &worktree_manager, + dry_run: context.dry_run, + lease_preacquired: context.lease_preacquired, + dispatch_mode: context.dispatch_mode, + preferred_issue_state: context.preferred_issue_state, + preferred_initial_issue_state: context.preferred_initial_issue_state, + preferred_run_identity, + preferred_retry_budget_base: context.preferred_retry_budget_base, + }, + issue, + )? + else { + return Ok(None); + }; + + complete_issue_run( + context.tracker, + context.project, + context.workflow, + context.state_store, + issue_run, + context.dry_run, + ) +} + +fn ensure_target_closeout_dispatch_is_unblocked( + context: &TargetIssueRunContext<'_, T>, + issue: &TrackerIssue, +) -> Result<()> +where + T: IssueTracker, +{ + if context.dry_run || context.dispatch_mode != IssueDispatchMode::Closeout { + return Ok(()); + } + + let Some(reason) = closeout_dispatch_block_reason( + context.tracker, + issue, + context.project, + context.workflow, + context.state_store, + )? + else { + return Ok(()); + }; + + eyre::bail!("retained closeout dispatch blocked: {reason}"); +} + +fn target_closeout_preferred_run_identity( + context: &TargetIssueRunContext<'_, T>, + issue: &TrackerIssue, +) -> Result> +where + T: IssueTracker, +{ + if context.dispatch_mode != IssueDispatchMode::Closeout + || context.preferred_run_identity.is_some() + { + return Ok(None); + } + + retained_closeout_preferred_run_identity( + context.state_store, + context.project.service_id(), + issue, + ) +} + +fn preferred_run_identity_with_closeout_fallback<'a>( + preferred_run_identity: Option>, + closeout_preferred_run_identity: Option<&'a RetainedReviewRunIdentity>, +) -> Option> { + match (preferred_run_identity, closeout_preferred_run_identity) { + (Some(identity), _) => Some(identity), + (None, Some(identity)) => Some(PreferredRunIdentity { + run_id: identity.run_id.as_str(), + attempt_number: identity.attempt_number, + }), + (None, None) => None, + } +} + +fn run_target_issue_once_with_inferred_dispatch( + context: TargetIssueRunContext<'_, T>, +) -> Result> +where + T: IssueTracker, +{ + if let Some(summary) = run_target_issue_once(target_issue_run_context_with_dispatch_mode( + &context, + IssueDispatchMode::Normal, + ))? { + return Ok(Some(summary)); + } + if let Some(summary) = run_target_issue_once(target_issue_run_context_with_dispatch_mode( + &context, + IssueDispatchMode::Retry, + ))? { + return Ok(Some(summary)); + } + + run_target_status_visible_closeout_once(context) +} + +fn run_target_status_visible_closeout_once( + context: TargetIssueRunContext<'_, T>, +) -> Result> +where + T: IssueTracker, +{ + let target_issue_id = resolve_target_issue_id(context.tracker, context.issue_id)?; + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(context.project.github().token_env_var().to_owned()), + }; + let Some(candidate) = select_target_post_review_closeout_issue_candidate_with_inspector( + context.tracker, + context.project, + context.workflow, + context.state_store, + &target_issue_id, + context.issue_id, + &review_state_inspector, + )? else { + return Ok(None); + }; + let preferred_run_identity = + candidate + .preferred_run_identity + .as_ref() + .map(|identity| PreferredRunIdentity { + run_id: identity.run_id.as_str(), + attempt_number: identity.attempt_number, + }); + + run_target_issue_once(TargetIssueRunContext { + tracker: context.tracker, + project: context.project, + workflow: context.workflow, + state_store: context.state_store, + issue_id: context.issue_id, + preferred_issue_state: context.preferred_issue_state, + preferred_initial_issue_state: context.preferred_initial_issue_state, + dry_run: context.dry_run, + lease_preacquired: context.lease_preacquired, + preferred_issue_claim_fd: context.preferred_issue_claim_fd, + preferred_dispatch_slot_fd: context.preferred_dispatch_slot_fd, + preferred_dispatch_slot_index: context.preferred_dispatch_slot_index, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity, + preferred_retry_budget_base: context.preferred_retry_budget_base, + }) +} + +fn select_target_post_review_closeout_issue_candidate_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + target_issue_id: &str, + target_issue_reference: &str, + review_state_inspector: &I, +) -> Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let completed_state = workflow.frontmatter().tracker().resolved_completed_state(); + let lanes = build_post_review_lane_statuses( + tracker, + project, + workflow, + state_store, + review_state_inspector, + )?; + let closeout_lanes = lanes + .into_iter() + .filter(|lane| post_review_lane_is_closeout_candidate(lane, completed_state)) + .collect::>(); + + if closeout_lanes.is_empty() { + return Ok(None); + } + + let Some(target_lane) = closeout_lanes + .iter() + .find(|lane| lane.issue_id == target_issue_id) + else { + let visible_lanes = closeout_lanes + .iter() + .map(|lane| lane.issue_identifier.as_str()) + .collect::>() + .join(", "); + + eyre::bail!( + "targeted retained closeout mismatch: requested issue `{}` does not match status-visible retained closeout lane(s) `{}`", + target_issue_reference, + visible_lanes, + ); + }; + + if state_store.issue_has_active_shared_claim(project.service_id(), &target_lane.issue_id)? { + return Ok(None); + } + + let issue_ids = [target_lane.issue_id.clone()]; + let mut issues = tracker.refresh_issues(&issue_ids)?; + let Some(issue_index) = issues.iter().position(|issue| issue.id == target_lane.issue_id) else { + return Ok(None); + }; + let issue = issues.swap_remove(issue_index); + let preferred_run_identity = + retained_closeout_preferred_run_identity(state_store, project.service_id(), &issue)?; + + Ok(Some(SelectedIssueRunCandidate { + issue, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity, + })) +} + +fn target_issue_run_context_with_dispatch_mode<'a, T>( + context: &TargetIssueRunContext<'a, T>, + dispatch_mode: IssueDispatchMode, +) -> TargetIssueRunContext<'a, T> { + TargetIssueRunContext { + tracker: context.tracker, + project: context.project, + workflow: context.workflow, + state_store: context.state_store, + issue_id: context.issue_id, + preferred_issue_state: context.preferred_issue_state, + preferred_initial_issue_state: context.preferred_initial_issue_state, + dry_run: context.dry_run, + lease_preacquired: context.lease_preacquired, + preferred_issue_claim_fd: context.preferred_issue_claim_fd, + preferred_dispatch_slot_fd: context.preferred_dispatch_slot_fd, + preferred_dispatch_slot_index: context.preferred_dispatch_slot_index, + dispatch_mode, + preferred_run_identity: context.preferred_run_identity, + preferred_retry_budget_base: context.preferred_retry_budget_base, + } +} + +fn resolve_target_issue_id(tracker: &T, issue_reference: &str) -> Result +where + T: IssueTracker, +{ + if commit_message::looks_like_issue_identifier(issue_reference) + && let Some(issue) = tracker.get_issue_by_identifier(issue_reference)? + { + return Ok(issue.id); + } + + Ok(issue_reference.to_owned()) +} + +fn adopt_preacquired_target_issue_lease( + context: &TargetIssueRunContext<'_, T>, + issue_id: &str, +) -> Result<()> +where + T: IssueTracker, +{ + let preferred_run_identity = context.preferred_run_identity.ok_or_else(|| { + eyre::eyre!("daemon child lease handoff requires a planned run identifier") + })?; + let preferred_issue_state = context + .preferred_issue_state + .ok_or_else(|| eyre::eyre!("daemon child lease handoff requires a planned issue state"))?; + let issue_claim_fd = context.preferred_issue_claim_fd.ok_or_else(|| { + eyre::eyre!("daemon child lease handoff requires an inherited issue-claim fd") + })?; + let dispatch_slot_fd = context.preferred_dispatch_slot_fd.ok_or_else(|| { + eyre::eyre!("daemon child lease handoff requires an inherited dispatch-slot fd") + })?; + let dispatch_slot_index = context.preferred_dispatch_slot_index.ok_or_else(|| { + eyre::eyre!("daemon child lease handoff requires an inherited dispatch-slot index") + })?; + + context.state_store.adopt_preacquired_lease( + context.project.service_id(), + issue_id, + preferred_run_identity.run_id, + preferred_issue_state, + PreacquiredLeaseGuards { issue_claim_fd, dispatch_slot_fd, dispatch_slot_index }, + )?; + + Ok(()) +} + +fn prepare_issue_run( + context: PrepareIssueRunContext<'_, T>, + issue: TrackerIssue, +) -> Result> +where + T: IssueTracker, +{ + let planned_worktree = context.worktree_manager.plan_for_issue(&issue.identifier); + let Some((attempt_number, run_id)) = + resolve_prepare_run_identity(context.state_store, &issue, context.preferred_run_identity)? + else { + return Ok(None); + }; + let retry_budget_base = + context.preferred_retry_budget_base.unwrap_or(0).max(retry_budget_base_for_issue_worktree( + context.state_store, + &issue.id, + &planned_worktree.path, + )?); + let lease_issue_id = issue.id.clone(); + let issue_state = planned_issue_state_for_dispatch( + context.workflow, + &issue, + context.dispatch_mode, + context.preferred_issue_state, + ); + + if !context.dry_run + && !context.lease_preacquired + && !context.state_store.try_acquire_lease( + context.project.service_id(), + &issue.id, + &run_id, + &issue_state, + )? { + return Ok(None); + } + + match (|| -> Result> { + let worktree = + context.worktree_manager.ensure_worktree_with_hooks( + &issue.identifier, + context.dry_run, + context.workflow.frontmatter().execution().workspace_hooks(), + )?; + + if !context.dry_run { + context.state_store.upsert_worktree( + context.project.service_id(), + &lease_issue_id, + &worktree.branch_name, + &worktree.path.display().to_string(), + )?; + } + + let Some(refreshed_issue) = refresh_issue(context.tracker, &lease_issue_id)? else { + return Ok(None); + }; + + if !prepare_issue_run_dispatch_allowed( + &context, + &refreshed_issue, + &lease_issue_id, + &worktree.branch_name, + &worktree.path, + )? { + return Ok(None); + } + if !context.dry_run { + record_starting_attempt(context.state_store, &run_id, &issue.id, attempt_number)?; + clear_terminal_guard_marker(&worktree.path)?; + } + + let initial_issue_state = context + .preferred_initial_issue_state + .map_or_else(|| refreshed_issue.state.name.clone(), str::to_owned); + let issue_run = IssueRunPlan { + issue: refreshed_issue, + issue_state: issue_state.clone(), + initial_issue_state, + worktree, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode: context.dispatch_mode, + attempt_number, + run_id: run_id.clone(), + retry_budget_base, + }; + + if !context.dry_run { + write_prepare_lifecycle_events( + context.tracker, + context.project, + context.state_store, + &issue_run, + )?; + } + + Ok(Some(issue_run)) + })() { + Ok(Some(issue_run)) => Ok(Some(issue_run)), + Ok(None) => { + clear_prepare_issue_run_lease( + context.state_store, + context.dry_run, + &lease_issue_id, + )?; + + Ok(None) + }, + Err(error) => { + clear_prepare_issue_run_lease( + context.state_store, + context.dry_run, + &lease_issue_id, + )?; + + Err(error) + }, + } +} + +fn prepare_issue_run_dispatch_allowed( + context: &PrepareIssueRunContext<'_, T>, + refreshed_issue: &TrackerIssue, + lease_issue_id: &str, + worktree_branch_name: &str, + worktree_path: &Path, +) -> Result +where + T: IssueTracker, +{ + let dispatch_allowed = context.dispatch_mode.allows_issue( + context.tracker, + refreshed_issue, + context.project, + context.workflow, + context.state_store, + RetryIssueStateHint { + preferred_issue_state: context.preferred_issue_state, + preferred_initial_issue_state: context.preferred_initial_issue_state, + }, + )?; + + if !dispatch_allowed { + if !context.dry_run + && context.dispatch_mode == IssueDispatchMode::Closeout + && let Some(reason) = closeout_dispatch_block_reason( + context.tracker, + refreshed_issue, + context.project, + context.workflow, + context.state_store, + )? + { + eyre::bail!("retained closeout dispatch blocked: {reason}"); + } + if !context.dry_run && is_terminal_issue(refreshed_issue, context.workflow) { + cleanup_terminal_worktree( + context.state_store, + context.worktree_manager, + context.workflow, + lease_issue_id, + &refreshed_issue.identifier, + worktree_branch_name, + worktree_path, + )?; + } + } + + Ok(dispatch_allowed) +} + +fn clear_prepare_issue_run_lease( + state_store: &StateStore, + dry_run: bool, + issue_id: &str, +) -> Result<()> { + if !dry_run { + state_store.clear_lease(issue_id)?; + } + + Ok(()) +} + +fn record_starting_attempt( + state_store: &StateStore, + run_id: &str, + issue_id: &str, + attempt_number: i64, +) -> Result<()> { + state_store.record_run_attempt(run_id, issue_id, attempt_number, "starting") +} + +fn resolve_prepare_run_identity( + state_store: &StateStore, + issue: &TrackerIssue, + preferred_run_identity: Option>, +) -> Result> { + let next_attempt_number = state_store.next_attempt_number(&issue.id)?; + + match preferred_run_identity { + Some(preferred_run_identity) => { + if next_attempt_number > preferred_run_identity.attempt_number { + let Some(existing_attempt) = + state_store.run_attempt(preferred_run_identity.run_id)? + else { + return Ok(None); + }; + + if existing_attempt.issue_id() != issue.id + || existing_attempt.attempt_number() != preferred_run_identity.attempt_number + { + return Ok(None); + } + } + + Ok(Some(( + preferred_run_identity.attempt_number, + preferred_run_identity.run_id.to_owned(), + ))) + }, + None => + Ok(Some((next_attempt_number, build_run_id(&issue.identifier, next_attempt_number)?))), + } +} + +fn complete_issue_run( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: IssueRunPlan, + dry_run: bool, +) -> Result> +where + T: IssueTracker, +{ + if dry_run { + return Ok(Some(run_summary_from_issue_run(project.service_id(), &issue_run))); + } + + let summary = execute_issue_run(tracker, project, workflow, state_store, issue_run)?; + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + + if let Some(retained_summary) = drain_internal_review_only_retained_tail_with_inspector( + tracker, + project, + workflow, + state_store, + &summary, + &review_state_inspector, + |source_summary| run_retained_closeout_for_handoff_summary( + tracker, + project, + workflow, + state_store, + source_summary, + ), + )? { + return Ok(Some(retained_summary)); + } + + Ok(Some(summary)) +} + +fn run_retained_closeout_for_handoff_summary( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + source_summary: &RunSummary, +) -> Result> +where + T: IssueTracker, +{ + run_target_issue_once(TargetIssueRunContext { + tracker, + project, + workflow, + state_store, + issue_id: source_summary.issue_id.as_str(), + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: false, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) +} + +fn drain_internal_review_only_retained_tail_with_inspector( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + summary: &RunSummary, + review_state_inspector: &I, + mut run_closeout: F, +) -> Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, + F: FnMut(&RunSummary) -> Result>, +{ + if project.codex().external_review_enabled() + || summary.continuation_pending + || !matches!( + summary.dispatch_mode, + IssueDispatchMode::Normal | IssueDispatchMode::ReviewRepair + ) + { + return Ok(None); + } + + let completed_state = workflow.frontmatter().tracker().resolved_completed_state(); + + for pass in 0..INTERNAL_RETAINED_DRAIN_MAX_PASSES { + reconcile_post_review_orchestration_with_inspector( + tracker, + project, + workflow, + state_store, + review_state_inspector, + )?; + + let Some(lane) = build_post_review_lane_statuses( + tracker, + project, + workflow, + state_store, + review_state_inspector, + )? + .into_iter() + .find(|lane| lane.issue_id == summary.issue_id) + else { + return Ok(None); + }; + + if post_review_lane_is_closeout_candidate(&lane, completed_state) { + if let Some(retained_summary) = run_closeout(summary)? { + return Ok(Some(retained_summary)); + } + + return Ok(None); + } + if lane.reason != "internal_review_only_waiting_for_merge" + || pass + 1 == INTERNAL_RETAINED_DRAIN_MAX_PASSES + { + return Ok(None); + } + } + + Ok(None) +} + +fn reconcile_project_state( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + worktree_manager: &WorktreeManager, +) -> Result<()> +where + T: IssueTracker, +{ + let leases = state_store.list_leases(project.service_id())?; + let worktrees = state_store.list_worktrees(project.service_id())?; + + if leases.is_empty() && worktrees.is_empty() { + return Ok(()); + } + + let mut issue_ids = HashSet::new(); + + for lease in &leases { + issue_ids.insert(lease.issue_id().to_owned()); + } + for mapping in &worktrees { + issue_ids.insert(mapping.issue_id().to_owned()); + } + + let refreshed_issues = tracker.refresh_issues(&issue_ids.into_iter().collect::>())?; + let issues_by_id = refreshed_issues + .into_iter() + .map(|issue| (issue.id.clone(), issue)) + .collect::>(); + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut cleared_terminal_lane_issue_ids = HashSet::new(); + + for lease in &leases { + if let Some(issue) = issues_by_id.get(lease.issue_id()) + && issue.state.name == workflow.frontmatter().tracker().success_state() + && retained_review_lease_matches_run(state_store, lease)? + { + mark_run_attempt_if_active(state_store, lease.run_id(), "succeeded")?; + + state_store.clear_lease(lease.issue_id())?; + + continue; + } + if let Some(issue) = issues_by_id.get(lease.issue_id()) + && terminal_issue_keeps_retained_closeout( + tracker, + issue, + project, + workflow, + state_store, + )? + { + if retained_closeout_lease_has_fresh_activity( + lease, + issue, + project, + now_unix_epoch, + )? { + continue; + } + + clear_terminal_lane_labels_once( + tracker, + project, + issue, + &mut cleared_terminal_lane_issue_ids, + )?; + mark_run_attempt_if_active(state_store, lease.run_id(), "interrupted")?; + + state_store.clear_lease(lease.issue_id())?; + + continue; + } + + let reconciled_status = match issues_by_id.get(lease.issue_id()) { + Some(issue) if is_terminal_issue(issue, workflow) => "terminated", + Some(_) | None => "interrupted", + }; + + if let Some(issue) = issues_by_id.get(lease.issue_id()) + && is_terminal_issue(issue, workflow) + { + clear_terminal_lane_labels_once( + tracker, + project, + issue, + &mut cleared_terminal_lane_issue_ids, + )?; + } + + mark_run_attempt_if_active(state_store, lease.run_id(), reconciled_status)?; + + state_store.clear_lease(lease.issue_id())?; + } + for mapping in &worktrees { + if let Some(issue) = issues_by_id.get(mapping.issue_id()) + && is_terminal_issue(issue, workflow) + && !terminal_issue_keeps_retained_closeout( + tracker, + issue, + project, + workflow, + state_store, + )? + { + clear_terminal_lane_labels_once( + tracker, + project, + issue, + &mut cleared_terminal_lane_issue_ids, + )?; + cleanup_worktree_mapping( + state_store, + worktree_manager, + workflow, + &issue.identifier, + mapping, + )?; + } + } + + Ok(()) +} + +fn clear_terminal_lane_labels_once( + tracker: &T, + project: &ServiceConfig, + issue: &TrackerIssue, + cleared_issue_ids: &mut HashSet, +) -> Result<()> +where + T: IssueTracker, +{ + if cleared_issue_ids.insert(issue.id.clone()) { + tracker::clear_automation_lane_labels(tracker, issue, project.service_id())?; + } + + Ok(()) +} + +fn retained_review_lease_matches_run( + state_store: &StateStore, + lease: &IssueLease, +) -> Result { + let Some(run_attempt) = state_store.run_attempt(lease.run_id())? else { + return Ok(false); + }; + let worktree_mapping = state_store.worktree_for_issue(lease.issue_id())?; + + retained_review_handoff_matches_run(state_store, &run_attempt, worktree_mapping.as_ref()) +} + +fn terminal_issue_keeps_retained_closeout( + tracker: &T, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> Result +where + T: IssueTracker + ?Sized, +{ + if !is_terminal_issue(issue, workflow) { + return Ok(false); + } + + Ok( + issue_passes_closeout_dispatch_policy(tracker, issue, project, workflow, state_store)? + || closeout_dispatch_block_reason(tracker, issue, project, workflow, state_store)? + .is_some(), + ) +} + +fn retained_closeout_lease_has_fresh_activity( + lease: &IssueLease, + issue: &TrackerIssue, + project: &ServiceConfig, + now_unix_epoch: i64, +) -> Result { + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + let worktree = worktree_manager.plan_for_issue(&issue.identifier); + let Some(marker) = state::read_run_activity_marker_snapshot(&worktree.path)? else { + return Ok(false); + }; + + Ok(marker.run_id() == lease.run_id() && worktree_activity_marker_is_fresh(&marker, now_unix_epoch)) +} diff --git a/apps/decodex/src/orchestrator/runtime_validation.rs b/apps/decodex/src/orchestrator/runtime_validation.rs new file mode 100644 index 00000000..22528664 --- /dev/null +++ b/apps/decodex/src/orchestrator/runtime_validation.rs @@ -0,0 +1,69 @@ +fn validate_review_handoff_runtime( + project: &ServiceConfig, + dry_run: bool, +) -> Result<()> { + if dry_run { + return Ok(()); + } + + validate_command_available("gh", "PR-backed review handoff")?; + resolve_configured_env_var("github.token_env_var", Some(project.github().token_env_var()))?; + + Ok(()) +} + +fn validate_review_repair_runtime( + project: &ServiceConfig, + dry_run: bool, +) -> Result<()> { + if dry_run { + return Ok(()); + } + + validate_command_available("gh", "retained review-repair re-entry")?; + resolve_configured_env_var("github.token_env_var", Some(project.github().token_env_var()))?; + + Ok(()) +} + +fn validate_closeout_runtime( + project: &ServiceConfig, + dry_run: bool, +) -> Result<()> { + if dry_run { + return Ok(()); + } + + validate_command_available("gh", "retained closeout re-entry")?; + resolve_configured_env_var("github.token_env_var", Some(project.github().token_env_var()))?; + + Ok(()) +} + +fn validate_daemon_runtime() -> Result<()> { + Ok(()) +} + +fn validate_command_available(command: &str, purpose: &str) -> Result<()> { + let output = Command::new(command).arg("--version").output().map_err(|error| { + eyre::eyre!("Required command `{command}` is unavailable for {purpose}: {error}") + })?; + + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let detail = if stderr.trim().is_empty() { stdout.trim() } else { stderr.trim() }; + + if detail.is_empty() { + eyre::bail!( + "Required command `{command}` is unavailable for {purpose}: `{command} --version` exited unsuccessfully." + ); + } + + eyre::bail!( + "Required command `{command}` is unavailable for {purpose}: `{command} --version` failed with `{detail}`." + ); +} diff --git a/apps/decodex/src/orchestrator/selection.rs b/apps/decodex/src/orchestrator/selection.rs new file mode 100644 index 00000000..67a888db --- /dev/null +++ b/apps/decodex/src/orchestrator/selection.rs @@ -0,0 +1,345 @@ +struct RetryComment<'a> { + run_id: &'a str, + attempt_number: i64, + retry_budget_attempt_number: i64, + max_attempts: i64, + worktree_path: String, + branch_name: &'a str, + error_class: &'a str, + next_action: &'a str, +} + +#[cfg_attr(not(test), allow(dead_code))] +fn select_issue_candidate( + tracker: &dyn IssueTracker, + issues: Vec, + workflow: &WorkflowDocument, + state_store: &StateStore, + project_id: &str, +) -> Result> { + select_issue_candidate_with_exclusions(tracker, issues, workflow, state_store, project_id, &[]) +} + +fn select_issue_candidate_with_exclusions( + tracker: &dyn IssueTracker, + issues: Vec, + workflow: &WorkflowDocument, + state_store: &StateStore, + project_id: &str, + excluded_issue_ids: &[&str], +) -> Result> { + let concurrency = ConcurrencySnapshot::new(project_id, state_store)?; + + if !concurrency.has_global_capacity(workflow.frontmatter().execution()) { + return Ok(None); + } + + let mut eligible_issues = Vec::new(); + + for issue in issues { + if excluded_issue_ids.contains(&issue.id.as_str()) { + continue; + } + if state_store.issue_has_active_shared_claim(project_id, &issue.id)? { + continue; + } + if is_issue_eligible(tracker, &issue, project_id, workflow, state_store)? { + eligible_issues.push(issue); + } + } + + eligible_issues.sort_by(compare_issue_candidates); + + Ok(eligible_issues.into_iter().next()) +} + +fn compare_issue_candidates(left: &TrackerIssue, right: &TrackerIssue) -> Ordering { + let left_priority = (left.priority.is_none(), left.priority.unwrap_or(i64::MAX)); + let right_priority = (right.priority.is_none(), right.priority.unwrap_or(i64::MAX)); + + left_priority + .cmp(&right_priority) + .then_with(|| left.created_at.cmp(&right.created_at)) + .then_with(|| left.identifier.cmp(&right.identifier)) +} + +fn format_no_eligible_issue_message( + project: &ServiceConfig, + workflow: &WorkflowDocument, +) -> String { + let tracker_policy = workflow.frontmatter().tracker(); + + format!( + "No eligible issue found for the configured project.\n{}", + format_no_eligible_issue_hint( + project.service_id(), + tracker_policy.opt_out_label(), + tracker_policy.needs_attention_label(), + ) + ) +} + +fn format_status_no_eligible_issue_hint(service_id: &str) -> String { + format!( + "Hint: check `Todo`, label {}, no opt-out/manual-only or needs-attention labels, non-terminal state, no open dependency blockers, and available capacity.", + format_no_eligible_queue_label_hint(service_id), + ) +} + +fn format_no_eligible_issue_hint( + service_id: &str, + opt_out_label: &str, + needs_attention_label: &str, +) -> String { + format!( + "Hint: check `Todo`, label {}, no `{opt_out_label}`/`{needs_attention_label}`, non-terminal state, no open dependency blockers, and available capacity.", + format_no_eligible_queue_label_hint(service_id), + ) +} + +fn format_no_eligible_queue_label_hint(service_id: &str) -> String { + let queue_label = tracker::automation_queue_label(service_id); + + if service_id == "all" { + String::from("`decodex:queued:`") + } else { + format!("`decodex:queued:` (this project: `{queue_label}`)") + } +} + +fn format_retry_comment(comment: RetryComment<'_>) -> String { + let RetryComment { + run_id, + attempt_number, + retry_budget_attempt_number, + max_attempts, + worktree_path, + branch_name, + error_class, + next_action, + } = comment; + + format!( + "decodex run failed and will retry\n\n- run_id: `{run_id}`\n- attempt: `{attempt_number}`\n- retry_budget_attempt: `{retry_budget_attempt_number}` / `{max_attempts}`\n- failed_at: `{failed_at}`\n- branch: `{branch}`\n- worktree_path: `{worktree}`\n- error_class: `{error_class}`\n- next_action: `{next_action}`\n- error_summary: `Sensitive runtime details were withheld from the tracker comment; inspect the local lane for the full failure context.`", + failed_at = current_timestamp(), + branch = branch_name, + worktree = worktree_path, + ) +} + +fn retry_comment_details(error: &Report) -> (&'static str, String) { + if let Some(repo_gate_failure) = error.downcast_ref::() { + match repo_gate_failure.disposition() { + RepoGateFailureDisposition::ContinueRepair + | RepoGateFailureDisposition::RetryAfterBackoff => { + return ( + repo_gate_failure.error_class(), + repo_gate_failure.retry_next_action().to_owned(), + ); + }, + RepoGateFailureDisposition::NeedsHumanAttention => {}, + } + } + + ("retryable_execution_failure", String::from("decodex will retry automatically")) +} + +fn format_terminal_failure_comment( + run_id: &str, + attempt_number: i64, + worktree_path: String, + branch_name: &str, + error_class: &str, + next_action: &str, +) -> String { + format!( + "decodex run failed and needs attention\n\n- run_id: `{run_id}`\n- attempt: `{attempt_number}`\n- failed_at: `{failed_at}`\n- branch: `{branch}`\n- worktree_path: `{worktree}`\n- error_class: `{error_class}`\n- next_action: `{next_action}`\n- error_summary: `Sensitive runtime details were withheld from the tracker comment; inspect the local lane for the full failure context.`", + failed_at = current_timestamp(), + branch = branch_name, + worktree = worktree_path + ) +} + +fn terminal_failure_comment_details( + manual_attention_requested: bool, + error: &Report, + recovery_gate: &str, +) -> (&'static str, String) { + if let Some(retained_review_needs_attention) = + error.downcast_ref::() + { + let error_class = + retained_review_needs_attention_error_class(&retained_review_needs_attention.reason); + + ( + error_class, + format!( + "inspect retained review orchestration reason `{}`, resolve the blocker manually, {recovery_gate}", + retained_review_needs_attention.reason + ), + ) + } else if manual_attention_requested { + ( + "human_attention_required", + format!( + "inspect the issue comment and worktree, resolve the blocker manually, {recovery_gate}" + ), + ) + } else if error.downcast_ref::().is_some() { + ( + "review_handoff_writeback_failed", + format!( + "inspect the tracker state, PR, and worktree, repair the incomplete review handoff manually, {recovery_gate}" + ), + ) + } else if let Some(partial_progress) = error.downcast_ref::() { + ( + "partial_progress_retained", + format!( + "inspect retained worktree `{}`, finish validation and PR handoff or reset the patch manually, {recovery_gate}", + partial_progress.worktree_path + ), + ) + } else if error.downcast_ref::().is_some() { + ( + "stalled_run_detected", + format!( + "inspect the worktree and app-server activity for the stalled lane, resolve the blocker manually, {recovery_gate}" + ), + ) + } else if let Some(credentials_failure) = + error.downcast_ref::() + { + ( + "github_credentials_unavailable", + format!( + "configure `{}` for the routed GitHub identity, verify noninteractive Git credentials, {recovery_gate}", + credentials_failure.token_env_var + ), + ) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(app_server_failure) = error.downcast_ref::() { + (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(review_policy_stop) = error.downcast_ref::() { + ( + review_policy_stop.reason.error_class(), + review_policy_stop_terminal_next_action(review_policy_stop.reason, recovery_gate), + ) + } else if let Some(repo_gate_failure) = error.downcast_ref::() { + (repo_gate_failure.error_class(), repo_gate_failure.terminal_next_action(recovery_gate)) + } else { + ( + "retry_budget_exhausted", + format!("inspect the worktree, resolve the issue manually, {recovery_gate}"), + ) + } +} + +fn review_policy_stop_terminal_next_action( + reason: ReviewPolicyStopReason, + recovery_gate: &str, +) -> String { + match reason { + ReviewPolicyStopReason::Exhausted => format!( + "inspect the repeated review findings and current worktree, decide the next repair or redesign manually, prepare a bounded convergence research follow-up only after the current head, review phase, non-clean round count, and validated findings are structured and machine-checkable, {recovery_gate}" + ), + ReviewPolicyStopReason::ArchitectureReviewRequired => format!( + "inspect the current findings and worktree, perform the required architecture review manually, prepare a bounded architecture research follow-up only after the current head, review phase, stop class, and architecture concern are structured and machine-checkable, {recovery_gate}" + ), + ReviewPolicyStopReason::Blocked => format!( + "inspect the blocking condition and worktree, resolve the blocker manually, do not dispatch research unless the blocker is reclassified as a structured architecture or convergence stop, {recovery_gate}" + ), + } +} + +fn retained_review_needs_attention_error_class(reason: &str) -> &'static str { + match reason { + "external_review_admin_merge_failed" => "external_review_admin_merge_failed", + "external_review_admin_merge_unavailable" => "external_review_admin_merge_unavailable", + "external_review_merge_visibility_timeout" => "external_review_merge_visibility_timeout", + "external_review_pass_signal_missing" => "external_review_pass_signal_missing", + "external_review_request_ci_red_manual_attention" => + "external_review_request_ci_red_manual_attention", + "internal_review_only_admin_merge_failed" => "internal_review_only_admin_merge_failed", + "internal_review_only_admin_merge_unavailable" => + "internal_review_only_admin_merge_unavailable", + "internal_review_only_merge_visibility_timeout" => + "internal_review_only_merge_visibility_timeout", + "pull_request_is_draft" => "pull_request_is_draft", + "pull_request_merge_commit_lineage_check_failed" => + "pull_request_merge_commit_lineage_check_failed", + "pull_request_not_open" => "pull_request_not_open", + "retained_admin_merge_subject_unavailable" => "retained_admin_merge_subject_unavailable", + "review_orchestration_branch_mismatch" => "review_orchestration_branch_mismatch", + "review_orchestration_head_mismatch" => "review_orchestration_head_mismatch", + "review_orchestration_pr_mismatch" => "review_orchestration_pr_mismatch", + "worktree_head_missing" => "worktree_head_missing", + _ => "retained_review_needs_attention", + } +} + +fn terminal_failure_recovery_gate( + needs_attention_label: &str, + needs_attention_label_available: bool, + guarded_by_nonstartable_state: bool, + nonstartable_guard_state: &str, +) -> String { + if needs_attention_label_available { + return format!( + "clear label `{needs_attention_label}`, then move the issue back to a startable state if another automated run is desired" + ); + } + if guarded_by_nonstartable_state { + return format!( + "`{needs_attention_label}` could not be applied because it does not exist on the team; the issue remains in `{nonstartable_guard_state}` to block automatic retries, so move it back to a startable state manually if another automated run is desired" + ); + } + + format!( + "`{needs_attention_label}` could not be applied because it does not exist on the team; move the issue back to a startable state manually if another automated run is desired" + ) +} + +fn current_timestamp() -> String { + OffsetDateTime::now_utc().format(&Rfc3339).expect("timestamp formatting should succeed") +} + +fn build_run_id(issue_identifier: &str, attempt_number: i64) -> Result { + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + Ok(format!("{}-attempt-{attempt_number}-{timestamp}", issue_identifier.to_lowercase())) +} + +fn resolve_config_path( + explicit_path: Option<&Path>, + state_store: &StateStore, +) -> Result> { + if let Some(path) = explicit_path { + return Ok(Some(path.to_path_buf())); + } + + runtime::registered_config_path_for_cwd(state_store, &env::current_dir()?) +} + +fn sleep_until_next_tick(poll_interval: Duration, tick_started_at: Instant) { + let elapsed = tick_started_at.elapsed(); + + if elapsed < poll_interval { + thread::sleep(poll_interval - elapsed); + } +} diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs new file mode 100644 index 00000000..dad9d074 --- /dev/null +++ b/apps/decodex/src/orchestrator/status.rs @@ -0,0 +1,4688 @@ +use records::LinearExecutionEventRecord; + +use crate::pull_request::{self, PullRequestLandingGateView}; +use crate::worktree; +use crate::worktree::MergedWorktreeCleanupDebt; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RetainedCloseoutPrMergeGate { + Merged, + NotMerged, + PullRequestStateReadFailed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExternalReviewRequestCiGate { + Ready, + WaitForGreenChecks, + RepairRequired, + ManualAttention(&'static str), +} + +struct PostReviewOrchestrationStatus { + phase: ReviewOrchestrationPhase, + request_acknowledged: bool, + review_result_arrived: bool, + strict_pass: bool, + clean_path_landing_gates_satisfied: bool, + landing_requires_agent_fallback: bool, +} +impl PostReviewOrchestrationStatus { + fn from_review_state( + review_state: &PullRequestReviewState, + orchestration_marker: &ReviewOrchestrationMarker, + ) -> crate::prelude::Result { + let phase = + ReviewOrchestrationPhase::parse(orchestration_marker.phase()).map_err(|error| { + eyre::eyre!("Failed to parse retained review orchestration phase: {error}") + })?; + + Ok(Self { + phase, + request_acknowledged: request_comment_has_eyes(review_state, orchestration_marker) + .unwrap_or(false), + review_result_arrived: external_review_result_arrived( + review_state, + orchestration_marker, + ), + strict_pass: external_review_has_strict_pass_signals( + review_state, + orchestration_marker, + ), + clean_path_landing_gates_satisfied: + review_state_clean_path_landing_gates_satisfied(review_state), + landing_requires_agent_fallback: review_state_landing_requires_agent_fallback( + review_state, + ), + }) + } +} + +struct OperatorRunTiming { + process_id: Option, + process_alive: Option, + last_run_activity_unix_epoch: Option, + last_protocol_activity_unix_epoch: Option, + last_progress_unix_epoch: Option, + idle_for_seconds: Option, + protocol_idle_for_seconds: Option, +} + +struct OperatorRunAppServerState { + thread_id: Option, + turn_id: Option, + thread_status: Option, + thread_active_flags: Vec, + interactive_requested: bool, + continuation_pending: bool, + effective_model: Option, + effective_model_provider: Option, + effective_cwd: Option, + effective_approval_policy: Option, + effective_approvals_reviewer: Option, + effective_sandbox_mode: Option, +} + +struct OperatorRunProtocolSummary { + last_event_type: Option, + last_event_at: Option, + event_count: i64, +} + +struct PostReviewLaneBuildContext<'a, I> { + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, + review_state_inspector: &'a I, + success_state: &'a str, + completed_state: &'a str, +} + +struct OperatorHistoryLedgerRecord { + record: LinearExecutionEventRecord, + event_unix_epoch: Option, + sort_unix_epoch: Option, + comment_index: usize, +} + +struct OperatorIssueDisplayMetadata { + issue_identifier: String, + title: Option, +} + +struct WorktreeOwnership { + kind: &'static str, + reason: String, +} + +pub(crate) fn ensure_project_has_no_merged_worktree_cleanup_debt( + project: &ServiceConfig, +) -> crate::prelude::Result<()> { + let debts = project_merged_worktree_cleanup_debts(project)?; + + if debts.is_empty() { + return Ok(()); + } + + eyre::bail!( + "Post-land worktree cleanup is pending for project `{}`; remove or salvage merged linked worktrees before continuing automation: {}", + project.service_id(), + format_merged_worktree_cleanup_debts(&debts) + ); +} + +fn build_operator_status_snapshot( + project: &ServiceConfig, + state_store: &StateStore, + limit: usize, +) -> crate::prelude::Result { + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut active_runs = state_store + .list_active_runs(project.service_id())? + .into_iter() + .map(|run| operator_run_status(project, state_store, run, now_unix_epoch)) + .collect::>>()? + .into_iter() + .filter(operator_run_counts_as_active) + .collect::>(); + let recent_run_fetch_limit = limit.saturating_add(active_runs.len()); + let recent_runs = state_store + .list_recent_runs(project.service_id(), recent_run_fetch_limit)? + .into_iter() + .map(|run| operator_run_status(project, state_store, run, now_unix_epoch)) + .collect::>>()?; + let mut active_run_ids = + active_runs.iter().map(|run| run.run_id.clone()).collect::>(); + + for run in &recent_runs { + if !active_run_ids.contains(&run.run_id) && operator_run_has_live_process(run) { + active_run_ids.insert(run.run_id.clone()); + active_runs.push(run.clone()); + } + } + + let history_lanes = operator_history_lanes(&active_runs, &recent_runs); + let (worktrees, mut warnings) = operator_status_worktrees(project, state_store)?; + let accounts = codex_account_activity_summaries(project, &mut warnings); + let mut snapshot = OperatorStatusSnapshot { + project_id: project.service_id().to_owned(), + run_limit: limit, + warnings, + connector_backoffs: Vec::new(), + projects: vec![OperatorProjectStatus { + project_id: project.service_id().to_owned(), + config_path: String::new(), + repo_root: project.repo_root().display().to_string(), + enabled: true, + active_run_count: active_runs.len(), + queued_candidate_count: 0, + post_review_lane_count: 0, + retained_worktree_count: 0, + waiting_lane_count: 0, + attention_count: 0, + connector_state: String::from("ok"), + last_activity_at: None, + warning_count: 0, + }], + accounts, + active_runs, + recent_runs, + history_lanes, + queued_candidates: Vec::new(), + worktrees, + post_review_lanes: Vec::new(), + }; + + refresh_worktree_ownership(&mut snapshot, None); + refresh_operator_project_summary(&mut snapshot); + + Ok(snapshot) +} + +fn build_live_operator_status_snapshot( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + limit: usize, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + build_live_operator_status_snapshot_with_history_ledger( + tracker, + project, + workflow, + state_store, + limit, + true, + ) +} + +fn build_control_plane_operator_status_snapshot( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + limit: usize, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + build_live_operator_status_snapshot_with_history_ledger( + tracker, + project, + workflow, + state_store, + limit, + false, + ) +} + +fn build_live_operator_status_snapshot_with_history_ledger( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + limit: usize, + hydrate_history_ledger: bool, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + state_store.configure_dispatch_slot_root( + project.service_id(), + project.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + )?; + + let review_state_inspector = GhPullRequestReviewStateInspector { + github_token_env_var: Some(project.github().token_env_var().to_owned()), + }; + let mut snapshot = build_operator_status_snapshot(project, state_store, limit)?; + + hydrate_history_lanes_from_local_ledger(project, state_store, &mut snapshot)?; + + if hydrate_operator_run_rows_from_tracker( + tracker, + project, + &mut snapshot, + ) { + add_operator_snapshot_warning(&mut snapshot, "run_issue_metadata_unavailable"); + } + if hydrate_history_ledger && hydrate_history_lanes_from_linear_ledger(tracker, project, &mut snapshot) { + add_operator_snapshot_warning(&mut snapshot, "execution_ledger_status_unavailable"); + } + + match build_queued_candidate_statuses(tracker, project, workflow, state_store) { + Ok(queued_candidates) => snapshot.queued_candidates = queued_candidates, + Err(error) => { + let _ = error; + + tracing::warn!( + "Skipped queued candidate status while publishing an operator snapshot; sensitive runtime details were withheld." + ); + + add_operator_snapshot_warning(&mut snapshot, "queued_candidate_status_unavailable"); + }, + } + match build_post_review_lane_statuses_and_hydrate_worktrees( + tracker, + project, + workflow, + state_store, + &review_state_inspector, + &mut snapshot, + ) { + Ok(post_review_lanes) => snapshot.post_review_lanes = post_review_lanes, + Err(error) => { + let _ = error; + + tracing::warn!( + "Skipped post-review lane status while publishing an operator snapshot; sensitive runtime details were withheld." + ); + + add_operator_snapshot_warning(&mut snapshot, "post_review_lane_status_unavailable"); + }, + } + + refresh_worktree_ownership( + &mut snapshot, + Some(workflow.frontmatter().tracker().resolved_completed_state()), + ); + refresh_operator_project_summary(&mut snapshot); + + Ok(snapshot) +} + +fn hydrate_history_lanes_from_local_ledger( + project: &ServiceConfig, + state_store: &StateStore, + snapshot: &mut OperatorStatusSnapshot, +) -> crate::prelude::Result<()> { + for lane in &mut snapshot.history_lanes { + let records = + state_store.list_linear_execution_events(project.service_id(), &lane.issue_id)?; + + if records.is_empty() { + lane.ledger_outcome = missing_history_ledger_outcome(); + + continue; + } + + let records = local_history_ledger_records(records); + + hydrate_history_lane_from_ledger_records(lane, &records); + + lane.ledger_outcome = operator_history_ledger_outcome(&records); + } + + Ok(()) +} + +fn refresh_worktree_ownership( + snapshot: &mut OperatorStatusSnapshot, + completed_state: Option<&str>, +) { + let ownership = snapshot + .worktrees + .iter() + .map(|worktree| worktree_ownership(worktree, snapshot, completed_state)) + .collect::>(); + + for (worktree, ownership) in snapshot.worktrees.iter_mut().zip(ownership) { + worktree.ownership = ownership.kind.to_owned(); + worktree.ownership_reason = ownership.reason; + } +} + +fn worktree_ownership( + worktree: &OperatorWorktreeStatus, + snapshot: &OperatorStatusSnapshot, + completed_state: Option<&str>, +) -> WorktreeOwnership { + if let Some(run) = worktree_active_run_owner(worktree, snapshot) { + return WorktreeOwnership { + kind: "active_lane", + reason: format!("Active lane `{}` owns this worktree.", run.run_id), + }; + } + if let Some(lane) = worktree_post_review_owner(worktree, snapshot) { + return WorktreeOwnership { + kind: "post_review_lane", + reason: format!( + "Review & Landing owns this worktree as `{}`.", + lane.classification + ), + }; + } + + if worktree_has_queued_attention_owner(worktree, snapshot) { + return WorktreeOwnership { + kind: "queued_attention", + reason: String::from( + "Intake Queue owns this worktree because the issue needs operator attention.", + ), + }; + } + + if let Some(hygiene) = &worktree.hygiene { + return WorktreeOwnership { + kind: "post_land_cleanup", + reason: hygiene.reason.clone(), + }; + } + + WorktreeOwnership { + kind: "cleanup_only", + reason: worktree_cleanup_only_reason(worktree, completed_state), + } +} + +fn worktree_active_run_owner<'a>( + worktree: &OperatorWorktreeStatus, + snapshot: &'a OperatorStatusSnapshot, +) -> Option<&'a OperatorRunStatus> { + snapshot.active_runs.iter().find(|run| { + run.worktree_path.as_deref() == Some(worktree.worktree_path.as_str()) + || run.branch_name.as_deref() == Some(worktree.branch_name.as_str()) + || run.issue_id == worktree.issue_id + }) +} + +fn worktree_post_review_owner<'a>( + worktree: &OperatorWorktreeStatus, + snapshot: &'a OperatorStatusSnapshot, +) -> Option<&'a OperatorPostReviewLaneStatus> { + snapshot.post_review_lanes.iter().find(|lane| { + lane.worktree_path == worktree.worktree_path + || lane.branch_name == worktree.branch_name + || lane.issue_id == worktree.issue_id + || lane.issue_identifier == worktree.issue_id + || worktree.issue_identifier.as_deref() == Some(lane.issue_identifier.as_str()) + }) +} + +fn worktree_has_queued_attention_owner( + worktree: &OperatorWorktreeStatus, + snapshot: &OperatorStatusSnapshot, +) -> bool { + snapshot.queued_candidates.iter().any(|candidate| { + candidate.reason == "issue_needs_attention" + && (candidate + .attention + .as_ref() + .and_then(|attention| attention.worktree_path.as_deref()) + == Some(worktree.worktree_path.as_str()) + || candidate.issue_id == worktree.issue_id + || candidate.issue_identifier == worktree.issue_id + || worktree.issue_identifier.as_deref() == Some(candidate.issue_identifier.as_str())) + }) +} + +fn worktree_cleanup_only_reason( + worktree: &OperatorWorktreeStatus, + completed_state: Option<&str>, +) -> String { + if let (Some(issue_state), Some(completed_state)) = (worktree.issue_state.as_deref(), completed_state) + && issue_state == completed_state + { + return format!( + "Issue is {completed_state}; no active or post-review lane owns this worktree, so it is local cleanup only." + ); + } + + String::from( + "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", + ) +} + +fn refresh_operator_project_summary(snapshot: &mut OperatorStatusSnapshot) { + let active_run_count = + snapshot.active_runs.iter().filter(|run| operator_run_counts_as_running(run)).count(); + let queued_candidate_count = snapshot + .queued_candidates + .iter() + .filter(|candidate| queued_candidate_counts_as_waiting_intake(candidate)) + .count(); + let post_review_lane_count = snapshot.post_review_lanes.len(); + let retained_worktree_count = rendered_recovery_worktrees(snapshot).len(); + let waiting_lane_count = project_waiting_lane_count(snapshot); + let attention_count = project_attention_count(snapshot); + let connector_state = project_connector_state(snapshot); + let last_activity_at = project_last_activity_at(snapshot); + let warning_count = snapshot.warnings.len(); + + if let Some(project_status) = snapshot.projects.first_mut() { + project_status.active_run_count = active_run_count; + project_status.queued_candidate_count = queued_candidate_count; + project_status.post_review_lane_count = post_review_lane_count; + project_status.retained_worktree_count = retained_worktree_count; + project_status.waiting_lane_count = waiting_lane_count; + project_status.attention_count = attention_count; + project_status.connector_state = connector_state; + project_status.last_activity_at = last_activity_at; + project_status.warning_count = warning_count; + } +} + +fn project_waiting_lane_count(snapshot: &OperatorStatusSnapshot) -> usize { + let waiting_run_count = project_summary_runs(snapshot) + .into_iter() + .filter(|run| operator_run_counts_as_waiting(run)) + .map(|run| run.run_id.as_str()) + .collect::>() + .len(); + let queued_waiting = snapshot + .queued_candidates + .iter() + .filter(|candidate| candidate.classification == "waiting") + .count(); + let review_waiting = snapshot + .post_review_lanes + .iter() + .filter(|lane| lane.classification == "wait_for_review") + .count(); + + waiting_run_count + queued_waiting + review_waiting +} + +fn project_summary_runs(snapshot: &OperatorStatusSnapshot) -> Vec<&OperatorRunStatus> { + let mut runs = snapshot.active_runs.iter().collect::>(); + + runs.extend(snapshot.history_lanes.iter().map(|lane| &lane.latest_run)); + + runs +} + +fn operator_run_counts_as_waiting(run: &OperatorRunStatus) -> bool { + run.phase == "retry_backoff" || run.phase == "waiting_continuation" || run.wait_reason.is_some() +} + +fn queued_candidate_counts_as_waiting_intake(candidate: &OperatorQueuedIssueStatus) -> bool { + !matches!(candidate.classification.as_str(), "claimed" | "closed") +} + +fn project_attention_count(snapshot: &OperatorStatusSnapshot) -> usize { + let active_attention = snapshot + .active_runs + .iter() + .filter(|run| operator_run_needs_attention(run)) + .count(); + let queued_attention = snapshot + .queued_candidates + .iter() + .filter(|candidate| candidate.classification == "blocked" || candidate.attention.is_some()) + .count(); + let review_attention = snapshot + .post_review_lanes + .iter() + .filter(|lane| { + matches!( + lane.classification.as_str(), + "blocked" | "needs_review_repair" | "closeout_blocked" | "cleanup_blocked" + ) + }) + .count(); + let hygiene_attention = snapshot + .worktrees + .iter() + .filter(|worktree| worktree.hygiene.is_some()) + .count(); + + active_attention + queued_attention + review_attention + hygiene_attention +} + +fn operator_run_counts_as_active(run: &OperatorRunStatus) -> bool { + (run.active_lease || operator_run_has_live_process(run)) + && !matches!(run.phase.as_str(), "completed" | "failed" | "terminated") +} + +fn operator_run_has_live_process(run: &OperatorRunStatus) -> bool { + matches!(run.status.as_str(), "starting" | "running") && run.process_alive == Some(true) +} + +fn operator_run_counts_as_running(run: &OperatorRunStatus) -> bool { + matches!(run.status.as_str(), "starting" | "running") + && run.phase == "executing" + && run.process_alive != Some(false) + && !operator_run_needs_attention(run) +} + +fn operator_run_needs_attention(run: &OperatorRunStatus) -> bool { + run.suspected_stall + || run.phase == "stalled" + || run.process_alive == Some(false) + && matches!(run.status.as_str(), "starting" | "running") + && run.wait_reason.is_none() + || operator_run_has_stale_execution_without_known_process(run) +} + +fn operator_run_has_stale_execution_without_known_process(run: &OperatorRunStatus) -> bool { + matches!(run.status.as_str(), "starting" | "running") + && run.phase == "executing" + && run.wait_reason.is_none() + && run.process_alive != Some(true) + && [run.idle_for_seconds, run.protocol_idle_for_seconds].iter().any(|idle_for| { + idle_for.is_some_and(|idle_for| { + u64::try_from(idle_for).is_ok_and(|idle_for| idle_for >= ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + }) + }) +} + +fn project_connector_state(snapshot: &OperatorStatusSnapshot) -> String { + if !snapshot.connector_backoffs.is_empty() + || snapshot.warnings.iter().any(|warning| warning == TRACKER_RATE_LIMIT_WARNING) + { + return String::from("backoff"); + } + if !snapshot.warnings.is_empty() { + return String::from("degraded"); + } + if project_summary_runs(snapshot) + .into_iter() + .any(|run| run.phase == "retry_backoff" || run.next_retry_at.is_some()) + { + return String::from("backoff"); + } + + String::from("ok") +} + +fn project_last_activity_at(snapshot: &OperatorStatusSnapshot) -> Option { + snapshot + .active_runs + .iter() + .chain(snapshot.recent_runs.iter()) + .flat_map(|run| { + [ + run.last_progress_at.as_deref(), + run.last_run_activity_at.as_deref(), + run.last_protocol_activity_at.as_deref(), + run.last_event_at.as_deref(), + Some(run.updated_at.as_str()), + ] + }) + .flatten() + .max() + .map(str::to_owned) +} + +fn operator_status_worktrees( + project: &ServiceConfig, + state_store: &StateStore, +) -> crate::prelude::Result<(Vec, Vec)> { + let mut worktrees = state_store + .list_worktrees(project.service_id())? + .into_iter() + .map(|mapping| OperatorWorktreeStatus { + issue_id: mapping.issue_id().to_owned(), + issue_identifier: issue_identifier_in_text(mapping.branch_name()) + .or_else(|| issue_identifier_in_text(&mapping.worktree_path().display().to_string())), + issue_state: None, + branch_name: mapping.branch_name().to_owned(), + worktree_path: relative_worktree_path_for_path(project, mapping.worktree_path()), + ownership: String::from("cleanup_only"), + ownership_reason: String::from( + "No active lane, queued recovery, or post-review lane currently owns this worktree.", + ), + hygiene: None, + }) + .collect::>(); + let mut seen_paths = + worktrees.iter().map(|worktree| worktree.worktree_path.clone()).collect::>(); + let mut warnings = Vec::new(); + + for issue_identifier in recoverable_worktree_identifiers(project.worktree_root())? { + let worktree_path = project.worktree_root().join(&issue_identifier); + let relative_path = relative_worktree_path_for_path(project, &worktree_path); + + if !seen_paths.insert(relative_path.clone()) { + continue; + } + + let branch_name = worktree_checkout_branch_name(&worktree_path) + .ok() + .flatten() + .unwrap_or_else(|| issue_identifier.clone()); + + worktrees.push(OperatorWorktreeStatus { + issue_identifier: Some(issue_identifier.clone()), + issue_id: issue_identifier, + issue_state: None, + branch_name, + worktree_path: relative_path, + ownership: String::from("cleanup_only"), + ownership_reason: String::from( + "No active lane, queued recovery, or post-review lane currently owns this worktree.", + ), + hygiene: None, + }); + } + + append_merged_worktree_cleanup_debts(project, &mut worktrees, &mut seen_paths, &mut warnings); + + worktrees.sort_by(|left, right| { + left.issue_id + .cmp(&right.issue_id) + .then_with(|| left.branch_name.cmp(&right.branch_name)) + .then_with(|| left.worktree_path.cmp(&right.worktree_path)) + }); + + Ok((worktrees, warnings)) +} + +fn append_merged_worktree_cleanup_debts( + project: &ServiceConfig, + worktrees: &mut Vec, + seen_paths: &mut HashSet, + warnings: &mut Vec, +) { + let debts = match project_merged_worktree_cleanup_debts(project) { + Ok(debts) => debts, + Err(error) => { + tracing::warn!( + project_id = project.service_id(), + error = %error, + "Skipped merged worktree cleanup debt scan while publishing an operator snapshot." + ); + + warnings.push(String::from("worktree_hygiene_unavailable")); + + return; + }, + }; + + if debts.is_empty() { + return; + } + + warnings.push(String::from("merged_worktree_cleanup_pending")); + + if debts.iter().any(|debt| debt.cleanliness.is_dirty()) { + warnings.push(String::from("merged_dirty_worktree")); + } + + for debt in debts { + let relative_path = relative_worktree_path_for_path(project, &debt.path); + let debt_status = operator_worktree_status_from_cleanup_debt(debt, relative_path.clone()); + + if !seen_paths.insert(relative_path.clone()) { + if let Some(existing) = + worktrees.iter_mut().find(|worktree| worktree.worktree_path == relative_path) + { + existing.hygiene = debt_status.hygiene; + } + + continue; + } + + worktrees.push(debt_status); + } +} + +fn operator_worktree_status_from_cleanup_debt( + debt: MergedWorktreeCleanupDebt, + relative_path: String, +) -> OperatorWorktreeStatus { + let dirty = debt.cleanliness.is_dirty(); + let classification = if dirty { + "merged_dirty_worktree" + } else { + "merged_worktree_cleanup_pending" + }; + let default_branch = debt.default_branch.clone(); + let reason = format!( + "Branch `{}` is already merged into `{}` but linked worktree `{}` still exists{}; remove or salvage it before continuing automation.", + debt.branch_name, + default_branch, + relative_path, + if dirty { " with local changes" } else { "" }, + ); + let branch_name = debt.branch_name; + + OperatorWorktreeStatus { + issue_id: branch_name.clone(), + issue_identifier: issue_identifier_in_text(&branch_name) + .or_else(|| issue_identifier_in_text(&relative_path)), + issue_state: None, + branch_name, + worktree_path: relative_path, + ownership: String::from("post_land_cleanup"), + ownership_reason: reason.clone(), + hygiene: Some(OperatorWorktreeHygieneStatus { + classification: String::from(classification), + default_branch, + dirty, + reason, + }), + } +} + +fn add_operator_snapshot_warning(snapshot: &mut OperatorStatusSnapshot, warning: &str) { + if !snapshot.warnings.iter().any(|existing| existing == warning) { + snapshot.warnings.push(warning.to_owned()); + } +} + +fn project_merged_worktree_cleanup_debts( + project: &ServiceConfig, +) -> crate::prelude::Result> { + let Some(default_branch) = worktree::infer_default_branch_name(project.repo_root())? else { + return Ok(Vec::new()); + }; + + worktree::merged_worktree_cleanup_debts( + project.repo_root(), + project.worktree_root(), + &default_branch, + ) +} + +fn format_merged_worktree_cleanup_debts( + debts: &[MergedWorktreeCleanupDebt], +) -> String { + debts + .iter() + .map(|debt| { + format!( + "{} on {} ({})", + debt.path.display(), + debt.branch_name, + if debt.cleanliness.is_dirty() { "dirty" } else { "clean" } + ) + }) + .collect::>() + .join(", ") +} + +fn codex_account_activity_summaries( + project: &ServiceConfig, + warnings: &mut Vec, +) -> Vec { + let Some(accounts_config) = project.codex().accounts() else { + return Vec::new(); + }; + + match CodexAccountPool::from_config(accounts_config) + .and_then(|pool| pool.account_activity_summaries()) + { + Ok(accounts) => accounts, + Err(error) => { + tracing::warn!( + project_id = project.service_id(), + error = %error, + "Codex accounts snapshot could not be loaded." + ); + + warnings.push(String::from("codex_accounts_unavailable")); + + Vec::new() + }, + } +} + +fn hydrate_operator_run_rows_from_tracker( + tracker: &T, + project: &ServiceConfig, + snapshot: &mut OperatorStatusSnapshot, +) -> bool +where + T: IssueTracker, +{ + let issue_ids = operator_snapshot_run_issue_ids(snapshot); + + if issue_ids.is_empty() { + return false; + } + + match tracker.refresh_issues(&issue_ids) { + Ok(issues) => { + let metadata_by_issue_id = issues + .into_iter() + .map(|issue| { + ( + issue.id, + OperatorIssueDisplayMetadata { + issue_identifier: issue.identifier, + title: Some(issue.title), + }, + ) + }) + .collect::>(); + + hydrate_operator_snapshot_run_rows(snapshot, &metadata_by_issue_id); + + false + }, + Err(error) => { + let _ = error; + + tracing::warn!( + project_id = project.service_id(), + "Skipped tracker issue metadata hydration for operator run rows; sensitive tracker details were withheld." + ); + + true + }, + } +} + +fn operator_snapshot_run_issue_ids(snapshot: &OperatorStatusSnapshot) -> Vec { + let mut issue_ids = BTreeSet::new(); + + for run in snapshot.active_runs.iter().chain(snapshot.recent_runs.iter()) { + append_operator_run_issue_id(&mut issue_ids, run); + } + for lane in &snapshot.history_lanes { + append_operator_run_issue_id(&mut issue_ids, &lane.latest_run); + + for attempt in &lane.attempts { + append_operator_run_issue_id(&mut issue_ids, attempt); + } + } + + issue_ids.into_iter().collect() +} + +fn append_operator_run_issue_id(issue_ids: &mut BTreeSet, run: &OperatorRunStatus) { + let issue_id = run.issue_id.trim(); + + if !issue_id.is_empty() && !issue_id.eq_ignore_ascii_case("unknown") { + issue_ids.insert(issue_id.to_owned()); + } +} + +fn hydrate_operator_snapshot_run_rows( + snapshot: &mut OperatorStatusSnapshot, + metadata_by_issue_id: &HashMap, +) { + for run in snapshot.active_runs.iter_mut().chain(snapshot.recent_runs.iter_mut()) { + hydrate_operator_run_row_from_issue_metadata(run, metadata_by_issue_id); + } + for lane in &mut snapshot.history_lanes { + hydrate_history_lane_from_issue_metadata(lane, metadata_by_issue_id); + } +} + +fn hydrate_history_lane_from_issue_metadata( + lane: &mut OperatorHistoryLaneStatus, + metadata_by_issue_id: &HashMap, +) { + if let Some(metadata) = metadata_by_issue_id.get(&lane.issue_id) { + apply_history_lane_issue_metadata(lane, metadata); + } + + hydrate_operator_run_row_from_issue_metadata(&mut lane.latest_run, metadata_by_issue_id); + + for attempt in &mut lane.attempts { + hydrate_operator_run_row_from_issue_metadata(attempt, metadata_by_issue_id); + } +} + +fn hydrate_operator_run_row_from_issue_metadata( + run: &mut OperatorRunStatus, + metadata_by_issue_id: &HashMap, +) { + if let Some(metadata) = metadata_by_issue_id.get(&run.issue_id) { + apply_run_issue_metadata(run, metadata); + } +} + +fn apply_history_lane_issue_metadata( + lane: &mut OperatorHistoryLaneStatus, + metadata: &OperatorIssueDisplayMetadata, +) { + if !metadata.issue_identifier.trim().is_empty() { + lane.issue_identifier = Some(metadata.issue_identifier.clone()); + lane.issue_key = metadata.issue_identifier.clone(); + } + + if let Some(title) = metadata.title.as_ref().filter(|title| !title.trim().is_empty()) { + lane.title = Some(title.clone()); + } +} + +fn apply_run_issue_metadata( + run: &mut OperatorRunStatus, + metadata: &OperatorIssueDisplayMetadata, +) { + if !metadata.issue_identifier.trim().is_empty() { + run.issue_identifier = Some(metadata.issue_identifier.clone()); + } + + if let Some(title) = metadata.title.as_ref().filter(|title| !title.trim().is_empty()) { + run.title = Some(title.clone()); + } +} + +fn fill_missing_history_lane_issue_metadata( + lane: &mut OperatorHistoryLaneStatus, + metadata: &OperatorIssueDisplayMetadata, +) { + if lane + .issue_identifier + .as_ref() + .is_none_or(|identifier| identifier.trim().is_empty()) + && !metadata.issue_identifier.trim().is_empty() + { + lane.issue_identifier = Some(metadata.issue_identifier.clone()); + lane.issue_key = metadata.issue_identifier.clone(); + } + if lane.title.as_ref().is_none_or(|title| title.trim().is_empty()) + && let Some(title) = metadata.title.as_ref().filter(|title| !title.trim().is_empty()) + { + lane.title = Some(title.clone()); + } +} + +fn fill_missing_run_issue_metadata( + run: &mut OperatorRunStatus, + metadata: &OperatorIssueDisplayMetadata, +) { + if run + .issue_identifier + .as_ref() + .is_none_or(|identifier| identifier.trim().is_empty()) + && !metadata.issue_identifier.trim().is_empty() + { + run.issue_identifier = Some(metadata.issue_identifier.clone()); + } + if run.title.as_ref().is_none_or(|title| title.trim().is_empty()) + && let Some(title) = metadata.title.as_ref().filter(|title| !title.trim().is_empty()) + { + run.title = Some(title.clone()); + } +} + +fn hydrate_history_lanes_from_linear_ledger( + tracker: &T, + project: &ServiceConfig, + snapshot: &mut OperatorStatusSnapshot, +) -> bool +where + T: IssueTracker, +{ + let mut unavailable = false; + + for lane in &mut snapshot.history_lanes { + match tracker.list_comments(&lane.issue_id) { + Ok(comments) => { + let records = + collect_history_ledger_records(project.service_id(), &lane.issue_id, &comments); + + hydrate_history_lane_from_ledger_records(lane, &records); + + lane.ledger_outcome = operator_history_ledger_outcome(&records); + }, + Err(error) => { + let _ = error; + + tracing::warn!( + issue_id = %lane.issue_id, + "Skipped Linear execution ledger lookup for a history lane; sensitive tracker details were withheld." + ); + + unavailable = true; + lane.ledger_outcome = unavailable_history_ledger_outcome(); + }, + } + } + + unavailable +} + +fn hydrate_history_lane_from_ledger_records( + lane: &mut OperatorHistoryLaneStatus, + records: &[OperatorHistoryLedgerRecord], +) { + let Some(record) = + records.iter().rev().find(|entry| !entry.record.issue_identifier.trim().is_empty()) + else { + return; + }; + let metadata = OperatorIssueDisplayMetadata { + issue_identifier: record.record.issue_identifier.clone(), + title: None, + }; + + fill_missing_history_lane_issue_metadata(lane, &metadata); + fill_missing_run_issue_metadata(&mut lane.latest_run, &metadata); + + for attempt in &mut lane.attempts { + fill_missing_run_issue_metadata(attempt, &metadata); + } +} + +fn local_history_ledger_records( + records: Vec, +) -> Vec { + let mut records = records + .into_iter() + .enumerate() + .map(|(comment_index, record)| { + let event_unix_epoch = parse_rfc3339_unix_epoch(&record.event_timestamp); + + OperatorHistoryLedgerRecord { + record, + event_unix_epoch, + sort_unix_epoch: event_unix_epoch, + comment_index, + } + }) + .collect::>(); + + records.sort_by(compare_history_ledger_record_position); + + records +} + +fn operator_history_ledger_outcome( + records: &[OperatorHistoryLedgerRecord], +) -> OperatorHistoryLedgerOutcome { + let Some(final_record) = final_history_ledger_record(records) else { + return missing_history_ledger_outcome(); + }; + let ledger_status = if history_ledger_event_outcome_rank(&final_record.record.event_type) > 1 { + String::from("present") + } else { + String::from("partial") + }; + let (started_at, finished_at, elapsed_seconds) = history_ledger_timing(records); + + OperatorHistoryLedgerOutcome { + ledger_status, + final_outcome: final_record.record.event_type.clone(), + final_event_type: Some(final_record.record.event_type.clone()), + final_event_at: Some(final_record.record.event_timestamp.clone()), + summary: history_ledger_summary(final_record, records), + pr_url: latest_history_ledger_text(records, |record| record.pr_url.as_deref()), + commit_sha: latest_history_ledger_text(records, |record| record.commit_sha.as_deref()), + branch: latest_history_ledger_text(records, |record| record.branch.as_deref()), + closeout_status: history_closeout_status(final_record, records), + needs_attention_reason: history_attention_reason(final_record), + lifecycle_started_at: started_at, + lifecycle_finished_at: finished_at, + lifecycle_elapsed_seconds: elapsed_seconds, + record_count: records.len(), + } +} + +fn collect_history_ledger_records( + service_id: &str, + issue_id: &str, + comments: &[TrackerComment], +) -> Vec { + let mut seen_keys = HashSet::new(); + let mut records = comments + .iter() + .enumerate() + .filter_map(|(comment_index, comment)| { + let record = records::parse_linear_execution_event_record(&comment.body)?; + + if record.service_id != service_id || record.issue_id != issue_id { + return None; + } + if !seen_keys.insert(record.idempotency_key.clone()) { + return None; + } + + let event_unix_epoch = parse_rfc3339_unix_epoch(&record.event_timestamp); + let comment_unix_epoch = parse_rfc3339_unix_epoch(&comment.created_at); + + Some(OperatorHistoryLedgerRecord { + record, + event_unix_epoch, + sort_unix_epoch: event_unix_epoch.or(comment_unix_epoch), + comment_index, + }) + }) + .collect::>(); + + records.sort_by(compare_history_ledger_record_position); + + records +} + +fn final_history_ledger_record( + records: &[OperatorHistoryLedgerRecord], +) -> Option<&OperatorHistoryLedgerRecord> { + records + .iter() + .filter(|entry| history_ledger_event_outcome_rank(&entry.record.event_type) > 1) + .max_by(|left, right| compare_history_ledger_record_position(left, right)) + .or_else(|| records.iter().max_by(|left, right| { + compare_history_ledger_record_position(left, right) + })) +} + +fn compare_history_ledger_record_position( + left: &OperatorHistoryLedgerRecord, + right: &OperatorHistoryLedgerRecord, +) -> Ordering { + left.sort_unix_epoch + .cmp(&right.sort_unix_epoch) + .then_with(|| left.comment_index.cmp(&right.comment_index)) +} + +fn history_ledger_event_outcome_rank(event_type: &str) -> u8 { + match event_type { + "cleanup_complete" => 7, + "closeout" => 6, + "needs_attention" | "terminal_failure" => 5, + "landed" => 4, + "review_handoff" | "repair_handoff" => 3, + "pr_opened" | "pr_updated" => 2, + _ => 1, + } +} + +fn history_ledger_timing( + records: &[OperatorHistoryLedgerRecord], +) -> (Option, Option, Option) { + let started = records.iter().filter_map(|entry| entry.event_unix_epoch).min(); + let finished = records.iter().filter_map(|entry| entry.event_unix_epoch).max(); + let elapsed = started + .zip(finished) + .and_then(|(started, finished)| finished.checked_sub(started)) + .filter(|elapsed| *elapsed >= 0); + + ( + started.and_then(|timestamp| format_optional_unix_timestamp(Some(timestamp))), + finished.and_then(|timestamp| format_optional_unix_timestamp(Some(timestamp))), + elapsed, + ) +} + +fn history_ledger_summary( + final_record: &OperatorHistoryLedgerRecord, + records: &[OperatorHistoryLedgerRecord], +) -> Option { + if history_ledger_event_outcome_rank(&final_record.record.event_type) > 1 { + return final_record.record.summary.clone(); + } + + Some(format!( + "Ledger has {} records but no final lane outcome yet; latest event is `{}`.", + records.len(), + final_record.record.event_type + )) +} + +fn latest_history_ledger_text( + records: &[OperatorHistoryLedgerRecord], + field: F, +) -> Option +where + F: Fn(&LinearExecutionEventRecord) -> Option<&str>, +{ + records.iter().rev().find_map(|entry| field(&entry.record).map(str::to_owned)) +} + +fn history_closeout_status( + final_record: &OperatorHistoryLedgerRecord, + records: &[OperatorHistoryLedgerRecord], +) -> Option { + match final_record.record.event_type.as_str() { + "closeout" => closeout_status_from_record(&final_record.record), + "cleanup_complete" => final_record.record.cleanup_status.clone().or_else(|| { + records.iter().rev().find_map(|entry| { + (entry.record.event_type == "closeout") + .then(|| closeout_status_from_record(&entry.record)) + .flatten() + }) + }), + _ => None, + } +} + +fn closeout_status_from_record(record: &LinearExecutionEventRecord) -> Option { + record + .target_state + .clone() + .or_else(|| record.validation_result.clone()) + .or_else(|| Some(String::from("recorded"))) +} + +fn history_attention_reason(final_record: &OperatorHistoryLedgerRecord) -> Option { + match final_record.record.event_type.as_str() { + "needs_attention" | "terminal_failure" => final_record + .record + .summary + .clone() + .or_else(|| final_record.record.error_class.clone()) + .or_else(|| final_record.record.next_action.clone()), + _ => None, + } +} + +fn parse_rfc3339_unix_epoch(value: &str) -> Option { + OffsetDateTime::parse(value, &Rfc3339).ok().map(|timestamp| timestamp.unix_timestamp()) +} + +fn not_loaded_history_ledger_outcome() -> OperatorHistoryLedgerOutcome { + OperatorHistoryLedgerOutcome { + ledger_status: String::from("not_loaded"), + final_outcome: String::from("local_attempt_history"), + final_event_type: None, + final_event_at: None, + summary: Some(String::from( + "Linear execution ledger was not loaded for this local-only snapshot.", + )), + pr_url: None, + commit_sha: None, + branch: None, + closeout_status: None, + needs_attention_reason: None, + lifecycle_started_at: None, + lifecycle_finished_at: None, + lifecycle_elapsed_seconds: None, + record_count: 0, + } +} + +fn missing_history_ledger_outcome() -> OperatorHistoryLedgerOutcome { + OperatorHistoryLedgerOutcome { + ledger_status: String::from("missing"), + final_outcome: String::from("execution_ledger_missing"), + final_event_type: None, + final_event_at: None, + summary: Some(String::from( + "No decodex.linear_execution_event records are available for this history lane.", + )), + pr_url: None, + commit_sha: None, + branch: None, + closeout_status: None, + needs_attention_reason: None, + lifecycle_started_at: None, + lifecycle_finished_at: None, + lifecycle_elapsed_seconds: None, + record_count: 0, + } +} + +fn unavailable_history_ledger_outcome() -> OperatorHistoryLedgerOutcome { + OperatorHistoryLedgerOutcome { + ledger_status: String::from("unavailable"), + final_outcome: String::from("ledger_unavailable"), + final_event_type: None, + final_event_at: None, + summary: Some(String::from( + "Linear execution ledger records could not be loaded for this issue.", + )), + pr_url: None, + commit_sha: None, + branch: None, + closeout_status: None, + needs_attention_reason: None, + lifecycle_started_at: None, + lifecycle_finished_at: None, + lifecycle_elapsed_seconds: None, + record_count: 0, + } +} + +fn build_queued_candidate_statuses( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> crate::prelude::Result> +where + T: IssueTracker, +{ + let queue_label = tracker::automation_queue_label(project.service_id()); + let concurrency = ConcurrencySnapshot::new(project.service_id(), state_store)?; + let mut issues = tracker.list_issues_with_label(&queue_label)?; + + issues.sort_by(compare_issue_candidates); + + issues + .into_iter() + .filter(|issue| !is_terminal_issue(issue, workflow)) + .map(|issue| { + operator_queued_issue_status( + tracker, + project, + workflow, + state_store, + &concurrency, + issue, + ) + }) + .collect() +} + +fn operator_queued_issue_status( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + concurrency: &ConcurrencySnapshot, + issue: TrackerIssue, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + let (classification, reason) = + classify_queued_issue(tracker, project, workflow, state_store, concurrency, &issue)?; + let attention = operator_queued_issue_attention_status( + tracker, + project, + workflow, + state_store, + &issue, + reason, + )?; + + Ok(OperatorQueuedIssueStatus { + issue_id: issue.id, + issue_identifier: issue.identifier, + title: issue.title, + state: issue.state.name, + priority: issue.priority, + created_at: issue.created_at, + classification: classification.to_owned(), + reason: reason.to_owned(), + attention, + blocker_identifiers: issue.blockers.into_iter().map(|blocker| blocker.identifier).collect(), + }) +} + +fn classify_queued_issue( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + concurrency: &ConcurrencySnapshot, + issue: &TrackerIssue, +) -> crate::prelude::Result<(&'static str, &'static str)> +where + T: IssueTracker, +{ + let tracker_policy = workflow.frontmatter().tracker(); + + if state_store.issue_has_active_shared_claim(project.service_id(), &issue.id)? { + return Ok(("claimed", "shared_claim_present")); + } + if tracker_policy.terminal_states().iter().any(|state| state == &issue.state.name) { + return Ok(("closed", "terminal_state")); + } + if !tracker_policy.startable_states().iter().any(|state| state == &issue.state.name) { + return Ok(("blocked", "non_startable_state")); + } + if issue.has_label(tracker_policy.opt_out_label()) { + return Ok(("blocked", "issue_opted_out")); + } + if issue.has_label(tracker_policy.needs_attention_label()) { + return Ok(("blocked", "issue_needs_attention")); + } + if !todo_blocker_rule_passes(issue, workflow) { + return Ok(("blocked", "open_tracker_blockers")); + } + if !issue_has_generic_dispatch_briefing(issue) { + return Ok(("blocked", "missing_dispatch_briefing")); + } + if !concurrency.has_global_capacity(workflow.frontmatter().execution()) { + return Ok(("waiting", "global_concurrency_exhausted")); + } + + let queue_label = tracker::automation_queue_label(project.service_id()); + + if !issue_passes_dispatch_policy(tracker, issue, workflow, &queue_label, true)? { + return Ok(("blocked", "dispatch_policy_rejected")); + } + + Ok(("ready", "eligible_for_dispatch")) +} + +fn operator_queued_issue_attention_status( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue: &TrackerIssue, + reason: &str, +) -> crate::prelude::Result> +where + T: IssueTracker, +{ + if !matches!(reason, "issue_needs_attention" | "retry_budget_exhausted") { + return Ok(None); + } + + let worktree_path = project.worktree_root().join(&issue.identifier); + let marker = state::read_run_activity_marker_snapshot(&worktree_path)?; + let state_retry_attempts = state_store.retry_budget_attempt_count(&issue.id)?; + let marker_retry_attempts = + marker.as_ref().and_then(RunActivityMarker::retry_budget_attempt_count).unwrap_or(0); + let retry_budget_attempts = state_retry_attempts.max(marker_retry_attempts); + let retry_budget_attempt_count = (retry_budget_attempts > 0).then_some(retry_budget_attempts); + let retry_budget_max_attempts = i64::from(workflow.frontmatter().execution().max_attempts()); + let auto_retry_blocked_reason = + (reason == "issue_needs_attention").then(|| String::from("needs_attention_label")); + let attention_record = + operator_queued_issue_latest_attention_record(tracker, project, state_store, issue); + let attempt_status = marker + .as_ref() + .and_then(|marker| state_store.run_attempt(marker.run_id()).transpose()) + .transpose()? + .map(|run_attempt| run_attempt.status().to_owned()); + let worktree_has_tracked_changes = worktree_has_tracked_changes(&worktree_path); + let summary = operator_queued_issue_attention_summary( + reason, + marker.as_ref(), + attempt_status.as_deref(), + retry_budget_attempts, + worktree_has_tracked_changes, + ); + + Ok(Some(OperatorQueuedIssueAttentionStatus { + summary, + run_id: marker.as_ref().map(|marker| marker.run_id().to_owned()), + attempt_number: marker.as_ref().map(RunActivityMarker::attempt_number), + current_operation: marker + .as_ref() + .and_then(RunActivityMarker::current_operation) + .map(str::to_owned), + thread_status: marker + .as_ref() + .and_then(RunActivityMarker::thread_status) + .map(str::to_owned), + attempt_status, + auto_retry_blocked_reason, + attention_error_class: attention_record + .as_ref() + .and_then(|record| record.error_class.clone()), + attention_next_action: attention_record + .as_ref() + .and_then(|record| record.next_action.clone()), + retry_budget_attempt_count, + retry_budget_max_attempts, + last_activity_at: marker + .as_ref() + .and_then(RunActivityMarker::last_activity_unix_epoch) + .and_then(|unix_epoch| format_optional_unix_timestamp(Some(unix_epoch))), + last_progress_at: marker + .as_ref() + .and_then(RunActivityMarker::last_progress_unix_epoch) + .and_then(|unix_epoch| format_optional_unix_timestamp(Some(unix_epoch))), + last_event_type: marker + .as_ref() + .and_then(RunActivityMarker::last_event_type) + .map(str::to_owned), + event_count: marker.as_ref().map_or(0, RunActivityMarker::event_count), + process_alive: marker.as_ref().and_then(|marker| marker.process_id().map(process_is_alive)), + worktree_path: worktree_path + .exists() + .then(|| relative_worktree_path_for_path(project, &worktree_path)), + worktree_has_tracked_changes, + })) +} + +fn operator_queued_issue_latest_attention_record( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, + issue: &TrackerIssue, +) -> Option +where + T: IssueTracker, +{ + let local_records = state_store + .list_linear_execution_events(project.service_id(), &issue.id) + .inspect_err(|error| { + tracing::debug!( + ?error, + issue_id = issue.id, + issue = issue.identifier, + "Failed to load local attention records for queued issue." + ); + }) + .ok(); + + if let Some(record) = + local_records.as_deref().and_then(latest_attention_record_from_linear_records) + { + return Some(record.clone()); + } + + let comments = tracker + .list_comments(&issue.id) + .inspect_err(|error| { + tracing::debug!( + ?error, + issue_id = issue.id, + issue = issue.identifier, + "Failed to load tracker comments for queued attention issue." + ); + }) + .ok()?; + let records = collect_history_ledger_records(project.service_id(), &issue.id, &comments); + + latest_attention_record_from_history_ledger_records(&records) + .map(|record| record.record.clone()) +} + +fn latest_attention_record_from_linear_records( + records: &[LinearExecutionEventRecord], +) -> Option<&LinearExecutionEventRecord> { + records + .iter() + .filter(|record| { + matches!(record.event_type.as_str(), "needs_attention" | "terminal_failure") + }) + .max_by(|left, right| { + parse_rfc3339_unix_epoch(&left.event_timestamp) + .cmp(&parse_rfc3339_unix_epoch(&right.event_timestamp)) + .then_with(|| left.idempotency_key.cmp(&right.idempotency_key)) + }) +} + +fn latest_attention_record_from_history_ledger_records( + records: &[OperatorHistoryLedgerRecord], +) -> Option<&OperatorHistoryLedgerRecord> { + records + .iter() + .filter(|entry| { + matches!(entry.record.event_type.as_str(), "needs_attention" | "terminal_failure") + }) + .max_by(|left, right| compare_history_ledger_record_position(left, right)) +} + +fn operator_queued_issue_attention_summary( + reason: &str, + marker: Option<&RunActivityMarker>, + attempt_status: Option<&str>, + retry_budget_attempts: i64, + worktree_has_tracked_changes: bool, +) -> String { + if retry_budget_attempts > 0 && worktree_has_tracked_changes { + return format!( + "Partial worktree changes are retained after {retry_budget_attempts} failed attempts; inspect the patch, finish validation, then land or reset manually." + ); + } + if marker + .and_then(RunActivityMarker::thread_status) + .is_some_and(|status| status == "systemError") + { + return if retry_budget_attempts > 0 { + format!( + "App-server thread ended with systemError after {retry_budget_attempts} retry-budget attempts." + ) + } else { + String::from("App-server thread ended with systemError.") + }; + } + if reason == "retry_budget_exhausted" { + return if retry_budget_attempts > 0 { + format!( + "Retry budget has {retry_budget_attempts} recorded failed attempts; operator recovery required." + ) + } else { + String::from("Retry budget exhausted; operator recovery required.") + }; + } + + if let Some(status) = attempt_status { + let operation = operator_recovery_operation_label(marker); + + match status { + "interrupted" => + return format!( + "Previous attempt was interrupted during {operation}; operator recovery required." + ), + "stalled" => + return format!( + "Previous attempt stalled during {operation}; operator recovery required." + ), + "failed" => + return format!( + "Previous attempt failed during {operation}; operator recovery required." + ), + "terminal_guarded" => + return format!( + "Previous attempt hit a terminal guard during {operation}; operator recovery required." + ), + _ => {}, + } + } + + if marker + .and_then(RunActivityMarker::last_event_type) + .is_some_and(|event_type| event_type == "item/tool/call") + { + return String::from("Stopped during a tool call; operator recovery required."); + } + + match marker.and_then(RunActivityMarker::current_operation) { + Some(RUN_OPERATION_GIT_CREDENTIALS) => { + String::from("Git credential preflight failed; operator recovery required.") + }, + Some(RUN_OPERATION_APP_SERVER_PREFLIGHT) => { + String::from("Codex app-server preflight failed; operator recovery required.") + }, + Some(RUN_OPERATION_RECONCILIATION) => { + String::from("Stopped during reconciliation or tracker handoff; operator recovery required.") + }, + Some(RUN_OPERATION_AGENT_RUN) => { + String::from("Stopped during agent execution; operator recovery required.") + }, + Some(operation) => + format!("Stopped during `{operation}`; operator recovery required."), + None => String::from("Needs operator recovery; no local run marker was found."), + } +} + +fn operator_recovery_operation_label(marker: Option<&RunActivityMarker>) -> String { + match marker.and_then(RunActivityMarker::current_operation) { + Some(RUN_OPERATION_GIT_CREDENTIALS) => String::from("git credential preflight"), + Some(RUN_OPERATION_APP_SERVER_PREFLIGHT) => { + String::from("Codex app-server preflight") + }, + Some(RUN_OPERATION_RECONCILIATION) => { + String::from("reconciliation or tracker handoff") + }, + Some(RUN_OPERATION_AGENT_RUN) => String::from("agent execution"), + Some(operation) => format!("`{operation}`"), + None => String::from("the lane"), + } +} + +fn build_post_review_lane_statuses( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> crate::prelude::Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let worktree_issues = load_post_review_worktree_issues(tracker, project, state_store)?; + + build_post_review_lane_statuses_from_worktree_issues( + project, + workflow, + state_store, + review_state_inspector, + worktree_issues, + ) +} + +fn build_post_review_lane_statuses_and_hydrate_worktrees( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, + snapshot: &mut OperatorStatusSnapshot, +) -> crate::prelude::Result> +where + T: IssueTracker, + I: PullRequestReviewStateInspector, +{ + let worktree_issues = load_post_review_worktree_issues(tracker, project, state_store)?; + + hydrate_worktree_issue_metadata(snapshot, &worktree_issues); + + build_post_review_lane_statuses_from_worktree_issues( + project, + workflow, + state_store, + review_state_inspector, + worktree_issues, + ) +} + +fn load_post_review_worktree_issues( + tracker: &T, + project: &ServiceConfig, + state_store: &StateStore, +) -> crate::prelude::Result> +where + T: IssueTracker, +{ + let worktrees = state_store.list_worktrees(project.service_id())?; + + if worktrees.is_empty() { + return Ok(Vec::new()); + } + + let issue_ids = + worktrees.iter().map(|mapping| mapping.issue_id().to_owned()).collect::>(); + let issues = tracker.refresh_issues(&issue_ids)?; + let issues_by_id = + issues.into_iter().map(|issue| (issue.id.clone(), issue)).collect::>(); + + Ok(worktrees + .into_iter() + .filter_map(|worktree| { + issues_by_id + .get(worktree.issue_id()) + .cloned() + .map(|issue| (worktree, issue)) + }) + .collect()) +} + +fn build_post_review_lane_statuses_from_worktree_issues( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, + worktree_issues: Vec<(WorktreeMapping, TrackerIssue)>, +) -> crate::prelude::Result> +where + I: PullRequestReviewStateInspector, +{ + let tracker_policy = workflow.frontmatter().tracker(); + let success_state = tracker_policy.success_state(); + let completed_state = tracker_policy.resolved_completed_state(); + let lane_context = PostReviewLaneBuildContext { + project, + workflow, + state_store, + review_state_inspector, + success_state, + completed_state, + }; + let mut lanes = Vec::new(); + + for (worktree, issue) in worktree_issues { + let Some(lane) = build_post_review_lane_status(&lane_context, issue, worktree)? else { + continue; + }; + + lanes.push(lane); + } + + lanes.sort_by(|left, right| left.issue_identifier.cmp(&right.issue_identifier)); + + Ok(lanes) +} + +fn hydrate_worktree_issue_metadata( + snapshot: &mut OperatorStatusSnapshot, + worktree_issues: &[(WorktreeMapping, TrackerIssue)], +) { + let issues_by_id = + worktree_issues.iter().map(|(_, issue)| (issue.id.as_str(), issue)).collect::>(); + + for worktree in &mut snapshot.worktrees { + let Some(issue) = issues_by_id.get(worktree.issue_id.as_str()) else { + continue; + }; + + worktree.issue_identifier = Some(issue.identifier.clone()); + worktree.issue_state = Some(issue.state.name.clone()); + } +} + +fn build_post_review_lane_status( + context: &PostReviewLaneBuildContext<'_, I>, + issue: TrackerIssue, + worktree: WorktreeMapping, +) -> crate::prelude::Result> +where + I: PullRequestReviewStateInspector, +{ + if issue.state.name != context.success_state && issue.state.name != context.completed_state { + return Ok(None); + } + + if let Some(reason) = post_review_lane_static_block_reason(&issue, context.workflow)? { + return Ok(Some(blocked_post_review_lane_status( + context.project, + &issue, + &worktree, + reason, + ))); + } + + let retry_budget_exhausted = issue_retry_budget_exhausted_for_worktree( + context.workflow, + context.state_store, + &issue.id, + worktree.worktree_path(), + )?; + let review_handoff = context.state_store.review_handoff_marker( + context.project.service_id(), + &issue.id, + worktree.branch_name(), + )?; + + if issue.state.name == context.completed_state && review_handoff.is_none() { + return Ok(None); + } + + let local_branch_name = match worktree_checkout_branch_name(worktree.worktree_path()) { + Ok(local_branch_name) => local_branch_name, + Err(_error) => + return Ok(Some(blocked_post_review_lane_status( + context.project, + &issue, + &worktree, + "worktree_checkout_branch_read_failed", + ))), + }; + let local_head_oid = match worktree_head_oid(worktree.worktree_path()) { + Ok(local_head_oid) => local_head_oid, + Err(_error) => + return Ok(Some(blocked_post_review_lane_status( + context.project, + &issue, + &worktree, + "worktree_head_read_failed", + ))), + }; + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff, + local_branch_name, + local_head_oid, + }; + let mut classification = classify_post_review_lane_with_project( + &snapshot, + context.project, + context.workflow, + context.state_store, + context.review_state_inspector, + )?; + + if retry_budget_exhausted { + classification = retry_budget_exhausted_post_review_lane_classification( + &snapshot, + context.project, + context.workflow, + context.review_state_inspector, + classification, + ); + } + + Ok(Some(post_review_lane_status_from_classification( + context.project, + &snapshot, + classification, + ))) +} + +fn post_review_lane_status_from_classification( + project: &ServiceConfig, + snapshot: &PostReviewLaneSnapshot, + classification: PostReviewLaneClassification, +) -> OperatorPostReviewLaneStatus { + OperatorPostReviewLaneStatus { + issue_id: snapshot.issue.id.clone(), + issue_identifier: snapshot.issue.identifier.clone(), + issue_state: snapshot.issue.state.name.clone(), + branch_name: snapshot.worktree.branch_name().to_owned(), + worktree_path: relative_worktree_path_for_path( + project, + snapshot.worktree.worktree_path(), + ), + classification: classification.decision.as_str().to_owned(), + reason: classification.reason, + pr_url: classification.pr_url, + pr_state: classification.pr_state, + review_decision: classification.review_decision, + mergeable: classification.mergeable, + check_state: classification.check_state, + unresolved_review_threads: classification.unresolved_review_threads, + } +} + +fn post_review_lane_static_block_reason( + issue: &TrackerIssue, + workflow: &WorkflowDocument, +) -> crate::prelude::Result> { + let tracker_policy = workflow.frontmatter().tracker(); + + if issue.has_label(tracker_policy.opt_out_label()) { + return Ok(Some("issue_opted_out")); + } + if issue.has_label(tracker_policy.needs_attention_label()) { + return Ok(Some("issue_needs_attention")); + } + + Ok(None) +} + +#[cfg_attr(not(test), allow(dead_code))] +#[cfg(test)] +fn classify_post_review_lane( + snapshot: &PostReviewLaneSnapshot, + state_store: &StateStore, + workflow: &WorkflowDocument, + review_state_inspector: &I, +) -> crate::prelude::Result +where + I: PullRequestReviewStateInspector, +{ + classify_post_review_lane_with_external_review( + snapshot, + workflow, + review_state_inspector, + true, + Some((state_store, "pubfi")), + ) +} + +fn classify_post_review_lane_with_project( + snapshot: &PostReviewLaneSnapshot, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + review_state_inspector: &I, +) -> crate::prelude::Result +where + I: PullRequestReviewStateInspector, +{ + classify_post_review_lane_with_external_review( + snapshot, + workflow, + review_state_inspector, + project.codex().external_review_enabled(), + Some((state_store, project.service_id())), + ) +} + +fn classify_post_review_lane_with_external_review( + snapshot: &PostReviewLaneSnapshot, + workflow: &WorkflowDocument, + review_state_inspector: &I, + external_review_enabled: bool, + runtime_state: Option<(&StateStore, &str)>, +) -> crate::prelude::Result +where + I: PullRequestReviewStateInspector, +{ + let review_state = match load_post_review_lane_review_state(snapshot, review_state_inspector)? { + PostReviewLaneStateLoad::Classification(classification) => return Ok(classification), + PostReviewLaneStateLoad::ReviewState(review_state) => review_state, + }; + let mut classification = initial_post_review_lane_classification(&review_state); + + if apply_pre_orchestration_post_review_classification( + snapshot, + workflow, + &review_state, + &mut classification, + ) { + return Ok(classification); + } + if !external_review_enabled { + let orchestration_marker = load_post_review_orchestration_marker( + snapshot, + &review_state, + &mut classification, + runtime_state, + )?; + + if classification.decision == PostReviewLaneDecision::Block { + return Ok(classification); + } + + apply_internal_review_only_post_review_classification( + &mut classification, + &review_state, + orchestration_marker.as_ref(), + OffsetDateTime::now_utc().unix_timestamp(), + )?; + + return Ok(classification); + } + + let Some(orchestration_marker) = + load_post_review_orchestration_marker( + snapshot, + &review_state, + &mut classification, + runtime_state, + )? + else { + return Ok(classification); + }; + let orchestration_status = + PostReviewOrchestrationStatus::from_review_state(&review_state, &orchestration_marker)?; + + apply_review_orchestration_phase_classification( + &mut classification, + &review_state, + &orchestration_marker, + &orchestration_status, + OffsetDateTime::now_utc().unix_timestamp(), + ); + + Ok(classification) +} + +fn retry_budget_exhausted_post_review_lane_classification( + snapshot: &PostReviewLaneSnapshot, + project: &ServiceConfig, + workflow: &WorkflowDocument, + review_state_inspector: &I, + mut classification: PostReviewLaneClassification, +) -> PostReviewLaneClassification +where + I: PullRequestReviewStateInspector, +{ + if classification.pr_url.is_none() { + classification.pr_url = + snapshot.review_handoff.as_ref().map(|marker| marker.pr_url().to_owned()); + } + if classification.pr_state.is_none() + && let Some(review_state) = + retry_budget_exhausted_merged_review_state(snapshot, review_state_inspector) + { + classification = initial_post_review_lane_classification(&review_state); + + apply_pre_orchestration_post_review_classification( + snapshot, + workflow, + &review_state, + &mut classification, + ); + } + if merged_closeout_pending_classification(&classification) + && worktree_has_no_tracked_changes(snapshot.worktree.worktree_path()) + { + if snapshot.issue.state.name == workflow.frontmatter().tracker().resolved_completed_state() + && !worktree_has_no_tracked_changes(project.repo_root()) + { + classification.decision = PostReviewLaneDecision::CleanupBlocked; + classification.reason = String::from("default_branch_worktree_dirty"); + + return classification; + } + + return classification; + } + if classification.pr_state.as_deref() == Some("MERGED") + && worktree_has_no_tracked_changes(snapshot.worktree.worktree_path()) + { + classification.decision = + if snapshot.issue.state.name == workflow.frontmatter().tracker().resolved_completed_state() + { + PostReviewLaneDecision::CleanupBlocked + } else { + PostReviewLaneDecision::CloseoutBlocked + }; + classification.reason = String::from("retry_budget_exhausted"); + + return classification; + } + + classification.decision = PostReviewLaneDecision::Block; + classification.reason = String::from("retry_budget_exhausted"); + + classification +} + +fn merged_closeout_pending_classification( + classification: &PostReviewLaneClassification, +) -> bool { + classification.decision == PostReviewLaneDecision::Continue + && classification.reason == "pull_request_merged_closeout_pending" + && classification.pr_state.as_deref() == Some("MERGED") +} + +fn retry_budget_exhausted_merged_review_state( + snapshot: &PostReviewLaneSnapshot, + review_state_inspector: &I, +) -> Option +where + I: PullRequestReviewStateInspector, +{ + let review_handoff = snapshot.review_handoff.as_ref()?; + + if !worktree_has_no_tracked_changes(snapshot.worktree.worktree_path()) { + return None; + } + + let review_state = review_state_inspector + .inspect_review_state(snapshot.worktree.worktree_path(), review_handoff.pr_url()) + .ok()?; + + (review_state.state == "MERGED").then_some(review_state) +} + +fn worktree_has_no_tracked_changes(worktree_path: &Path) -> bool { + let Ok(output) = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["status", "--porcelain", "--untracked-files=no"]) + .output() + else { + return false; + }; + + output.status.success() && String::from_utf8_lossy(&output.stdout).trim().is_empty() +} + +fn worktree_has_tracked_changes(worktree_path: &Path) -> bool { + let Ok(output) = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["status", "--porcelain", "--untracked-files=no"]) + .output() + else { + return false; + }; + + output.status.success() && !String::from_utf8_lossy(&output.stdout).trim().is_empty() +} + +fn apply_pre_orchestration_post_review_classification( + snapshot: &PostReviewLaneSnapshot, + workflow: &WorkflowDocument, + review_state: &PullRequestReviewState, + classification: &mut PostReviewLaneClassification, +) -> bool { + if review_state.state == "MERGED" { + classification.decision = PostReviewLaneDecision::Continue; + classification.reason = String::from("pull_request_merged_closeout_pending"); + + return true; + } + if snapshot.issue.state.name == workflow.frontmatter().tracker().resolved_completed_state() { + *classification = blocked_post_review_lane_from_state( + review_state, + "issue_completed_before_pull_request_merged", + ); + + return true; + } + if review_state.state != "OPEN" { + *classification = + blocked_post_review_lane_from_state(review_state, "pull_request_not_open"); + + return true; + } + if review_state.is_draft { + *classification = + blocked_post_review_lane_from_state(review_state, "pull_request_is_draft"); + + return true; + } + if review_state.unresolved_review_threads > 0 { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("unresolved_review_threads"); + + return true; + } + if matches!(review_state.review_decision.as_deref(), Some("CHANGES_REQUESTED")) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("review_changes_requested"); + + return true; + } + if failed_checks_require_repair( + review_state.status_check_rollup_state.as_deref(), + &review_state.merge_state_status, + ) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("required_checks_failed"); + + return true; + } + + if let Some(reason) = merge_state_requires_review_repair( + &review_state.mergeable, + &review_state.merge_state_status, + ) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from(reason); + + return true; + } + + false +} + +fn apply_internal_review_only_post_review_classification( + classification: &mut PostReviewLaneClassification, + review_state: &PullRequestReviewState, + orchestration_marker: Option<&ReviewOrchestrationMarker>, + now_unix_epoch: i64, +) -> crate::prelude::Result<()> { + if let Some(orchestration_marker) = orchestration_marker { + let phase = + ReviewOrchestrationPhase::parse(orchestration_marker.phase()).map_err(|error| { + eyre::eyre!("Failed to parse retained review orchestration phase: {error}") + })?; + + if phase == ReviewOrchestrationPhase::WaitingForMerge { + if let Some(auto_merge_enabled_at) = + orchestration_marker.auto_merge_enabled_at_unix_epoch() + && now_unix_epoch - auto_merge_enabled_at + > EXTERNAL_REVIEW_MERGE_VISIBILITY_TIMEOUT_SECS + { + *classification = blocked_post_review_lane_from_state( + review_state, + "internal_review_only_merge_visibility_timeout", + ); + } else { + classification.reason = String::from("internal_review_only_waiting_for_merge"); + } + + return Ok(()); + } + if phase == ReviewOrchestrationPhase::RepairRequired { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = if review_state_landing_requires_agent_fallback(review_state) { + String::from("retained_landing_agent_fallback_required") + } else { + String::from("internal_review_only_repair_required") + }; + + return Ok(()); + } + } + + if review_state_clean_path_landing_gates_satisfied(review_state) { + classification.decision = PostReviewLaneDecision::ReadyToLand; + classification.reason = String::from("internal_review_only_ready_to_land"); + } else if review_state_landing_requires_agent_fallback(review_state) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("retained_landing_agent_fallback_required"); + } else { + classification.reason = String::from("internal_review_only_waiting_gates"); + } + + Ok(()) +} + +fn load_post_review_orchestration_marker( + snapshot: &PostReviewLaneSnapshot, + review_state: &PullRequestReviewState, + classification: &mut PostReviewLaneClassification, + runtime_state: Option<(&StateStore, &str)>, +) -> crate::prelude::Result> { + let review_handoff = snapshot + .review_handoff + .as_ref() + .expect("review handoff should exist before orchestration classification"); + let orchestration_marker = if let Some((state_store, project_id)) = runtime_state { + state_store.review_orchestration_marker(project_id, &snapshot.issue.id, review_handoff)? + } else { + None + }; + let Some(orchestration_marker) = orchestration_marker else { + classification.reason = String::from("external_review_request_pending"); + + return Ok(None); + }; + + if let Some(reason) = + validate_review_orchestration_marker(snapshot, review_state, &orchestration_marker) + { + *classification = blocked_post_review_lane_from_state(review_state, reason); + + return Ok(None); + } + + Ok(Some(orchestration_marker)) +} + +fn apply_review_orchestration_phase_classification( + classification: &mut PostReviewLaneClassification, + review_state: &PullRequestReviewState, + orchestration_marker: &ReviewOrchestrationMarker, + orchestration_status: &PostReviewOrchestrationStatus, + now_unix_epoch: i64, +) { + match orchestration_status.phase { + ReviewOrchestrationPhase::RequestPending => { + match external_review_request_ci_gate(review_state) { + ExternalReviewRequestCiGate::Ready => { + classification.reason = String::from("external_review_request_pending"); + }, + ExternalReviewRequestCiGate::WaitForGreenChecks => { + classification.reason = + String::from("external_review_request_waiting_for_green_checks"); + }, + ExternalReviewRequestCiGate::RepairRequired => { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = + String::from("external_review_request_ci_red_repair_required"); + }, + ExternalReviewRequestCiGate::ManualAttention(reason) => { + *classification = blocked_post_review_lane_from_state(review_state, reason); + }, + } + }, + ReviewOrchestrationPhase::WaitingForAck => { + if orchestration_status.request_acknowledged { + classification.reason = String::from("external_review_result_pending"); + } else if request_ack_timed_out(orchestration_marker, now_unix_epoch) { + *classification = blocked_post_review_lane_from_state( + review_state, + "external_review_ack_timeout", + ); + } else { + classification.reason = String::from("external_review_ack_pending"); + } + }, + ReviewOrchestrationPhase::WaitingForResult => { + if !orchestration_status.request_acknowledged { + classification.reason = String::from("external_review_ack_pending"); + } else if external_review_has_actionable_feedback(review_state, orchestration_marker) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("external_review_feedback_pending_repair"); + } else if orchestration_status.strict_pass + && orchestration_status.clean_path_landing_gates_satisfied + { + classification.decision = PostReviewLaneDecision::ReadyToLand; + classification.reason = String::from("external_review_passed_strict"); + } else if orchestration_status.strict_pass + && orchestration_status.landing_requires_agent_fallback + { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("retained_landing_agent_fallback_required"); + } else if orchestration_status.strict_pass { + classification.reason = String::from("external_review_passed_waiting_gates"); + } else if orchestration_status.review_result_arrived { + *classification = blocked_post_review_lane_from_state( + review_state, + "external_review_pass_signal_missing", + ); + } else { + classification.reason = String::from("external_review_result_pending"); + } + }, + ReviewOrchestrationPhase::RepairRequired => { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = if orchestration_status.landing_requires_agent_fallback { + String::from("retained_landing_agent_fallback_required") + } else { + String::from("external_review_feedback_pending_repair") + }; + }, + ReviewOrchestrationPhase::PassWaitingForGates => { + if external_review_has_actionable_feedback(review_state, orchestration_marker) { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("external_review_feedback_pending_repair"); + } else if orchestration_status.strict_pass + && orchestration_status.clean_path_landing_gates_satisfied + { + classification.decision = PostReviewLaneDecision::ReadyToLand; + classification.reason = String::from("external_review_passed_strict"); + } else if orchestration_status.strict_pass + && orchestration_status.landing_requires_agent_fallback + { + classification.decision = PostReviewLaneDecision::NeedsReviewRepair; + classification.reason = String::from("retained_landing_agent_fallback_required"); + } else if orchestration_status.strict_pass { + classification.reason = String::from("external_review_passed_waiting_gates"); + } else { + *classification = blocked_post_review_lane_from_state( + review_state, + "external_review_pass_signal_missing", + ); + } + }, + ReviewOrchestrationPhase::WaitingForMerge => { + if let Some(auto_merge_enabled_at) = + orchestration_marker.auto_merge_enabled_at_unix_epoch() + && now_unix_epoch - auto_merge_enabled_at + > EXTERNAL_REVIEW_MERGE_VISIBILITY_TIMEOUT_SECS + { + *classification = blocked_post_review_lane_from_state( + review_state, + "external_review_merge_visibility_timeout", + ); + } else { + classification.reason = String::from("external_review_waiting_for_merge"); + } + }, + } +} + +fn load_post_review_lane_review_state( + snapshot: &PostReviewLaneSnapshot, + review_state_inspector: &I, +) -> crate::prelude::Result +where + I: PullRequestReviewStateInspector, +{ + if let Some(review_handoff) = snapshot.review_handoff.as_ref() { + let local_head_oid = match validate_post_review_lane_worktree(snapshot, review_handoff) { + Ok(local_head_oid) => local_head_oid, + Err(reason) => { + return Ok(PostReviewLaneStateLoad::Classification(blocked_post_review_lane( + reason, + ))); + }, + }; + let review_state = match review_state_inspector + .inspect_review_state(snapshot.worktree.worktree_path(), review_handoff.pr_url()) + { + Ok(review_state) => review_state, + Err(_error) => { + return Ok(PostReviewLaneStateLoad::Classification(blocked_post_review_lane( + "pull_request_state_read_failed", + ))); + }, + }; + + return Ok(validate_post_review_lane_review_state( + review_state, + snapshot.worktree.branch_name(), + local_head_oid, + snapshot.worktree.worktree_path(), + )); + } + + Ok(PostReviewLaneStateLoad::Classification(blocked_post_review_lane( + "missing_review_handoff_record", + ))) +} + +fn validate_post_review_lane_review_state( + review_state: PullRequestReviewState, + expected_branch_name: &str, + local_head_oid: &str, + worktree_path: &Path, +) -> PostReviewLaneStateLoad { + let Some(pr_owner) = + github::parse_pull_request_url(&review_state.url).ok().map(|locator| locator.owner) + else { + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_repository_parse_failed", + )); + }; + let Some(pr_repo) = + github::parse_pull_request_url(&review_state.url).ok().map(|locator| locator.repo) + else { + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_repository_parse_failed", + )); + }; + + if review_state.head_repository_owner.as_deref() != Some(pr_owner.as_str()) { + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_head_repository_owner_mismatch", + )); + } + if review_state.head_repository_name.as_deref() != Some(pr_repo.as_str()) { + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_head_repository_name_mismatch", + )); + } + if review_state.head_ref_name != expected_branch_name { + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_branch_mismatch", + )); + } + if review_state.head_ref_oid != local_head_oid { + match merged_pr_local_head_matches_landed_lineage( + worktree_path, + &review_state, + local_head_oid, + ) { + Ok(true) => return PostReviewLaneStateLoad::ReviewState(review_state), + Ok(false) => {}, + Err(reason) => { + return PostReviewLaneStateLoad::Classification( + blocked_post_review_lane_from_state(&review_state, reason), + ); + }, + } + + return PostReviewLaneStateLoad::Classification(blocked_post_review_lane_from_state( + &review_state, + "pull_request_head_mismatch", + )); + } + + PostReviewLaneStateLoad::ReviewState(review_state) +} + +fn merged_pr_local_head_matches_landed_lineage( + worktree_path: &Path, + review_state: &PullRequestReviewState, + local_head_oid: &str, +) -> std::result::Result { + if review_state.state != "MERGED" { + return Ok(false); + } + + let Some(merge_commit_oid) = review_state.merge_commit_oid.as_deref() else { + return Ok(false); + }; + + if merge_commit_oid == local_head_oid { + return Ok(true); + } + + worktree_head_descends_from_review_handoff( + worktree_path, + merge_commit_oid, + local_head_oid, + ) + .map_err(|()| "pull_request_merge_commit_lineage_check_failed") +} + +fn validate_review_orchestration_marker( + snapshot: &PostReviewLaneSnapshot, + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> Option<&'static str> { + let Some(local_head_oid) = snapshot.local_head_oid.as_deref() else { + return Some("worktree_head_missing"); + }; + + if marker.branch_name() != snapshot.worktree.branch_name() { + return Some("review_orchestration_branch_mismatch"); + } + if marker.pr_url() != review_state.url { + return Some("review_orchestration_pr_mismatch"); + } + if marker.head_sha() != local_head_oid { + return Some("review_orchestration_head_mismatch"); + } + + None +} + +fn request_comment_has_eyes( + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> Option { + let request_comment_id = marker.request_comment_database_id()?; + + Some( + review_state + .issue_comments + .iter() + .find(|comment| comment.database_id == request_comment_id) + .is_some_and(|comment| comment.external_review_eyes_reaction_count > 0), + ) +} + +fn request_ack_timed_out(marker: &ReviewOrchestrationMarker, now_unix_epoch: i64) -> bool { + let Some(request_created_at_unix_epoch) = marker.request_created_at_unix_epoch() else { + return false; + }; + + now_unix_epoch - request_created_at_unix_epoch > EXTERNAL_REVIEW_ACK_TIMEOUT_SECS + && marker.request_retry_count() >= 1 +} + +fn external_review_result_arrived( + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> bool { + let Some(request_created_at_unix_epoch) = marker.request_created_at_unix_epoch() else { + return false; + }; + + review_state.reviews.iter().any(|review| { + review.submitted_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(review.author_login.as_deref()) + }) || review_state.issue_comments.iter().any(|comment| { + Some(comment.database_id) != marker.request_comment_database_id() + && comment.created_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(comment.author_login.as_deref()) + }) +} + +fn external_review_has_strict_pass_signals( + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> bool { + let Some(request_created_at_unix_epoch) = marker.request_created_at_unix_epoch() else { + return false; + }; + let pass_phrase_seen_after_request = review_state.reviews.iter().any(|review| { + review.submitted_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(review.author_login.as_deref()) + && external_review_body_is_strict_pass_signal(&review.body) + }) || review_state.issue_comments.iter().any(|comment| { + Some(comment.database_id) != marker.request_comment_database_id() + && comment.created_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(comment.author_login.as_deref()) + && external_review_body_is_strict_pass_signal(&comment.body) + }); + + pass_phrase_seen_after_request + && review_state.issue_description_external_review_thumbs_up_count > 0 +} + +fn external_review_has_actionable_feedback( + review_state: &PullRequestReviewState, + marker: &ReviewOrchestrationMarker, +) -> bool { + let Some(request_created_at_unix_epoch) = marker.request_created_at_unix_epoch() else { + return false; + }; + + review_state.reviews.iter().any(|review| { + review.submitted_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(review.author_login.as_deref()) + && matches!(review.state.as_str(), "COMMENTED" | "CHANGES_REQUESTED") + && external_review_body_has_actionable_feedback(&review.body) + }) || review_state.issue_comments.iter().any(|comment| { + Some(comment.database_id) != marker.request_comment_database_id() + && comment.created_at_unix_epoch >= request_created_at_unix_epoch + && is_external_review_actor_login(comment.author_login.as_deref()) + && external_review_body_has_actionable_feedback(&comment.body) + }) +} + +fn is_external_review_actor_login(login: Option<&str>) -> bool { + login.is_some_and(|login| login.eq_ignore_ascii_case(EXTERNAL_REVIEW_ACTOR_LOGIN)) +} + +fn external_review_body_is_strict_pass_signal(body: &str) -> bool { + body.trim() == EXTERNAL_REVIEW_PASS_PHRASE +} + +fn external_review_body_has_actionable_feedback(body: &str) -> bool { + let trimmed = body.trim(); + + !trimmed.is_empty() && !external_review_body_is_strict_pass_signal(trimmed) +} + +fn retained_closeout_pr_merge_gate_with_inspector( + worktree_path: &Path, + expected_branch_name: &str, + pr_url: &str, + review_state_inspector: &I, +) -> crate::prelude::Result +where + I: PullRequestReviewStateInspector + ?Sized, +{ + let Some(local_branch_name) = worktree_checkout_branch_name(worktree_path)? else { + return Ok(RetainedCloseoutPrMergeGate::NotMerged); + }; + let Some(local_head_oid) = worktree_head_oid(worktree_path)? else { + return Ok(RetainedCloseoutPrMergeGate::NotMerged); + }; + + if local_branch_name != expected_branch_name { + return Ok(RetainedCloseoutPrMergeGate::NotMerged); + } + + let review_state = match review_state_inspector.inspect_review_state(worktree_path, pr_url) { + Ok(review_state) => review_state, + Err(_error) => return Ok(RetainedCloseoutPrMergeGate::PullRequestStateReadFailed), + }; + + Ok( + if matches!( + validate_post_review_lane_review_state( + review_state, + expected_branch_name, + &local_head_oid, + worktree_path, + ), + PostReviewLaneStateLoad::ReviewState(PullRequestReviewState { + state, + is_draft: false, + .. + }) if state == "MERGED" + ) { + RetainedCloseoutPrMergeGate::Merged + } else { + RetainedCloseoutPrMergeGate::NotMerged + }, + ) +} + +fn validate_post_review_lane_worktree<'a>( + snapshot: &'a PostReviewLaneSnapshot, + review_handoff: &ReviewHandoffMarker, +) -> std::result::Result<&'a str, &'static str> { + if review_handoff.branch_name() != snapshot.worktree.branch_name() { + return Err("worktree_branch_mismatch"); + } + + let Some(local_branch_name) = snapshot.local_branch_name.as_deref() else { + return Err("worktree_checkout_branch_missing"); + }; + + if local_branch_name != review_handoff.branch_name() + || local_branch_name != snapshot.worktree.branch_name() + { + return Err("worktree_checkout_branch_mismatch"); + } + + let Some(local_head_oid) = snapshot.local_head_oid.as_deref() else { + return Err("worktree_head_missing"); + }; + + if local_head_oid != review_handoff.pr_head_oid() { + match worktree_head_descends_from_review_handoff( + snapshot.worktree.worktree_path(), + review_handoff.pr_head_oid(), + local_head_oid, + ) { + Ok(true) => {}, + Ok(false) => return Err("review_handoff_lineage_mismatch"), + Err(()) => return Err("review_handoff_lineage_check_failed"), + } + } + + Ok(local_head_oid) +} + +fn worktree_head_descends_from_review_handoff( + worktree_path: &Path, + recorded_head_oid: &str, + local_head_oid: &str, +) -> std::result::Result { + if recorded_head_oid == local_head_oid { + return Ok(true); + } + + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["merge-base", "--is-ancestor", recorded_head_oid, local_head_oid]) + .output() + .map_err(|_| ())?; + + match output.status.code() { + Some(0) => Ok(true), + Some(1) => Ok(false), + _ => Err(()), + } +} + +fn initial_post_review_lane_classification( + review_state: &PullRequestReviewState, +) -> PostReviewLaneClassification { + PostReviewLaneClassification { + decision: PostReviewLaneDecision::WaitForReview, + reason: String::from("waiting_for_review_or_checks"), + pr_url: Some(review_state.url.clone()), + pr_state: Some(review_state.state.clone()), + review_decision: review_state.review_decision.clone(), + mergeable: Some(review_state.mergeable.clone()), + check_state: review_state.status_check_rollup_state.clone(), + unresolved_review_threads: Some(review_state.unresolved_review_threads), + } +} + +fn blocked_post_review_lane_from_state( + review_state: &PullRequestReviewState, + reason: &str, +) -> PostReviewLaneClassification { + let mut classification = initial_post_review_lane_classification(review_state); + + classification.decision = PostReviewLaneDecision::Block; + classification.reason = reason.to_owned(); + + classification +} + +fn blocked_post_review_lane(reason: &str) -> PostReviewLaneClassification { + PostReviewLaneClassification { + decision: PostReviewLaneDecision::Block, + reason: reason.to_owned(), + pr_url: None, + pr_state: None, + review_decision: None, + mergeable: None, + check_state: None, + unresolved_review_threads: None, + } +} + +fn blocked_post_review_lane_status( + project: &ServiceConfig, + issue: &TrackerIssue, + worktree: &WorktreeMapping, + reason: &str, +) -> OperatorPostReviewLaneStatus { + OperatorPostReviewLaneStatus { + issue_id: issue.id.clone(), + issue_identifier: issue.identifier.clone(), + issue_state: issue.state.name.clone(), + branch_name: worktree.branch_name().to_owned(), + worktree_path: relative_worktree_path_for_path(project, worktree.worktree_path()), + classification: String::from("blocked"), + reason: String::from(reason), + pr_url: None, + pr_state: None, + review_decision: None, + mergeable: None, + check_state: None, + unresolved_review_threads: None, + } +} + +fn worktree_head_oid(worktree_path: &Path) -> crate::prelude::Result> { + let output = + Command::new("git").arg("-C").arg(worktree_path).args(["rev-parse", "HEAD"]).output()?; + + if !output.status.success() { + if !worktree_path.exists() { + return Ok(None); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect worktree HEAD in `{}`: {}", + worktree_path.display(), + stderr.trim() + ); + } + + Ok(Some(String::from_utf8_lossy(&output.stdout).trim().to_owned())) +} + +fn worktree_checkout_branch_name(worktree_path: &Path) -> crate::prelude::Result> { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["branch", "--show-current"]) + .output()?; + + if !output.status.success() { + if !worktree_path.exists() { + return Ok(None); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect worktree checkout branch in `{}`: {}", + worktree_path.display(), + stderr.trim() + ); + } + + let branch_name = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + + if branch_name.is_empty() { + return Ok(None); + } + + Ok(Some(branch_name)) +} + +fn resolve_configured_env_var( + field_name: &str, + env_var: Option<&str>, +) -> crate::prelude::Result { + let env_var = env_var.ok_or_else(|| { + eyre::eyre!("`{field_name}` must be configured for this GitHub-backed operation.") + })?; + let value = env::var(env_var).map_err(|error| { + eyre::eyre!( + "Failed to read environment variable `{env_var}` referenced by `{field_name}`: {error}" + ) + })?; + + if value.trim().is_empty() { + eyre::bail!( + "Environment variable `{env_var}` referenced by `{field_name}` must not be blank." + ); + } + + Ok(value) +} + +fn external_review_request_ci_gate( + review_state: &PullRequestReviewState, +) -> ExternalReviewRequestCiGate { + match review_state.status_check_rollup_state.as_deref() { + None | Some("SUCCESS") => ExternalReviewRequestCiGate::Ready, + Some("EXPECTED" | "PENDING") => ExternalReviewRequestCiGate::WaitForGreenChecks, + Some("ERROR" | "FAILURE") + if failed_checks_require_repair( + review_state.status_check_rollup_state.as_deref(), + &review_state.merge_state_status, + ) => + { + ExternalReviewRequestCiGate::RepairRequired + }, + Some("ERROR" | "FAILURE") | Some(_) => ExternalReviewRequestCiGate::ManualAttention( + "external_review_request_ci_red_manual_attention", + ), + } +} + +fn failed_checks_require_repair(check_state: Option<&str>, merge_state_status: &str) -> bool { + pull_request::failed_checks_require_repair(check_state, merge_state_status) +} + +fn merge_state_requires_review_repair( + mergeable: &str, + merge_state_status: &str, +) -> Option<&'static str> { + pull_request::merge_state_requires_review_repair(mergeable, merge_state_status) +} + +fn review_state_landing_gates_satisfied(review_state: &PullRequestReviewState) -> bool { + pull_request::retained_landing_gates_satisfied(review_state_landing_gate_view(review_state)) +} + +fn review_state_clean_path_landing_gates_satisfied( + review_state: &PullRequestReviewState, +) -> bool { + pull_request::retained_clean_path_landing_gates_satisfied(review_state_landing_gate_view( + review_state, + )) +} + +fn review_state_landing_requires_agent_fallback(review_state: &PullRequestReviewState) -> bool { + pull_request::retained_landing_requires_agent_fallback(review_state_landing_gate_view( + review_state, + )) +} + +fn review_state_landing_gate_view( + review_state: &PullRequestReviewState, +) -> PullRequestLandingGateView<'_> { + PullRequestLandingGateView { + state: review_state.state.as_str(), + is_draft: review_state.is_draft, + review_decision: review_state.review_decision.as_deref(), + pending_review_requests: review_state.pending_review_requests, + mergeable: review_state.mergeable.as_str(), + merge_state_status: review_state.merge_state_status.as_str(), + status_check_rollup_state: review_state.status_check_rollup_state.as_deref(), + unresolved_review_threads: review_state.unresolved_review_threads, + } +} + +fn recover_runtime_state_from_tracker_and_worktrees( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + let worktree_manager = + WorktreeManager::new(project.service_id(), project.repo_root(), project.worktree_root()); + let mut issue_ids = state_store + .list_worktrees(project.service_id())? + .into_iter() + .map(|mapping| mapping.issue_id().to_owned()) + .collect::>(); + + for lease in state_store.list_active_shared_leases(project.service_id())? { + if !issue_ids.iter().any(|issue_id| issue_id == lease.issue_id()) { + issue_ids.push(lease.issue_id().to_owned()); + } + } + + let mut issues = tracker.refresh_issues(&issue_ids)?; + let mut known_identifiers = + issues.iter().map(|issue| issue.identifier.to_ascii_uppercase()).collect::>(); + + for issue_identifier in recoverable_worktree_identifiers(project.worktree_root())? { + append_recoverable_tracker_issue( + tracker, + project, + &issue_identifier, + &mut known_identifiers, + &mut issues, + )?; + } + + let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut active_issues = Vec::new(); + + for issue in issues { + let worktree = worktree_manager.plan_for_issue(&issue.identifier); + + if !worktree.path.exists() { + continue; + } + + state_store.canonicalize_issue_identity(&issue.identifier, &issue.id)?; + state_store.upsert_worktree( + project.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + )?; + + let activity_marker = state::read_run_activity_marker_snapshot(&worktree.path)?; + + if issue.state.name == workflow.frontmatter().tracker().success_state() + && issue_has_service_ownership(tracker, &issue, project.service_id())? + && let Some(marker) = activity_marker.as_ref() + && worktree_activity_marker_is_fresh(marker, now_unix_epoch) + { + record_recovered_activity_lease(project, state_store, &issue, marker)?; + + continue; + } + if issue_passes_closeout_dispatch_policy(tracker, &issue, project, workflow, state_store)? + { + match activity_marker.as_ref() { + Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { + record_recovered_activity_lease(project, state_store, &issue, marker)?; + + continue; + }, + _ => {}, + } + } + if issue_passes_retry_dispatch_policy( + tracker, + &issue, + project, + workflow, + state_store, + RetryIssueStateHint::default(), + )? { + match activity_marker.as_ref() { + Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { + record_recovered_activity_lease(project, state_store, &issue, marker)?; + + continue; + }, + Some(marker) => { + clear_recovered_issue_lease( + project.service_id(), + &issue.id, + Some(marker.run_id()), + state_store, + )?; + }, + None => { + clear_recovered_issue_lease( + project.service_id(), + &issue.id, + None, + state_store, + )?; + }, + } + + active_issues.push(issue); + } + } + + active_issues.sort_by(compare_issue_candidates); + + Ok(RecoveredRuntimeState { active_issues }) +} + +fn record_recovered_activity_lease( + project: &ServiceConfig, + state_store: &StateStore, + issue: &TrackerIssue, + marker: &RunActivityMarker, +) -> crate::prelude::Result<()> { + state_store.record_run_attempt( + marker.run_id(), + &issue.id, + marker.attempt_number(), + "running", + )?; + state_store.upsert_lease( + project.service_id(), + &issue.id, + marker.run_id(), + &issue.state.name, + )?; + + Ok(()) +} + +fn issue_has_recovered_service_ownership( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, +) -> crate::prelude::Result +where + T: IssueTracker, +{ + tracker::issue_has_label_with_server_confirmation( + tracker, + issue, + &tracker::automation_active_label(service_id), + ) +} + +fn append_recoverable_tracker_issue( + tracker: &T, + project: &ServiceConfig, + issue_identifier: &str, + known_identifiers: &mut BTreeSet, + issues: &mut Vec, +) -> crate::prelude::Result<()> +where + T: IssueTracker, +{ + let canonical_identifier = issue_identifier.to_ascii_uppercase(); + + if known_identifiers.contains(&canonical_identifier) { + return Ok(()); + } + + let Some(issue) = tracker.get_issue_by_identifier(issue_identifier)? else { + return Ok(()); + }; + + if !issue_has_recovered_service_ownership(tracker, &issue, project.service_id())? { + tracing::warn!( + issue = issue.identifier, + active_label = tracker::automation_active_label(project.service_id()), + labels_complete = issue.labels_complete, + "Skipping retained worktree recovery because the tracker issue is not explicitly owned by this service." + ); + + return Ok(()); + } + + known_identifiers.insert(issue.identifier.to_ascii_uppercase()); + issues.push(issue); + + Ok(()) +} + +fn recoverable_worktree_identifiers(worktree_root: &Path) -> crate::prelude::Result> { + if !worktree_root.exists() { + return Ok(Vec::new()); + } + + let mut issue_identifiers = fs::read_dir(worktree_root)? + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|file_type| file_type.is_dir()) + .and_then(|_| entry.file_name().into_string().ok()) + }) + .filter(|name| commit_message::looks_like_issue_identifier(name)) + .collect::>(); + + issue_identifiers.sort(); + issue_identifiers.dedup(); + + Ok(issue_identifiers) +} + +fn worktree_activity_marker_is_fresh(marker: &RunActivityMarker, now_unix_epoch: i64) -> bool { + marker.process_id().is_some_and(process_is_alive) + && marker + .last_activity_unix_epoch() + .and_then(|last_activity| observed_idle_duration(last_activity, now_unix_epoch)) + .is_some_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT) +} + +fn process_is_alive(process_id: u32) -> bool { + let Ok(process_id) = pid_t::try_from(process_id) else { + return false; + }; + + if process_id <= 0 { + return false; + } + + // Use the kernel liveness probe directly so recovery does not depend on a shell + // builtin or `kill` binary being present on PATH. + match unsafe { libc::kill(process_id, 0) } { + 0 => true, + -1 => matches!(std::io::Error::last_os_error().raw_os_error(), Some(libc::EPERM)), + _ => false, + } +} + +fn hydrate_status_snapshot_state( + _project: &ServiceConfig, + _state_store: &StateStore, + _recovered_state: RecoveredRuntimeState, +) -> crate::prelude::Result<()> { + Ok(()) +} + +fn operator_run_status( + project: &ServiceConfig, + state_store: &StateStore, + run: ProjectRunStatus, + now_unix_epoch: i64, +) -> crate::prelude::Result { + let marker = load_operator_run_marker(&run)?; + let timing = operator_run_timing(state_store, run.run_id(), marker.as_ref(), now_unix_epoch)?; + let app_server_state = operator_run_app_server_state(&run, marker.as_ref()); + let protocol_summary = operator_run_protocol_summary(&run, marker.as_ref()); + let status = + operator_run_visible_status(run.status(), &app_server_state, &protocol_summary, &timing); + let (retry_kind, retry_ready_at_unix_epoch) = visible_operator_run_retry_schedule( + &status, + marker.as_ref().and_then(RunActivityMarker::retry_kind), + marker.as_ref().and_then(RunActivityMarker::retry_ready_at_unix_epoch), + now_unix_epoch, + ); + let (phase, mut wait_reason) = classify_operator_run_phase( + &status, + retry_kind.as_deref(), + retry_ready_at_unix_epoch, + now_unix_epoch, + ); + let current_operation = classify_operator_run_operation( + &phase, + marker.as_ref().and_then(RunActivityMarker::current_operation), + ); + let suspected_stall = + operator_run_is_suspected_stall(&phase, timing.last_progress_unix_epoch, now_unix_epoch); + let child_agent_activity = operator_run_child_agent_activity(marker.as_ref(), now_unix_epoch); + let protocol_activity = operator_run_protocol_activity( + marker.as_ref(), + &app_server_state, + child_agent_activity.as_ref(), + timing.protocol_idle_for_seconds, + matches!(status.as_str(), "starting" | "running"), + ); + + if wait_reason.is_none() && phase == "executing" { + wait_reason = protocol_activity + .as_ref() + .and_then(|summary| summary.waiting_reason.clone()) + .filter(|reason| reason != "turn_completed"); + } + + let account = marker.as_ref().and_then(RunActivityMarker::account).cloned(); + let mut accounts = marker + .as_ref() + .map(|marker| marker.accounts().to_vec()) + .unwrap_or_default(); + + if accounts.is_empty() + && let Some(account) = &account + { + accounts.push(account.clone()); + } + + let branch_name = run.branch_name().map(str::to_owned); + let worktree_path = run + .worktree_path() + .map(|path| relative_worktree_path_for_path(project, path)); + let issue_identifier = operator_run_issue_identifier_from_fields( + run.run_id(), + branch_name.as_deref(), + worktree_path.as_deref(), + ); + let execution_liveness = + operator_run_execution_liveness(&status, &timing, &app_server_state, &protocol_summary); + + Ok(OperatorRunStatus { + project_id: project.service_id().to_owned(), + run_id: run.run_id().to_owned(), + issue_id: run.issue_id().to_owned(), + issue_identifier, + title: None, + attempt_number: run.attempt_number(), + status, + attempt_status: run.status().to_owned(), + phase, + wait_reason, + current_operation, + thread_id: app_server_state.thread_id, + turn_id: app_server_state.turn_id, + thread_status: app_server_state.thread_status, + thread_active_flags: app_server_state.thread_active_flags, + interactive_requested: app_server_state.interactive_requested, + continuation_pending: app_server_state.continuation_pending, + active_lease: run.active_lease(), + queue_lease_state: operator_run_queue_lease_state(run.active_lease()), + execution_liveness, + updated_at: run.updated_at().to_owned(), + last_run_activity_at: format_optional_unix_timestamp(timing.last_run_activity_unix_epoch), + last_protocol_activity_at: format_optional_unix_timestamp( + timing.last_protocol_activity_unix_epoch, + ), + last_progress_at: format_optional_unix_timestamp(timing.last_progress_unix_epoch), + idle_for_seconds: timing.idle_for_seconds, + protocol_idle_for_seconds: timing.protocol_idle_for_seconds, + suspected_stall, + last_event_type: protocol_summary.last_event_type, + last_event_at: protocol_summary.last_event_at, + event_count: protocol_summary.event_count, + process_id: timing.process_id, + process_alive: timing.process_alive, + retry_kind, + next_retry_at: format_optional_unix_timestamp(retry_ready_at_unix_epoch), + effective_model: app_server_state.effective_model, + effective_model_provider: app_server_state.effective_model_provider, + effective_cwd: app_server_state.effective_cwd, + effective_approval_policy: app_server_state.effective_approval_policy, + effective_approvals_reviewer: app_server_state.effective_approvals_reviewer, + effective_sandbox_mode: app_server_state.effective_sandbox_mode, + child_agent_activity, + protocol_activity, + account, + accounts, + branch_name, + worktree_path, + }) +} + +fn load_operator_run_marker( + run: &ProjectRunStatus, +) -> crate::prelude::Result> { + Ok(run + .worktree_path() + .map(state::read_run_activity_marker_snapshot) + .transpose()? + .flatten() + .filter(|marker| { + marker.run_id() == run.run_id() && marker.attempt_number() == run.attempt_number() + })) +} + +fn operator_run_timing( + state_store: &StateStore, + run_id: &str, + marker: Option<&RunActivityMarker>, + now_unix_epoch: i64, +) -> crate::prelude::Result { + let process_id = marker.and_then(RunActivityMarker::process_id); + let last_run_activity_unix_epoch = max_optional_i64( + state_store.last_run_activity_unix_epoch(run_id)?, + marker.and_then(RunActivityMarker::last_activity_unix_epoch), + ); + let last_protocol_activity_unix_epoch = max_optional_i64( + state_store.last_protocol_activity_unix_epoch(run_id)?, + marker.and_then(RunActivityMarker::last_protocol_activity_unix_epoch), + ); + let last_progress_unix_epoch = max_optional_i64( + marker.and_then(RunActivityMarker::last_progress_unix_epoch), + last_protocol_activity_unix_epoch, + ); + + Ok(OperatorRunTiming { + process_alive: process_id.map(process_is_alive), + process_id, + last_run_activity_unix_epoch, + last_protocol_activity_unix_epoch, + last_progress_unix_epoch, + idle_for_seconds: idle_duration_seconds(last_run_activity_unix_epoch, now_unix_epoch), + protocol_idle_for_seconds: idle_duration_seconds( + last_protocol_activity_unix_epoch, + now_unix_epoch, + ), + }) +} + +fn operator_run_app_server_state( + run: &ProjectRunStatus, + marker: Option<&RunActivityMarker>, +) -> OperatorRunAppServerState { + let thread_active_flags = + marker.map(|marker| marker.thread_active_flags().to_vec()).unwrap_or_default(); + + OperatorRunAppServerState { + thread_id: run + .thread_id() + .or_else(|| marker.and_then(RunActivityMarker::thread_id)) + .map(str::to_owned), + turn_id: run + .turn_id() + .or_else(|| marker.and_then(RunActivityMarker::turn_id)) + .map(str::to_owned), + thread_status: marker.and_then(RunActivityMarker::thread_status).map(str::to_owned), + interactive_requested: thread_active_flags + .iter() + .any(|flag| matches!(flag.as_str(), "waitingOnApproval" | "waitingOnUserInput")), + continuation_pending: run.status() == CONTINUATION_PENDING_RUN_STATUS, + effective_model: marker.and_then(RunActivityMarker::effective_model).map(str::to_owned), + effective_model_provider: marker + .and_then(RunActivityMarker::effective_model_provider) + .map(str::to_owned), + effective_cwd: marker.and_then(RunActivityMarker::effective_cwd).map(str::to_owned), + effective_approval_policy: marker + .and_then(RunActivityMarker::effective_approval_policy) + .map(str::to_owned), + effective_approvals_reviewer: marker + .and_then(RunActivityMarker::effective_approvals_reviewer) + .map(str::to_owned), + effective_sandbox_mode: marker + .and_then(RunActivityMarker::effective_sandbox_mode) + .map(str::to_owned), + thread_active_flags, + } +} + +fn operator_run_protocol_summary( + run: &ProjectRunStatus, + marker: Option<&RunActivityMarker>, +) -> OperatorRunProtocolSummary { + let use_marker_protocol_summary = + run.event_count() == 0 && run.last_event_type().is_none() && run.last_event_at().is_none(); + + if use_marker_protocol_summary { + return OperatorRunProtocolSummary { + last_event_type: marker.and_then(RunActivityMarker::last_event_type).map(str::to_owned), + last_event_at: marker + .and_then(RunActivityMarker::last_protocol_activity_unix_epoch) + .and_then(|unix_epoch| format_optional_unix_timestamp(Some(unix_epoch))), + event_count: marker.map_or(0, RunActivityMarker::event_count), + }; + } + + OperatorRunProtocolSummary { + last_event_type: run.last_event_type().map(str::to_owned), + last_event_at: run.last_event_at().map(str::to_owned), + event_count: run.event_count(), + } +} + +fn operator_run_visible_status( + attempt_status: &str, + app_server_state: &OperatorRunAppServerState, + protocol_summary: &OperatorRunProtocolSummary, + timing: &OperatorRunTiming, +) -> String { + if attempt_status == "starting" + && operator_run_has_app_server_execution_evidence( + app_server_state, + protocol_summary, + timing, + ) + { + return String::from("running"); + } + + attempt_status.to_owned() +} + +fn operator_run_has_app_server_execution_evidence( + app_server_state: &OperatorRunAppServerState, + protocol_summary: &OperatorRunProtocolSummary, + timing: &OperatorRunTiming, +) -> bool { + matches!(app_server_state.thread_status.as_deref(), Some("active")) + || !app_server_state.thread_active_flags.is_empty() + || app_server_state.effective_model.is_some() + || app_server_state.effective_model_provider.is_some() + || protocol_summary.event_count > 0 + || protocol_summary.last_event_type.is_some() + || timing.protocol_idle_for_seconds.is_some_and(|idle_for| { + u64::try_from(idle_for) + .is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + }) +} + +fn operator_run_queue_lease_state(active_lease: bool) -> String { + if active_lease { + String::from("held") + } else { + String::from("not_held") + } +} + +fn operator_run_execution_liveness( + status: &str, + timing: &OperatorRunTiming, + app_server_state: &OperatorRunAppServerState, + protocol_summary: &OperatorRunProtocolSummary, +) -> String { + if !matches!(status, "starting" | "running") { + return String::from("not_running"); + } + if timing.process_alive == Some(true) { + return String::from("process_alive"); + } + if timing.process_alive == Some(false) { + return String::from("process_stopped"); + } + if matches!(app_server_state.thread_status.as_deref(), Some("active")) + || !app_server_state.thread_active_flags.is_empty() + { + return String::from("thread_active"); + } + if operator_run_has_app_server_execution_evidence(app_server_state, protocol_summary, timing) { + return String::from("protocol_observed"); + } + + String::from("not_captured") +} + +fn operator_run_child_agent_activity( + marker: Option<&RunActivityMarker>, + now_unix_epoch: i64, +) -> Option { + let mut summary = marker.and_then(RunActivityMarker::child_agent_activity).cloned()?; + + summary.current_elapsed_seconds = + summary.current_started_unix_epoch.and_then(|started_at| { + now_unix_epoch.checked_sub(started_at).filter(|elapsed| *elapsed >= 0) + }); + + if let (Some(current_bucket), Some(current_elapsed_seconds)) = + (summary.current_bucket.as_deref(), summary.current_elapsed_seconds) + && current_elapsed_seconds > 0 + { + if let Some(bucket) = summary.buckets.iter_mut().find(|bucket| bucket.name == current_bucket) { + bucket.wall_seconds = bucket.wall_seconds.saturating_add(current_elapsed_seconds); + } else { + summary.buckets.push(ChildAgentActivityBucket { + name: current_bucket.to_owned(), + wall_seconds: current_elapsed_seconds, + ..ChildAgentActivityBucket::default() + }); + } + } + + Some(summary) +} + +fn operator_run_protocol_activity( + marker: Option<&RunActivityMarker>, + app_server_state: &OperatorRunAppServerState, + child_agent_activity: Option<&ChildAgentActivitySummary>, + protocol_idle_for_seconds: Option, + is_running: bool, +) -> Option { + let mut summary = marker.and_then(RunActivityMarker::protocol_activity).cloned().unwrap_or_default(); + + if is_running && summary.waiting_reason.is_none() && app_server_state.interactive_requested { + summary.waiting_reason = Some(String::from("approval_or_user_input")); + } + if is_running + && summary.waiting_reason.is_none() + && let Some(child_agent_activity) = child_agent_activity + && let Some(current_bucket) = child_agent_activity.current_bucket.as_deref() + { + summary.waiting_reason = Some(protocol_wait_reason_from_child_bucket(current_bucket)); + } + if is_running + && summary.waiting_reason.is_none() + && protocol_idle_for_seconds.is_some_and(|idle_for| { + u64::try_from(idle_for).is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + }) { + summary.waiting_reason = Some(String::from("protocol_idleness")); + } + if summary.turn_status.is_none() + && summary.waiting_reason.is_none() + && summary.rate_limit_status.is_none() + && summary.recent_events.is_empty() + { + return None; + } + + Some(summary) +} + +fn protocol_wait_reason_from_child_bucket(current_bucket: &str) -> String { + match current_bucket { + "Model" => String::from("model_execution"), + "Protocol" => String::from("protocol_activity"), + _ => String::from("tool_execution"), + } +} + +fn idle_duration_seconds( + last_activity_unix_epoch: Option, + now_unix_epoch: i64, +) -> Option { + last_activity_unix_epoch + .and_then(|last_activity| now_unix_epoch.checked_sub(last_activity)) + .filter(|idle_for| *idle_for >= 0) +} + +fn max_optional_i64(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.max(right)), + (Some(value), None) | (None, Some(value)) => Some(value), + (None, None) => None, + } +} + +fn format_optional_unix_timestamp(unix_epoch: Option) -> Option { + unix_epoch.and_then(|unix_epoch| { + OffsetDateTime::from_unix_timestamp(unix_epoch) + .ok() + .and_then(|timestamp| timestamp.format(&Rfc3339).ok()) + }) +} + +fn classify_operator_run_operation(phase: &str, marker_current_operation: Option<&str>) -> String { + match phase { + "retry_backoff" | "waiting_continuation" => { + String::from(RUN_OPERATION_WAITING_EXTERNAL) + }, + "completed" | "failed" => String::from(RUN_OPERATION_IDLE), + "stalled" => marker_current_operation + .map(str::to_owned) + .unwrap_or_else(|| String::from(RUN_OPERATION_IDLE)), + "executing" => marker_current_operation + .map(str::to_owned) + .unwrap_or_else(|| String::from(RUN_OPERATION_AGENT_RUN)), + _ => marker_current_operation + .map(str::to_owned) + .unwrap_or_else(|| String::from(RUN_OPERATION_IDLE)), + } +} + +fn operator_run_is_suspected_stall( + phase: &str, + last_progress_unix_epoch: Option, + now_unix_epoch: i64, +) -> bool { + if phase != "executing" { + return false; + } + + last_progress_unix_epoch + .and_then(|last_progress| observed_idle_duration(last_progress, now_unix_epoch)) + .is_some_and(|idle_for| { + idle_for >= suspected_operator_run_stall_threshold() + && idle_for < ACTIVE_RUN_IDLE_TIMEOUT + }) +} + +fn suspected_operator_run_stall_threshold() -> Duration { + Duration::from_secs((ACTIVE_RUN_IDLE_TIMEOUT.as_secs() / 2).max(1)) +} + +fn visible_operator_run_retry_schedule( + status: &str, + retry_kind: Option<&str>, + retry_ready_at_unix_epoch: Option, + now_unix_epoch: i64, +) -> (Option, Option) { + let Some(retry_ready_at_unix_epoch) = retry_ready_at_unix_epoch else { + return (None, None); + }; + + if matches!(status, "starting" | "running") || retry_ready_at_unix_epoch <= now_unix_epoch { + return (None, None); + } + + (retry_kind.map(str::to_owned), Some(retry_ready_at_unix_epoch)) +} + +fn classify_operator_run_phase( + status: &str, + retry_kind: Option<&str>, + retry_ready_at_unix_epoch: Option, + now_unix_epoch: i64, +) -> (String, Option) { + if status == "stalled" { + return (String::from("stalled"), Some(String::from("app_server_idle_timeout"))); + } + + if let Some(retry_ready_at_unix_epoch) = retry_ready_at_unix_epoch + && retry_ready_at_unix_epoch > now_unix_epoch + { + return ( + String::from("retry_backoff"), + Some(match retry_kind { + Some("continuation") => String::from("continuation_retry"), + Some("failure") => String::from("failure_retry"), + Some(other) => other.to_owned(), + None => String::from("scheduled_retry"), + }), + ); + } + + match status { + "starting" | "running" => (String::from("executing"), None), + CONTINUATION_PENDING_RUN_STATUS => { + (String::from("waiting_continuation"), Some(String::from("turn_boundary"))) + }, + "succeeded" => (String::from("completed"), None), + "failed" | "interrupted" | TERMINAL_GUARDED_RUN_STATUS => (String::from("failed"), None), + other => (other.to_owned(), None), + } +} + +fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { + let session_history_attempt_count = snapshot + .history_lanes + .iter() + .map(|lane| lane.attempt_count) + .sum::(); + let hides_running_lanes = session_history_attempt_count < snapshot.recent_runs.len(); + let (running_inline_claims, non_running_queued_candidates): (Vec<_>, Vec<_>) = snapshot + .queued_candidates + .iter() + .partition(|queued_issue| queue_claim_belongs_to_active_run(queued_issue, snapshot)); + let (stale_closed_queue_labels, backlog_candidates) = + rendered_backlog_queue_groups(non_running_queued_candidates); + let recovery_worktrees = rendered_recovery_worktrees(snapshot); + let hides_owned_worktrees = recovery_worktrees.len() < snapshot.worktrees.len(); + let mut output = String::new(); + + output.push_str(&format!("Project: {}\n", snapshot.project_id)); + output.push_str(&format!("Warnings: {}\n", snapshot.warnings.len())); + + if !snapshot.warnings.is_empty() { + output.push_str(&format!("Warning details: {}\n", snapshot.warnings.join(", "))); + } + + output.push_str(&format!("Running lanes: {}\n", snapshot.active_runs.len())); + output.push_str(&format!( + "Run ledger shown: {} issue lanes from {} history attempts{}\n", + snapshot.history_lanes.len(), + session_history_attempt_count, + if hides_running_lanes { " (running lanes inline)" } else { "" }, + )); + output.push_str(&format!("Backlog: {}\n", backlog_candidates.len())); + output.push_str(&format!( + "Active queue echoes: {}\n", + running_inline_claims.len() + )); + output.push_str(&format!( + "Stale closed queue labels: {}\n", + stale_closed_queue_labels.len() + )); + output.push_str(&format!("Recovery worktrees: {}\n", recovery_worktrees.len())); + output.push_str(&format!("Post-review lanes: {}\n", snapshot.post_review_lanes.len())); + output.push_str("\nRunning Lanes\n"); + + if snapshot.active_runs.is_empty() { + output.push_str("- none\n"); + } else { + for run in &snapshot.active_runs { + append_rendered_run(&mut output, run); + } + } + + output.push_str("\nRun Ledger\n"); + + if snapshot.history_lanes.is_empty() { + if hides_running_lanes { + output.push_str("- none (running lanes are shown above)\n"); + } else { + output.push_str("- none\n"); + } + } else { + for lane in &snapshot.history_lanes { + append_rendered_history_lane(&mut output, lane); + } + } + + append_rendered_queued_issue_section(&mut output, "Backlog", &backlog_candidates, snapshot, false); + append_rendered_queued_issue_section( + &mut output, + "Active Queue Echoes", + &running_inline_claims, + snapshot, + true, + ); + append_rendered_queued_issue_section( + &mut output, + "Stale Closed Queue Labels", + &stale_closed_queue_labels, + snapshot, + false, + ); + + output.push_str("\nRecovery Worktrees\n"); + + append_rendered_recovery_worktrees(&mut output, &recovery_worktrees, hides_owned_worktrees); + + output.push_str("\nPost-Review Lanes\n"); + + if snapshot.post_review_lanes.is_empty() { + output.push_str("- none\n"); + } else { + for lane in &snapshot.post_review_lanes { + output.push_str(&format!( + "- issue_id: {}\n issue: {}\n state: {}\n classification: {}\n reason: {}\n branch: {}\n worktree_path: {}\n pr_url: {}\n pr_state: {}\n review_decision: {}\n mergeable: {}\n check_state: {}\n unresolved_review_threads: {}\n", + lane.issue_id, + lane.issue_identifier, + lane.issue_state, + lane.classification, + lane.reason, + lane.branch_name, + lane.worktree_path, + lane.pr_url.as_deref().unwrap_or("none"), + lane.pr_state.as_deref().unwrap_or("none"), + lane.review_decision.as_deref().unwrap_or("none"), + lane.mergeable.as_deref().unwrap_or("none"), + lane.check_state.as_deref().unwrap_or("none"), + lane + .unresolved_review_threads + .map_or_else(|| String::from("none"), |value| value.to_string()) + )); + } + } + + output +} + +fn rendered_backlog_queue_groups( + queued_candidates: Vec<&OperatorQueuedIssueStatus>, +) -> (Vec<&OperatorQueuedIssueStatus>, Vec<&OperatorQueuedIssueStatus>) { + let (stale_closed_queue_labels, non_closed_queue_candidates): (Vec<_>, Vec<_>) = + queued_candidates.into_iter().partition(|queued_issue| queued_issue.classification == "closed"); + let backlog_candidates = non_closed_queue_candidates + .into_iter() + .filter(|queued_issue| queued_candidate_counts_as_waiting_intake(queued_issue)) + .collect::>(); + + (stale_closed_queue_labels, backlog_candidates) +} + +fn rendered_recovery_worktrees( + snapshot: &OperatorStatusSnapshot, +) -> Vec<(&str, &OperatorWorktreeStatus)> { + let mut rendered_worktrees = snapshot + .worktrees + .iter() + .map(|worktree| (rendered_worktree_role(worktree, snapshot), worktree)) + .filter(|(role, _)| rendered_worktree_role_rank(role) > 0) + .collect::>(); + + rendered_worktrees.sort_by(|(left_role, left), (right_role, right)| { + rendered_worktree_role_rank(left_role) + .cmp(&rendered_worktree_role_rank(right_role)) + .then_with(|| left.issue_id.cmp(&right.issue_id)) + .then_with(|| left.branch_name.cmp(&right.branch_name)) + .then_with(|| left.worktree_path.cmp(&right.worktree_path)) + }); + + rendered_worktrees +} + +fn append_rendered_recovery_worktrees( + output: &mut String, + rendered_worktrees: &[(&str, &OperatorWorktreeStatus)], + hides_owned_worktrees: bool, +) { + if rendered_worktrees.is_empty() { + if hides_owned_worktrees { + output.push_str("- none (owned worktrees are shown in their lane sections above)\n"); + } else { + output.push_str("- none\n"); + } + + return; + } + + for (role, worktree) in rendered_worktrees { + output.push_str(&format!( + "- issue_id: {}\n issue: {}\n state: {}\n role: {}\n reason: {}\n branch: {}\n worktree_path: {}\n", + worktree.issue_id, + worktree.issue_identifier.as_deref().unwrap_or("none"), + worktree.issue_state.as_deref().unwrap_or("unknown"), + role, + worktree.ownership_reason, + worktree.branch_name, + worktree.worktree_path + )); + } +} + +fn append_rendered_queued_issue_section( + output: &mut String, + title: &str, + queued_issues: &[&OperatorQueuedIssueStatus], + snapshot: &OperatorStatusSnapshot, + show_running_owner: bool, +) { + output.push_str(&format!("\n{title}\n")); + + if queued_issues.is_empty() { + output.push_str("- none\n"); + + if title == "Backlog" { + output.push_str(&format!( + " {}\n", + format_status_no_eligible_issue_hint(&snapshot.project_id) + )); + } + + return; + } + + for queued_issue in queued_issues { + let running_owner = show_running_owner + .then(|| active_run_id_for_queue_candidate(queued_issue, snapshot)) + .flatten(); + + append_rendered_queued_issue(output, queued_issue, running_owner); + } +} + +fn operator_history_lanes( + active_runs: &[OperatorRunStatus], + recent_runs: &[OperatorRunStatus], +) -> Vec { + let active_run_ids = active_runs + .iter() + .map(|run| run.run_id.as_str()) + .collect::>(); + let active_issue_ids = active_runs + .iter() + .map(|run| run.issue_id.as_str()) + .collect::>(); + let mut lane_indexes = HashMap::new(); + let mut lanes = Vec::new(); + + for run in recent_runs { + if active_run_ids.contains(run.run_id.as_str()) + || active_issue_ids.contains(run.issue_id.as_str()) + { + continue; + } + + let group_key = operator_run_group_key(run); + + if let Some(index) = lane_indexes.get(&group_key) { + let lane: &mut OperatorHistoryLaneStatus = &mut lanes[*index]; + + lane.attempt_count += 1; + + if run.attempt_number > lane.latest_run.attempt_number { + lane.latest_run = run.clone(); + } + + hydrate_history_lane_from_run(lane, run); + + lane.attempts.push(run.clone()); + + continue; + } + + lane_indexes.insert(group_key, lanes.len()); + lanes.push(OperatorHistoryLaneStatus { + project_id: run.project_id.clone(), + issue_id: run.issue_id.clone(), + issue_identifier: run.issue_identifier.clone(), + title: run.title.clone(), + issue_key: operator_run_issue_key(run), + attempt_count: 1, + ledger_outcome: not_loaded_history_ledger_outcome(), + latest_run: run.clone(), + attempts: vec![run.clone()], + }); + } + + lanes +} + +fn hydrate_history_lane_from_run(lane: &mut OperatorHistoryLaneStatus, run: &OperatorRunStatus) { + if lane.issue_identifier.is_none() + && let Some(issue_identifier) = run + .issue_identifier + .as_ref() + .filter(|value| !value.trim().is_empty()) + { + lane.issue_identifier = Some(issue_identifier.clone()); + lane.issue_key = issue_identifier.clone(); + } + if lane.title.is_none() { + lane.title = run.title.clone(); + } +} + +fn operator_run_group_key(run: &OperatorRunStatus) -> String { + let issue_id = run.issue_id.trim(); + + if !issue_id.is_empty() && !issue_id.eq_ignore_ascii_case("unknown") { + return issue_id.to_ascii_uppercase(); + } + + operator_run_issue_key(run) +} + +fn operator_run_issue_key(run: &OperatorRunStatus) -> String { + if let Some(issue_identifier) = run.issue_identifier.as_ref().filter(|value| { + !value.trim().is_empty() && !value.eq_ignore_ascii_case("unknown") + }) { + return issue_identifier.clone(); + } + if let Some(issue_identifier) = operator_run_issue_identifier_from_fields( + &run.run_id, + run.branch_name.as_deref(), + run.worktree_path.as_deref(), + ) { + return issue_identifier; + } + + let issue_id = run.issue_id.trim(); + + if issue_id.is_empty() { + String::from("unknown") + } else { + issue_id.to_owned() + } +} + +fn operator_run_issue_identifier_from_fields( + run_id: &str, + branch_name: Option<&str>, + worktree_path: Option<&str>, +) -> Option { + if let Some(issue_identifier) = issue_identifier_from_run_id(run_id) { + return Some(issue_identifier); + } + + for value in [branch_name, worktree_path] { + if let Some(issue_identifier) = value.and_then(issue_identifier_in_text) { + return Some(issue_identifier); + } + } + + None +} + +fn issue_identifier_from_run_id(run_id: &str) -> Option { + if let Some((candidate, _attempt_suffix)) = run_id.split_once("-attempt-") { + return issue_identifier_in_text(candidate); + } + if let Some(candidate) = run_id.strip_prefix("recovered-") { + return issue_identifier_in_text(candidate); + } + + None +} + +fn issue_identifier_in_text(value: &str) -> Option { + let bytes = value.as_bytes(); + + for index in 0..bytes.len() { + if !bytes[index].is_ascii_alphabetic() { + continue; + } + + let mut prefix_end = index + 1; + + while prefix_end < bytes.len() && bytes[prefix_end].is_ascii_alphanumeric() { + prefix_end += 1; + } + + if prefix_end >= bytes.len() || bytes[prefix_end] != b'-' { + continue; + } + + let mut digit_end = prefix_end + 1; + + while digit_end < bytes.len() && bytes[digit_end].is_ascii_digit() { + digit_end += 1; + } + + if digit_end > prefix_end + 1 { + return Some(value[index..digit_end].to_ascii_uppercase()); + } + } + + None +} + +fn queue_claim_belongs_to_active_run( + queued_issue: &OperatorQueuedIssueStatus, + snapshot: &OperatorStatusSnapshot, +) -> bool { + queued_issue.classification == "claimed" + && active_run_id_for_queue_candidate(queued_issue, snapshot).is_some() +} + +fn active_run_id_for_queue_candidate<'a>( + queued_issue: &OperatorQueuedIssueStatus, + snapshot: &'a OperatorStatusSnapshot, +) -> Option<&'a str> { + snapshot + .active_runs + .iter() + .find(|run| run.issue_id == queued_issue.issue_id) + .map(|run| run.run_id.as_str()) +} + +fn append_rendered_queued_issue( + output: &mut String, + queued_issue: &OperatorQueuedIssueStatus, + active_run_id: Option<&str>, +) { + let priority = queued_issue + .priority + .map_or_else(|| String::from("none"), |value| value.to_string()); + let blockers = if queued_issue.blocker_identifiers.is_empty() { + String::from("none") + } else { + queued_issue.blocker_identifiers.join(", ") + }; + let running_owner = active_run_id.unwrap_or("none"); + + output.push_str(&format!( + "- issue_id: {}\n issue: {}\n title: {}\n state: {}\n priority: {}\n created_at: {}\n classification: {}\n reason: {}\n running_owner_run: {}\n blockers: {}\n", + queued_issue.issue_id, + queued_issue.issue_identifier, + queued_issue.title, + queued_issue.state, + priority, + queued_issue.created_at, + queued_issue.classification, + queued_issue.reason, + running_owner, + blockers, + )); + + if let Some(attention) = &queued_issue.attention { + output.push_str(&format!( + " attention: {}\n attention_run: {}\n attention_attempt: {}\n attention_operation: {}\n attention_thread: {}\n attention_cause: {}\n attention_next_action: {}\n attention_auto_retry: {}\n attention_retry_budget_attempts: {}\n attention_worktree: {}\n attention_last_activity: {}\n", + attention.summary, + attention.run_id.as_deref().unwrap_or("none"), + attention + .attempt_number + .map_or_else(|| String::from("none"), |value| value.to_string()), + attention.current_operation.as_deref().unwrap_or("none"), + attention.thread_status.as_deref().unwrap_or("none"), + attention.attention_error_class.as_deref().unwrap_or("none"), + attention.attention_next_action.as_deref().unwrap_or("none"), + attention.auto_retry_blocked_reason.as_deref().unwrap_or("none"), + attention + .retry_budget_attempt_count + .map_or_else(|| String::from("none"), |value| value.to_string()), + attention.worktree_path.as_deref().unwrap_or("none"), + attention.last_activity_at.as_deref().unwrap_or("none"), + )); + } +} + +fn rendered_worktree_role<'a>( + worktree: &'a OperatorWorktreeStatus, + snapshot: &'a OperatorStatusSnapshot, +) -> &'a str { + if !worktree.ownership.trim().is_empty() { + return worktree.ownership.as_str(); + } + if snapshot.active_runs.iter().any(|run| { + run.worktree_path.as_deref() == Some(worktree.worktree_path.as_str()) + || run.branch_name.as_deref() == Some(worktree.branch_name.as_str()) + || run.issue_id == worktree.issue_id + }) { + return "active_lane"; + } + if snapshot.post_review_lanes.iter().any(|lane| { + lane.worktree_path == worktree.worktree_path + || lane.branch_name == worktree.branch_name + || lane.issue_id == worktree.issue_id + || lane.issue_identifier == worktree.issue_id + }) { + return "post_review_lane"; + } + if snapshot.queued_candidates.iter().any(|candidate| { + candidate.reason == "issue_needs_attention" + && (candidate + .attention + .as_ref() + .and_then(|attention| attention.worktree_path.as_deref()) + == Some(worktree.worktree_path.as_str()) + || candidate.issue_id == worktree.issue_id + || candidate.issue_identifier == worktree.issue_id) + }) { + return "blocked_queue_issue"; + } + + "orphaned_local_worktree" +} + +fn rendered_worktree_role_rank(role: &str) -> u8 { + match role { + "active_lane" | "running_lane" | "blocked_queue_issue" | "queued_attention" => 0, + "post_review_lane" => 1, + _ => 2, + } +} + +fn render_child_agent_activity_summary( + summary: Option<&ChildAgentActivitySummary>, +) -> String { + let Some(summary) = summary else { + return String::from("none"); + }; + let current = match (&summary.current_bucket, summary.current_elapsed_seconds) { + (Some(bucket), Some(seconds)) => format!("{bucket} {}", format_seconds_compact(seconds)), + (Some(bucket), None) => bucket.clone(), + (None, _) => String::from("none"), + }; + let buckets = render_child_agent_bucket_distribution(&summary.buckets); + + format!( + "current={current}; wall={}; buckets={}; tool_calls={}", + format_seconds_compact(summary.wall_seconds), + buckets, + summary.tool_call_count + ) +} + +fn render_protocol_activity_summary(summary: Option<&ProtocolActivitySummary>) -> String { + let Some(summary) = summary else { + return String::from("none"); + }; + let turn = summary.turn_status.as_deref().unwrap_or("none"); + let wait = summary.waiting_reason.as_deref().unwrap_or("none"); + let rate_limit = summary.rate_limit_status.as_deref().unwrap_or("none"); + let recent = if summary.recent_events.is_empty() { + String::from("none") + } else { + summary + .recent_events + .iter() + .rev() + .take(5) + .map(|event| { + event + .detail + .as_ref() + .map_or_else(|| event.event_type.clone(), |detail| { + format!("{}:{detail}", event.event_type) + }) + }) + .collect::>() + .join(", ") + }; + + format!("turn={turn}; waiting={wait}; rate_limit={rate_limit}; recent={recent}") +} + +fn render_account_summary(summary: Option<&CodexAccountActivitySummary>) -> String { + let Some(summary) = summary else { + return String::from("none"); + }; + let plan = summary.plan_type.as_deref().unwrap_or("unknown"); + let reached = summary.rate_limit_reached_type.as_deref().unwrap_or("none"); + let credits = render_codex_account_credits(summary); + let token_status = render_codex_account_token_status(&summary.refresh_status); + let primary = render_codex_account_window( + summary.primary_window_seconds, + summary.primary_remaining_percent, + summary.primary_resets_at_unix_epoch, + ); + let secondary = render_codex_account_window( + summary.secondary_window_seconds, + summary.secondary_remaining_percent, + summary.secondary_resets_at_unix_epoch, + ); + + format!( + "account={}; plan={plan}; status={}; token={token_status}; primary={primary}; secondary={secondary}; credits={credits}; reached={reached}", + summary.account_fingerprint, + summary.status, + ) +} + +fn render_accounts_summary(accounts: &[CodexAccountActivitySummary]) -> String { + if accounts.is_empty() { + return String::from("none"); + } + + accounts + .iter() + .map(|summary| render_account_summary(Some(summary))) + .collect::>() + .join(" | ") +} + +fn render_codex_account_window( + window_seconds: Option, + remaining_percent: Option, + resets_at_unix_epoch: Option, +) -> String { + let label = window_seconds.map(codex_window_label).unwrap_or_else(|| String::from("window")); + let remaining = remaining_percent.map_or_else(|| String::from("unknown"), |value| format!("{value}%")); + let reset = format_optional_unix_timestamp(resets_at_unix_epoch).unwrap_or_else(|| String::from("unknown")); + + format!("{label} remaining={remaining} reset={reset}") +} + +fn render_codex_account_credits(summary: &CodexAccountActivitySummary) -> String { + if summary.credits_unlimited == Some(true) { + return String::from("unlimited"); + } + + match (summary.credits_has_credits, summary.credits_balance.as_deref()) { + (Some(false), Some(balance)) => format!("depleted balance={balance}"), + (Some(false), None) => String::from("depleted"), + (_, Some(balance)) => format!("balance={balance}"), + (Some(true), None) => String::from("available"), + (None, None) => String::from("unknown"), + } +} + +fn render_codex_account_token_status(refresh_status: &str) -> &'static str { + match refresh_status { + "not_needed" | "none" => "ok", + "succeeded" | "refreshed" => "refreshed", + "failed" => "refresh_failed", + _ => "unknown", + } +} + +fn codex_window_label(window_seconds: i64) -> String { + match window_seconds { + 18_000 => String::from("5h"), + 604_800 => String::from("7d"), + seconds => format_seconds_compact(seconds), + } +} + +fn render_child_agent_context_pressure( + summary: Option<&ChildAgentActivitySummary>, +) -> String { + let Some(summary) = summary else { + return String::from("none"); + }; + let current_input = summary + .input_tokens_current + .map(format_count_compact) + .unwrap_or_else(|| String::from("none")); + let max_input = + summary.input_tokens_max.map(format_count_compact).unwrap_or_else(|| String::from("none")); + let max_input_relation = match (summary.input_tokens_current, summary.input_tokens_max) { + (Some(current), Some(max)) if current == max => " (same as current)", + _ => "", + }; + let largest_output = summary + .largest_tool_output_bytes + .map(format_bytes_compact) + .unwrap_or_else(|| String::from("none")); + let largest_tool = summary.largest_tool_output_tool.as_deref().unwrap_or("none"); + let warnings = if summary.large_output_warnings.is_empty() { + String::from("none") + } else { + summary.large_output_warnings.join(" | ") + }; + + format!( + "input=current_window {current_input}, peak_window {max_input}{max_input_relation}, cumulative_input {}; output_tokens={}; largest_output={largest_output} by {largest_tool}; warnings={warnings}", + format_count_compact(summary.input_tokens_cumulative), + format_count_compact(summary.output_tokens_cumulative) + ) +} + +fn render_child_agent_bucket_distribution( + buckets: &[ChildAgentActivityBucket], +) -> String { + if buckets.is_empty() { + return String::from("none"); + } + + let mut buckets = buckets.iter().collect::>(); + + buckets.sort_by(|left, right| { + right + .wall_seconds + .cmp(&left.wall_seconds) + .then_with(|| right.event_count.cmp(&left.event_count)) + .then_with(|| left.name.cmp(&right.name)) + }); + + buckets + .into_iter() + .take(5) + .map(|bucket| format!("{} {}", bucket.name, format_seconds_compact(bucket.wall_seconds))) + .collect::>() + .join(", ") +} + +fn format_seconds_compact(seconds: i64) -> String { + if seconds >= 3_600 { + return format!("{}h{}m", seconds / 3_600, (seconds % 3_600) / 60); + } + if seconds >= 60 { + return format!("{}m{}s", seconds / 60, seconds % 60); + } + + format!("{seconds}s") +} + +fn format_count_compact(count: i64) -> String { + if count >= 1_000_000 { + return format!("{:.2}M", count as f64 / 1_000_000.0); + } + if count >= 1_000 { + return format!("{:.1}k", count as f64 / 1_000.0); + } + + count.to_string() +} + +fn format_bytes_compact(bytes: i64) -> String { + if bytes >= 1_048_576 { + return format!("{:.1}MiB", bytes as f64 / 1_048_576.0); + } + if bytes >= 1_024 { + return format!("{:.1}KiB", bytes as f64 / 1_024.0); + } + + format!("{bytes}B") +} + +fn append_rendered_history_lane(output: &mut String, lane: &OperatorHistoryLaneStatus) { + output.push_str(&format!( + "- issue: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempts: {}\n ledger_status: {}\n outcome: {}\n", + lane.issue_key, + lane.project_id, + lane.issue_id, + lane.issue_identifier.as_deref().unwrap_or("none"), + lane.title.as_deref().unwrap_or("none"), + lane.attempt_count, + lane.ledger_outcome.ledger_status, + lane.ledger_outcome.final_outcome + )); + + append_rendered_history_ledger_outcome(output, &lane.ledger_outcome); + + if history_ledger_outcome_has_records(&lane.ledger_outcome) { + output.push_str(&format!( + " local_attempts: {}\n latest_run_id: {}\n", + lane.attempt_count, lane.latest_run.run_id + )); + } else { + append_rendered_run(output, &lane.latest_run); + } + if lane.attempts.len() <= 1 { + return; + } + + output.push_str(" attempt_timeline:\n"); + + for attempt in &lane.attempts { + output.push_str(&format!( + " - run_id: {} attempt: {} status: {} phase: {} updated_at: {}\n", + attempt.run_id, + attempt.attempt_number, + attempt.status, + attempt.phase, + attempt.updated_at + )); + } +} + +fn append_rendered_history_ledger_outcome( + output: &mut String, + outcome: &OperatorHistoryLedgerOutcome, +) { + append_rendered_history_field(output, "event_type", outcome.final_event_type.as_deref()); + append_rendered_history_field(output, "event_at", outcome.final_event_at.as_deref()); + append_rendered_history_field(output, "summary", outcome.summary.as_deref()); + append_rendered_history_field(output, "pr_url", outcome.pr_url.as_deref()); + append_rendered_history_field(output, "commit_sha", outcome.commit_sha.as_deref()); + append_rendered_history_field(output, "branch", outcome.branch.as_deref()); + append_rendered_history_field(output, "closeout_status", outcome.closeout_status.as_deref()); + append_rendered_history_field( + output, + "needs_attention_reason", + outcome.needs_attention_reason.as_deref(), + ); + append_rendered_history_field( + output, + "lifecycle_started_at", + outcome.lifecycle_started_at.as_deref(), + ); + append_rendered_history_field( + output, + "lifecycle_finished_at", + outcome.lifecycle_finished_at.as_deref(), + ); + + if let Some(elapsed) = outcome.lifecycle_elapsed_seconds { + output.push_str(&format!(" lifecycle_elapsed_seconds: {elapsed}\n")); + } + + output.push_str(&format!(" ledger_records: {}\n", outcome.record_count)); +} + +fn append_rendered_history_field(output: &mut String, label: &str, value: Option<&str>) { + if let Some(value) = value.filter(|value| !value.trim().is_empty()) { + output.push_str(&format!(" {label}: {value}\n")); + } +} + +fn history_ledger_outcome_has_records(outcome: &OperatorHistoryLedgerOutcome) -> bool { + matches!(outcome.ledger_status.as_str(), "present" | "partial") +} + +fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { + let (freshness_source, freshness_at) = operator_run_freshness(run); + let protocol_event = match (&run.last_event_type, &run.last_event_at) { + (Some(event_type), Some(timestamp)) => format!("{event_type} @ {timestamp}"), + (Some(event_type), None) => event_type.clone(), + (None, Some(timestamp)) => timestamp.clone(), + (None, None) => String::from("none"), + }; + let thread_id = run.thread_id.as_deref().unwrap_or("none"); + let turn_id = run.turn_id.as_deref().unwrap_or("none"); + let thread_status = run.thread_status.as_deref().unwrap_or("none"); + let thread_active_flags = if run.thread_active_flags.is_empty() { + String::from("none") + } else { + run.thread_active_flags.join(",") + }; + let idle_for_seconds = + run.idle_for_seconds.map_or_else(|| String::from("none"), |value| value.to_string()); + let protocol_idle_for_seconds = run + .protocol_idle_for_seconds + .map_or_else(|| String::from("none"), |value| value.to_string()); + let branch_name = run.branch_name.as_deref().unwrap_or("none"); + let worktree_path = run.worktree_path.as_deref().unwrap_or("none"); + let queue_lease = operator_run_queue_lease_summary(run); + let child_agent_activity = + render_child_agent_activity_summary(run.child_agent_activity.as_ref()); + let context_pressure = render_child_agent_context_pressure(run.child_agent_activity.as_ref()); + let protocol_activity = render_protocol_activity_summary(run.protocol_activity.as_ref()); + let account = render_account_summary(run.account.as_ref()); + let accounts = render_accounts_summary(&run.accounts); + + output.push_str(&format!( + "- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n process_id: {}\n process_alive: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n", + run.run_id, + run.project_id, + run.issue_id, + run.issue_identifier.as_deref().unwrap_or("none"), + run.title.as_deref().unwrap_or("none"), + run.attempt_number, + run.status, + run.attempt_status, + run.phase, + run.wait_reason.as_deref().unwrap_or("none"), + run.current_operation, + if run.active_lease { "yes" } else { "no" }, + run.queue_lease_state, + queue_lease, + run.execution_liveness, + freshness_at, + freshness_source, + idle_for_seconds, + protocol_idle_for_seconds, + run.last_progress_at.as_deref().unwrap_or("none"), + protocol_event, + run.event_count, + account, + accounts, + child_agent_activity, + protocol_activity, + context_pressure, + thread_id, + turn_id, + thread_status, + thread_active_flags, + if run.interactive_requested { "yes" } else { "no" }, + if run.continuation_pending { "yes" } else { "no" }, + branch_name, + worktree_path, + run.updated_at, + run.last_run_activity_at.as_deref().unwrap_or("none"), + run.last_protocol_activity_at.as_deref().unwrap_or("none"), + run.last_progress_at.as_deref().unwrap_or("none"), + idle_for_seconds, + protocol_idle_for_seconds, + if run.suspected_stall { "yes" } else { "no" }, + run.process_id.map_or_else(|| String::from("none"), |value| value.to_string()), + run.process_alive.map_or_else( + || String::from("none"), + |value| if value { String::from("yes") } else { String::from("no") }, + ), + run.retry_kind.as_deref().unwrap_or("none"), + run.next_retry_at.as_deref().unwrap_or("none"), + run.effective_model.as_deref().unwrap_or("none"), + run.effective_model_provider.as_deref().unwrap_or("none"), + run.effective_cwd.as_deref().unwrap_or("none"), + run.effective_approval_policy.as_deref().unwrap_or("none"), + run.effective_approvals_reviewer.as_deref().unwrap_or("none"), + run.effective_sandbox_mode.as_deref().unwrap_or("none"), + protocol_event, + run.event_count + )); +} + +fn operator_run_queue_lease_summary(run: &OperatorRunStatus) -> String { + if run.active_lease { + return String::from("held"); + } + + match run.execution_liveness.as_str() { + "process_alive" => String::from("not_held (process_alive keeps lane visible)"), + "thread_active" => String::from("not_held (thread_active keeps lane visible)"), + "protocol_observed" => String::from("not_held (protocol_observed keeps lane visible)"), + "process_stopped" => String::from("not_held (process_stopped needs attention)"), + _ => String::from("not_held"), + } +} + +fn operator_run_freshness(run: &OperatorRunStatus) -> (&'static str, &str) { + if operator_run_counts_as_active(run) { + if let Some(timestamp) = run.last_run_activity_at.as_deref() { + return ("last_run_activity_at", timestamp); + } + if let Some(timestamp) = run.last_progress_at.as_deref() { + return ("last_progress_at", timestamp); + } + if let Some(timestamp) = run.last_protocol_activity_at.as_deref() { + return ("last_protocol_activity_at", timestamp); + } + + return ("none", "none"); + } + + ("updated_at", run.updated_at.as_str()) +} diff --git a/apps/decodex/src/orchestrator/tests.rs b/apps/decodex/src/orchestrator/tests.rs new file mode 100644 index 00000000..68c55fe4 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests.rs @@ -0,0 +1,1371 @@ +#[cfg(unix)] use std::os::fd::IntoRawFd; +use std::{ + cell::RefCell, + collections::{BTreeSet, HashMap}, + env, + ffi::{OsStr, OsString}, + fs, + io::{Read as _, Write as _}, + iter, + net::{Shutdown, TcpListener, TcpStream}, + path::{Path, PathBuf}, + process::{self, Command}, + sync::{Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; + +use color_eyre::{Report, eyre}; +use tempfile::TempDir; +use time::OffsetDateTime; + +use crate::tracker::records; +#[rustfmt::skip] +use crate::agent::{ + ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, + AppServerHomePreflightFailure, AppServerTransportFailure, AppServerTurnFailure, + DynamicToolHandler, ReviewPolicyStopReason, ReviewPolicyStopRequested, TrackerToolBridge, + TurnContinuationGuard, +}; +#[rustfmt::skip] +use crate::config::{InternalReviewMode, ServiceConfig}; +#[rustfmt::skip] +use crate::github; +#[rustfmt::skip] +use crate::orchestrator::{self, ActiveChildRunContext, ActiveRunDisposition, ActiveRunReconciliation, ActiveWorkflowOverride, ChildExitRetryContext, ChildRunRef, ControlPlaneProjectTick, CONTINUATION_PENDING_RUN_STATUS, DaemonRunChild, DaemonTickRuntimeContext, DashboardEventHub, GhPullRequestReviewStateInspector, ISSUE_COMMENT_TOOL_NAME, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, IssueDispatchMode, IssueRunPlan, IssueTurnContinuationGuard, ManualAttentionRequested, OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH, OPERATOR_DASHBOARD_ENDPOINT_PATH, OperatorStatusSnapshot, PostReviewLaneClassification, PostReviewLaneDecision, PostReviewLaneSnapshot, PreferredRunIdentity, PrepareIssueRunContext, PublishedOperatorSnapshot, PullRequestCommitConnection, PullRequestCommitNode, PullRequestCommitPayload, PullRequestIssueCommentConnection, PullRequestIssueCommentState, PullRequestIssueCommentsNode, PullRequestPageInfo, PullRequestReactionGroup, PullRequestReactionUsersConnection, PullRequestActor, PullRequestRepository, PullRequestRepositoryOwner, PullRequestReviewConnection, PullRequestIssueCommentNode, PullRequestReviewNode, PullRequestReviewRequestConnection, PullRequestReviewState, PullRequestReviewStateInspector, PullRequestReviewStateNode, PullRequestReviewStateRepository, PullRequestReviewSummaryState, PullRequestReviewThreadConnection, PullRequestReviewThreadNode, PullRequestStatusCheckRollup, RecoveredRuntimeState, RetainedPartialProgress, RetainedReviewRunIdentity, RetryComment, RetryDispatchDecision, RetryEntry, RetryKind, RetryQueue, RunCompletionDisposition, RunSummary, RepoGateFailure, TERMINAL_GUARD_MARKER_FILE, TERMINAL_GUARDED_RUN_STATUS, TRACKER_RATE_LIMIT_WARNING, TargetIssueRunContext, EXTERNAL_REVIEW_ACTOR_LOGIN, EXTERNAL_REVIEW_PASS_PHRASE, EXTERNAL_REVIEW_REQUEST_BODY}; +#[rustfmt::skip] +use crate::prelude::Result; +#[rustfmt::skip] +use crate::state::{ + self, ChildAgentActivitySummary, CodexAccountActivitySummary, CodexAccountMarker, + EffectiveRuntimeMarker, ProjectRegistration, ProtocolActivityMarker, ProtocolActivitySummary, + RUN_ACTIVITY_MARKER_FILE, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_GIT_CREDENTIALS, + RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, StateStore, WorktreeMapping, +}; +use crate::test_support::TestEnvVarGuard; +#[rustfmt::skip] +use crate::tracker::{self, IssueTracker, TrackerComment, TrackerIssue, TrackerIssueBlocker, TrackerLabel, TrackerState, TrackerTeam, records::{LinearExecutionEventIdentity}}; +#[rustfmt::skip] +use crate::workflow::WorkflowDocument; +#[rustfmt::skip] +use crate::worktree::{WorktreeManager, WorktreeSpec}; +use crate::orchestrator::{ReviewHandoffMarker, ReviewOrchestrationMarker}; + +// Workflow reload, intake eligibility, prompting, and candidate selection. +include!("tests/intake/workflow_reload.rs"); +include!("tests/intake/eligibility.rs"); +include!("tests/intake/run_and_prompting.rs"); +include!("tests/intake/prepare_issue_run.rs"); +include!("tests/intake/candidate_selection.rs"); + +// Retry scheduling, runtime failure classes, and recovery cleanup. +include!("tests/retry/scheduling.rs"); +include!("tests/retry/selection.rs"); +include!("tests/runtime/repo_gate.rs"); +include!("tests/runtime/failure.rs"); +include!("tests/recovery/reconciliation.rs"); +include!("tests/recovery/terminal_support.rs"); +include!("tests/recovery/closeout/dispatch.rs"); +include!("tests/recovery/closeout/identity.rs"); +include!("tests/recovery/closeout/cleanup.rs"); +include!("tests/recovery/terminal_failures.rs"); +include!("tests/recovery/runtime_reentry.rs"); + +// Operator status plus retained post-review review/landing behavior. +include!("tests/operator/status_support.rs"); +include!("tests/operator/status/control_plane.rs"); +include!("tests/operator/status/running_lanes.rs"); +include!("tests/operator/status/history.rs"); +include!("tests/operator/status/text.rs"); +include!("tests/operator/status/publishing.rs"); +include!("tests/operator/status/queue.rs"); +include!("tests/operator/status/http.rs"); +include!("tests/operator/status/dashboard.rs"); +include!("tests/review_landing/status_support.rs"); +include!("tests/review_landing/status_rows.rs"); +include!("tests/review_landing/orchestration.rs"); +include!("tests/review_landing/status_markers.rs"); +include!("tests/review_landing/classification_review.rs"); +include!("tests/review_landing/classification_checks.rs"); +include!("tests/review_landing/review_state.rs"); + +const TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID: i64 = 991; +const TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT: i64 = 1_763_600_000; +const TEST_EXTERNAL_REVIEW_AUTO_MERGE_ENABLED_AT: i64 = 1_763_600_120; +const TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN: &str = "someone-else"; +const TEST_SERVICE_ID: &str = "pubfi"; +const TEST_PROJECT_CONFIG_FILE: &str = "project.toml"; + +struct FakeTracker { + listed_issues: Vec, + identifier_lookup_issues: Option>, + issues_by_label: HashMap>, + team_label_ids_by_name: HashMap<(String, String), String>, + refresh_snapshots: RefCell>>, + refresh_error: RefCell>, + label_queries: RefCell>, + comment_queries: RefCell>, + comments: RefCell>, + issue_comments: RefCell>>, + state_updates: RefCell>, + label_updates: RefCell)>>, + label_additions: RefCell)>>, + label_removals: RefCell)>>, +} +impl FakeTracker { + fn new(issues: Vec) -> Self { + Self::with_refresh_snapshots_and_project(issues.clone(), vec![issues], true) + } + + fn with_refresh_snapshots( + listed_issues: Vec, + refresh_snapshots: Vec>, + ) -> Self { + Self::with_refresh_snapshots_and_project(listed_issues, refresh_snapshots, true) + } + + fn with_refresh_snapshots_and_project( + listed_issues: Vec, + refresh_snapshots: Vec>, + _project_exists: bool, + ) -> Self { + Self { + listed_issues, + identifier_lookup_issues: None, + issues_by_label: HashMap::new(), + team_label_ids_by_name: HashMap::new(), + refresh_snapshots: RefCell::new(refresh_snapshots), + refresh_error: RefCell::new(None), + label_queries: RefCell::new(Vec::new()), + comment_queries: RefCell::new(Vec::new()), + comments: RefCell::new(Vec::new()), + issue_comments: RefCell::new(HashMap::new()), + state_updates: RefCell::new(Vec::new()), + label_updates: RefCell::new(Vec::new()), + label_additions: RefCell::new(Vec::new()), + label_removals: RefCell::new(Vec::new()), + } + } + + fn with_refresh_error(listed_issues: Vec, message: &str) -> Self { + let tracker = Self::with_refresh_snapshots_and_project( + listed_issues.clone(), + vec![listed_issues], + true, + ); + + *tracker.refresh_error.borrow_mut() = Some(message.to_owned()); + + tracker + } + + fn with_identifier_lookup_issues(mut self, issues: Vec) -> Self { + self.identifier_lookup_issues = Some(issues); + + self + } + + fn with_label_lookup_issues(mut self, label_name: &str, issues: Vec) -> Self { + self.issues_by_label.insert(label_name.to_owned(), issues); + + self + } + + fn with_team_label_lookup_id( + mut self, + team_id: &str, + label_name: &str, + label_id: &str, + ) -> Self { + self.team_label_ids_by_name + .insert((team_id.to_owned(), label_name.to_owned()), label_id.to_owned()); + + self + } + + #[allow(dead_code)] + fn with_resolved_project_slug(self, _project_slug: &str) -> Self { + self + } + + #[allow(dead_code)] + fn with_required_list_project_slug(self, _project_slug: &str) -> Self { + self + } + + fn with_project_lookup_error(self, _message: &str) -> Self { + self + } +} + +impl IssueTracker for FakeTracker { + fn list_issues_with_label(&self, label_name: &str) -> Result> { + self.label_queries.borrow_mut().push(label_name.to_owned()); + + if let Some(issues) = self.issues_by_label.get(label_name) { + return Ok(issues.clone()); + } + + Ok(self.listed_issues.iter().filter(|issue| issue.has_label(label_name)).cloned().collect()) + } + + fn find_team_label_id(&self, team_id: &str, label_name: &str) -> Result> { + if let Some(label_id) = + self.team_label_ids_by_name.get(&(team_id.to_owned(), label_name.to_owned())) + { + return Ok(Some(label_id.clone())); + } + + Ok(self + .listed_issues + .iter() + .find(|issue| issue.team.id == team_id) + .and_then(|issue| issue.label_id_for_name(label_name).map(ToOwned::to_owned))) + } + + fn get_issue_by_identifier(&self, issue_identifier: &str) -> Result> { + let issues = self.identifier_lookup_issues.as_ref().unwrap_or(&self.listed_issues); + + Ok(issues + .iter() + .find(|issue| issue.identifier.eq_ignore_ascii_case(issue_identifier)) + .cloned()) + } + + fn refresh_issues(&self, issue_ids: &[String]) -> Result> { + if let Some(message) = self.refresh_error.borrow_mut().take() { + return Err(Report::msg(message)); + } + + let snapshot = { + let mut refresh_snapshots = self.refresh_snapshots.borrow_mut(); + + if refresh_snapshots.is_empty() { + self.listed_issues.clone() + } else { + refresh_snapshots.remove(0) + } + }; + + Ok(snapshot + .iter() + .filter(|issue| issue_ids.iter().any(|issue_id| issue_id == &issue.id)) + .cloned() + .collect()) + } + + fn list_comments(&self, issue_id: &str) -> Result> { + self.comment_queries.borrow_mut().push(issue_id.to_owned()); + + Ok(self.issue_comments.borrow().get(issue_id).cloned().unwrap_or_default()) + } + + fn update_issue_state(&self, _issue_id: &str, _state_id: &str) -> Result<()> { + self.state_updates.borrow_mut().push((_issue_id.to_owned(), _state_id.to_owned())); + + Ok(()) + } + + fn add_issue_labels(&self, _issue_id: &str, _label_ids: &[String]) -> Result<()> { + self.label_additions.borrow_mut().push((_issue_id.to_owned(), _label_ids.to_vec())); + + Ok(()) + } + + fn remove_issue_labels(&self, _issue_id: &str, _label_ids: &[String]) -> Result<()> { + self.label_removals.borrow_mut().push((_issue_id.to_owned(), _label_ids.to_vec())); + + Ok(()) + } + + fn create_comment(&self, _issue_id: &str, body: &str) -> Result<()> { + self.comments.borrow_mut().push(body.to_owned()); + self.issue_comments.borrow_mut().entry(_issue_id.to_owned()).or_default().push( + TrackerComment { + body: body.to_owned(), + created_at: String::from("2026-04-12T00:00:00Z"), + }, + ); + + Ok(()) + } +} + +struct FakePullRequestReviewStateInspector { + responses: RefCell>>, +} +impl FakePullRequestReviewStateInspector { + fn new(responses: Vec>) -> Self { + Self { responses: RefCell::new(responses) } + } +} + +impl PullRequestReviewStateInspector for FakePullRequestReviewStateInspector { + fn inspect_review_state(&self, _cwd: &Path, _pr_url: &str) -> Result { + self.responses.borrow_mut().remove(0) + } +} + +fn install_fake_post_issue_comment_gh_response( + temp_dir: &TempDir, + comment_id: i64, + created_at: &str, +) -> TestEnvVarGuard { + let fake_gh_dir = temp_dir.path().join("fake-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let fake_gh_response = serde_json::json!({ + "id": comment_id, + "created_at": created_at, + }) + .to_string(); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write(&fake_gh_path, format!("#!/bin/sh\nprintf '%s' '{fake_gh_response}'\n")) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())) +} + +fn install_fake_admin_merge_gh_response(temp_dir: &TempDir) -> (TestEnvVarGuard, PathBuf) { + install_fake_admin_merge_gh_response_with_merge_exit_code(temp_dir, "deadbeef", 0) +} + +fn install_fake_admin_merge_gh_response_with_merge_exit_code( + temp_dir: &TempDir, + pr_head_oid: &str, + merge_exit_code: i32, +) -> (TestEnvVarGuard, PathBuf) { + let fake_gh_dir = temp_dir.path().join("fake-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let invocation_log_path = temp_dir.path().join("gh-invocation.log"); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +printf '%s\\n' \"$@\" >> '{}'\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"merge\" ]; then\n\ + exit {}\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"view\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + invocation_log_path.display(), + merge_exit_code, + serde_json::json!({ + "state": "MERGED", + "headRefOid": pr_head_oid, + "mergeCommit": { "oid": "cafebabe" }, + }), + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + ( + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())), + invocation_log_path, + ) +} + +fn sample_issue(state_name: &str, labels: &[&str]) -> TrackerIssue { + sample_issue_with_project_slug_and_sort_fields( + "issue-1", + "PUB-101", + "pubfi", + state_name, + labels, + Some(3), + "2026-03-13T04:16:17.133Z", + ) +} + +fn sample_blocker(id: &str, identifier: &str, state_name: &str) -> TrackerIssueBlocker { + TrackerIssueBlocker { + id: id.to_owned(), + identifier: identifier.to_owned(), + state: TrackerState { id: format!("state-{id}"), name: state_name.to_owned() }, + } +} + +fn sample_issue_with_sort_fields( + id: &str, + identifier: &str, + state_name: &str, + labels: &[&str], + priority: Option, + created_at: &str, +) -> TrackerIssue { + sample_issue_with_project_slug_and_sort_fields( + id, identifier, "pubfi", state_name, labels, priority, created_at, + ) +} + +fn sample_issue_with_project_slug_and_sort_fields( + id: &str, + identifier: &str, + _project_slug: &str, + state_name: &str, + labels: &[&str], + priority: Option, + created_at: &str, +) -> TrackerIssue { + let team_labels = vec![ + TrackerLabel { + id: String::from("label-queued"), + name: crate::tracker::automation_queue_label(_project_slug), + }, + TrackerLabel { + id: String::from("label-active"), + name: crate::tracker::automation_active_label(_project_slug), + }, + TrackerLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }, + TrackerLabel { + id: String::from("label-needs-attention"), + name: String::from("decodex:needs-attention"), + }, + ]; + + TrackerIssue { + id: id.to_owned(), + identifier: identifier.to_owned(), + #[cfg(test)] + project_slug: Some(_project_slug.to_owned()), + title: String::from("Implement orchestration"), + description: String::from("Body"), + priority, + created_at: created_at.to_owned(), + updated_at: created_at.to_owned(), + state: TrackerState { id: String::from("state-current"), name: state_name.to_owned() }, + team: TrackerTeam { + id: String::from("team-1"), + name: String::from("Pubfi"), + states: vec![ + TrackerState { id: String::from("state-todo"), name: String::from("Todo") }, + TrackerState { + id: String::from("state-progress"), + name: String::from("In Progress"), + }, + TrackerState { id: String::from("state-review"), name: String::from("In Review") }, + ], + labels: team_labels.clone(), + }, + labels_complete: true, + labels: labels + .iter() + .copied() + .chain(iter::once(tracker::automation_queue_label(_project_slug).as_str())) + .collect::>() + .into_iter() + .enumerate() + .map(|(index, label)| TrackerLabel { + id: format!("label-{index}"), + name: label.to_owned(), + }) + .collect(), + blockers: Vec::new(), + } +} + +fn sample_issue_without_needs_attention_team_label( + state_name: &str, + labels: &[&str], +) -> TrackerIssue { + let mut issue = sample_issue(state_name, labels); + + issue.team.labels.retain(|label| label.name != "decodex:needs-attention"); + + issue +} + +fn sample_review_handoff_marker( + branch_name: &str, + pr_url: &str, + head_oid: &str, +) -> state::ReviewHandoffMarker { + state::ReviewHandoffMarker::new("run-1", 1, branch_name, pr_url, "main", branch_name, head_oid) +} + +fn seed_review_handoff_marker( + state_store: &StateStore, + project_id: &str, + issue_id: &str, + branch_name: &str, + pr_url: &str, + head_oid: &str, +) { + state_store + .upsert_review_handoff_marker( + project_id, + issue_id, + &sample_review_handoff_marker(branch_name, pr_url, head_oid), + ) + .expect("review handoff marker should persist"); +} + +fn seed_review_handoff_marker_value( + state_store: &StateStore, + project_id: &str, + issue_id: &str, + marker: &state::ReviewHandoffMarker, +) { + state_store + .upsert_review_handoff_marker(project_id, issue_id, marker) + .expect("review handoff marker should persist"); +} + +fn seed_review_handoff_marker_for_path( + state_store: &StateStore, + project_id: &str, + worktree_path: &Path, + marker: &state::ReviewHandoffMarker, +) { + let worktree = worktree_mapping_for_path(state_store, project_id, worktree_path); + + seed_review_handoff_marker_value(state_store, project_id, worktree.issue_id(), marker); +} + +fn seed_review_orchestration_marker( + state_store: &StateStore, + project_id: &str, + issue_id: &str, + marker: &state::ReviewOrchestrationMarker, +) { + state_store + .upsert_review_handoff_marker( + project_id, + issue_id, + &state::ReviewHandoffMarker::new( + marker.run_id().to_owned(), + marker.attempt_number(), + marker.branch_name().to_owned(), + marker.pr_url().to_owned(), + "main", + marker.branch_name().to_owned(), + marker.head_sha().to_owned(), + ), + ) + .expect("review handoff marker should persist"); + state_store + .upsert_review_orchestration_marker(project_id, issue_id, marker) + .expect("review orchestration marker should persist"); +} + +fn seed_review_orchestration_marker_for_path( + state_store: &StateStore, + project_id: &str, + worktree_path: &Path, + marker: &state::ReviewOrchestrationMarker, +) { + let worktree = worktree_mapping_for_path(state_store, project_id, worktree_path); + + seed_review_orchestration_marker(state_store, project_id, worktree.issue_id(), marker); +} + +fn persisted_review_handoff_marker( + state_store: &StateStore, + project_id: &str, + issue_id: &str, + branch_name: &str, +) -> state::ReviewHandoffMarker { + state_store + .review_handoff_marker(project_id, issue_id, branch_name) + .expect("review handoff marker should read") + .expect("review handoff marker should exist") +} + +fn persisted_review_orchestration_marker( + state_store: &StateStore, + project_id: &str, + issue_id: &str, + branch_name: &str, +) -> state::ReviewOrchestrationMarker { + let handoff = persisted_review_handoff_marker(state_store, project_id, issue_id, branch_name); + + state_store + .review_orchestration_marker(project_id, issue_id, &handoff) + .expect("review orchestration marker should read") + .expect("review orchestration marker should exist") +} + +fn persisted_review_orchestration_marker_for_path( + state_store: &StateStore, + project_id: &str, + worktree_path: &Path, +) -> state::ReviewOrchestrationMarker { + let worktree = worktree_mapping_for_path(state_store, project_id, worktree_path); + + persisted_review_orchestration_marker( + state_store, + project_id, + worktree.issue_id(), + worktree.branch_name(), + ) +} + +fn worktree_mapping_for_path( + state_store: &StateStore, + project_id: &str, + worktree_path: &Path, +) -> WorktreeMapping { + state_store + .list_worktrees(project_id) + .expect("worktree list should read") + .into_iter() + .find(|worktree| worktree.worktree_path() == worktree_path) + .expect("worktree mapping should exist for path") +} + +fn sample_review_orchestration_marker( + branch_name: &str, + pr_url: &str, + head_oid: &str, + phase: &str, + external_round_count: i64, +) -> state::ReviewOrchestrationMarker { + state::ReviewOrchestrationMarker::new( + "run-1", + 1, + branch_name, + pr_url, + head_oid, + phase, + Some(TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID), + Some(TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT), + Some(0), + 0, + external_round_count, + if phase == "waiting_for_merge" { + Some(TEST_EXTERNAL_REVIEW_AUTO_MERGE_ENABLED_AT) + } else { + None + }, + ) +} + +fn add_external_review_ack(review_state: &mut PullRequestReviewState) { + add_review_request_ack_from_actor(review_state, EXTERNAL_REVIEW_ACTOR_LOGIN); +} + +fn add_review_request_ack_from_actor(review_state: &mut PullRequestReviewState, actor_login: &str) { + review_state.issue_comments.push(PullRequestIssueCommentState { + database_id: TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID, + author_login: Some(actor_login.to_owned()), + body: String::from(EXTERNAL_REVIEW_REQUEST_BODY), + created_at_unix_epoch: TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT, + external_review_eyes_reaction_count: usize::from( + actor_login.eq_ignore_ascii_case(EXTERNAL_REVIEW_ACTOR_LOGIN), + ), + }); +} + +fn add_external_review_summary( + review_state: &mut PullRequestReviewState, + body: &str, + state: &str, + submitted_at_unix_epoch: i64, +) { + add_review_summary_from_actor( + review_state, + EXTERNAL_REVIEW_ACTOR_LOGIN, + body, + state, + submitted_at_unix_epoch, + ); +} + +fn add_review_summary_from_actor( + review_state: &mut PullRequestReviewState, + actor_login: &str, + body: &str, + state: &str, + submitted_at_unix_epoch: i64, +) { + review_state.reviews.push(PullRequestReviewSummaryState { + author_login: Some(actor_login.to_owned()), + body: body.to_owned(), + state: state.to_owned(), + submitted_at_unix_epoch, + }); +} + +fn add_external_review_pass(review_state: &mut PullRequestReviewState) { + add_external_review_pass_from_actor(review_state, EXTERNAL_REVIEW_ACTOR_LOGIN); +} + +fn add_external_review_pass_from_actor( + review_state: &mut PullRequestReviewState, + actor_login: &str, +) { + if actor_login.eq_ignore_ascii_case(EXTERNAL_REVIEW_ACTOR_LOGIN) { + review_state.issue_description_external_review_thumbs_up_count += 1; + } + + add_review_summary_from_actor( + review_state, + actor_login, + EXTERNAL_REVIEW_PASS_PHRASE, + "APPROVED", + TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT + 1, + ); +} + +fn add_external_review_findings(review_state: &mut PullRequestReviewState, body: &str) { + add_review_summary_from_actor( + review_state, + EXTERNAL_REVIEW_ACTOR_LOGIN, + body, + "COMMENTED", + TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT + 1, + ); +} + +fn git_output(worktree_path: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(args) + .output() + .expect("git command should run"); + + assert!( + output.status.success(), + "git {} should succeed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr), + ); + + String::from_utf8(output.stdout).expect("git output should be utf-8").trim().to_owned() +} + +fn git_status_success(worktree_path: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(args) + .output() + .expect("git command should run"); + + assert!( + output.status.success(), + "git {} should succeed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr), + ); +} + +fn commit_worktree_change( + worktree_path: &Path, + file_name: &str, + contents: &str, + message: &str, +) -> String { + git_status_success(worktree_path, &["config", "user.name", "Decodex Tests"]); + git_status_success(worktree_path, &["config", "user.email", "decodex-tests@example.com"]); + + let absolute_path = worktree_path.join(file_name); + + if let Some(parent) = absolute_path.parent() { + fs::create_dir_all(parent).expect("worktree file parent should exist"); + } + + fs::write(absolute_path, contents).expect("worktree file should write"); + + git_status_success(worktree_path, &["add", file_name]); + git_status_success(worktree_path, &["commit", "-m", message]); + + git_output(worktree_path, &["rev-parse", "HEAD"]) +} + +#[allow(clippy::too_many_arguments)] +fn sample_pull_request_review_state( + pr_url: &str, + branch_name: &str, + head_oid: &str, + review_decision: Option<&str>, + mergeable: &str, + merge_state_status: &str, + check_state: Option<&str>, + unresolved_review_threads: usize, +) -> PullRequestReviewState { + sample_pull_request_review_state_with_pending_requests( + pr_url, + branch_name, + head_oid, + review_decision, + mergeable, + merge_state_status, + check_state, + unresolved_review_threads, + 0, + ) +} + +#[allow(clippy::too_many_arguments)] +fn sample_pull_request_review_state_with_pending_requests( + pr_url: &str, + branch_name: &str, + head_oid: &str, + review_decision: Option<&str>, + mergeable: &str, + merge_state_status: &str, + check_state: Option<&str>, + unresolved_review_threads: usize, + pending_review_requests: usize, +) -> PullRequestReviewState { + let head_repository_owner = + github::parse_pull_request_url(pr_url).expect("pull request URL should parse").owner; + + PullRequestReviewState { + url: pr_url.to_owned(), + state: String::from("OPEN"), + is_draft: false, + review_decision: review_decision.map(str::to_owned), + merge_commit_allowed: true, + pending_review_requests, + mergeable: mergeable.to_owned(), + merge_state_status: merge_state_status.to_owned(), + head_ref_name: branch_name.to_owned(), + head_ref_oid: head_oid.to_owned(), + merge_commit_oid: None, + head_repository_name: Some( + github::parse_pull_request_url(pr_url).expect("pull request URL should parse").repo, + ), + head_repository_owner: Some(head_repository_owner), + status_check_rollup_state: check_state.map(str::to_owned), + unresolved_review_threads, + issue_description_external_review_thumbs_up_count: 0, + issue_comments: Vec::new(), + reviews: Vec::new(), + } +} + +fn sample_pull_request_review_state_page( + pr_url: &str, + branch_name: &str, + head_oid: &str, + unresolved_review_threads: usize, + has_next_page: bool, + end_cursor: Option<&str>, +) -> PullRequestReviewStateNode { + let locator = github::parse_pull_request_url(pr_url).expect("pull request URL should parse"); + + PullRequestReviewStateNode { + url: pr_url.to_owned(), + state: String::from("OPEN"), + is_draft: false, + review_decision: Some(String::from("APPROVED")), + review_requests: PullRequestReviewRequestConnection { total_count: 0 }, + mergeable: String::from("MERGEABLE"), + merge_state_status: String::from("CLEAN"), + head_ref_name: branch_name.to_owned(), + head_ref_oid: head_oid.to_owned(), + merge_commit: None, + head_repository: Some(PullRequestRepository { name: locator.repo }), + head_repository_owner: Some(PullRequestRepositoryOwner { login: locator.owner }), + reaction_groups: Vec::new(), + comments: PullRequestIssueCommentConnection { + nodes: Vec::new(), + page_info: PullRequestPageInfo { has_next_page: false, end_cursor: None }, + }, + reviews: PullRequestReviewConnection { nodes: Vec::new() }, + review_threads: PullRequestReviewThreadConnection { + nodes: (0..unresolved_review_threads) + .map(|_| PullRequestReviewThreadNode { is_resolved: false, is_outdated: false }) + .collect(), + page_info: PullRequestPageInfo { + has_next_page, + end_cursor: end_cursor.map(str::to_owned), + }, + }, + commits: PullRequestCommitConnection { + nodes: vec![PullRequestCommitNode { + commit: PullRequestCommitPayload { + status_check_rollup: Some(PullRequestStatusCheckRollup { + state: String::from("SUCCESS"), + }), + }, + }], + }, + } +} + +fn sample_pull_request_review_state_repository( + pull_request: PullRequestReviewStateNode, +) -> PullRequestReviewStateRepository { + PullRequestReviewStateRepository { + merge_commit_allowed: true, + pull_request: Some(pull_request), + } +} + +fn try_git_local_config_value(repo_root: &Path, key: &str) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["config", "--local", "--get", key]) + .output() + .expect("git config should run"); + + if !output.status.success() { + return None; + } + + Some( + String::from_utf8(output.stdout) + .expect("git config output should be utf-8") + .trim() + .to_owned(), + ) +} + +fn git_remote_url(repo_root: &Path, remote_name: &str) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["remote", "get-url", remote_name]) + .output() + .expect("git remote get-url should run"); + + if !output.status.success() { + return None; + } + + Some( + String::from_utf8(output.stdout) + .expect("git remote get-url output should be utf-8") + .trim() + .to_owned(), + ) +} + +fn temp_project_layout() -> (TempDir, ServiceConfig, WorkflowDocument) { + temp_project_layout_with_tracker_project_slug_and_read_first( + "pubfi", + &[], + "Follow the repository policy.\n", + ) +} + +fn sample_workflow() -> WorkflowDocument { + temp_project_layout().2 +} + +fn write_service_config(repo_root: &Path, contents: &str) { + fs::create_dir_all(service_config_dir(repo_root)).expect("service config dir should exist"); + + let contents = + contents.replace("repo_root = \".\"", &format!("repo_root = \"{}\"", repo_root.display())); + + fs::write(service_config_path(repo_root), contents).expect("service config should write"); +} + +fn load_service_config(repo_root: &Path) -> ServiceConfig { + ServiceConfig::from_path(service_config_path(repo_root)).expect("service config should load") +} + +fn service_config_path(repo_root: &Path) -> PathBuf { + service_config_dir(repo_root).join(TEST_PROJECT_CONFIG_FILE) +} + +fn service_config_dir(repo_root: &Path) -> PathBuf { + repo_root + .parent() + .expect("repo root should have temp parent") + .join(".codex/decodex/projects/project") +} + +fn service_workflow_path(repo_root: &Path) -> PathBuf { + service_config_dir(repo_root).join("WORKFLOW.md") +} + +fn sample_service_config_toml( + service_id: &str, + tracker_api_key_env_var: &str, + github_token_env_var: &str, + worktree_root: Option<&Path>, + internal_review_mode: InternalReviewMode, + external_review_enabled: bool, +) -> String { + let mut toml = format!( + r#"service_id = "{service_id}" + +[tracker] +api_key_env_var = "{tracker_api_key_env_var}" + +[github] +token_env_var = "{github_token_env_var}" +"# + ); + + if internal_review_mode != InternalReviewMode::Loop || !external_review_enabled { + toml.push_str("\n\n[codex]\n"); + + if internal_review_mode != InternalReviewMode::Loop { + toml.push_str(&format!( + "internal_review_mode = \"{}\"\n", + internal_review_mode.as_str() + )); + } + if !external_review_enabled { + toml.push_str("external_review_enabled = false\n"); + } + } + + toml.push_str( + r#" + +[paths] +repo_root = "." +"#, + ); + + if let Some(worktree_root) = worktree_root { + toml.push_str(&format!("worktree_root = \"{}\"\n", worktree_root.display())); + } + + toml +} + +fn service_config_toml_for_config( + config: &ServiceConfig, + github_token_env_var: &str, + internal_review_mode: InternalReviewMode, + external_review_enabled: bool, +) -> String { + let default_worktree_root = config.repo_root().join(".worktrees"); + let worktree_root = + (config.worktree_root() != default_worktree_root).then_some(config.worktree_root()); + + sample_service_config_toml( + config.service_id(), + config.tracker().api_key_env_var(), + github_token_env_var, + worktree_root, + internal_review_mode, + external_review_enabled, + ) +} + +fn service_config_with_github_token_env_var( + config: &ServiceConfig, + token_env_var: &str, +) -> ServiceConfig { + write_service_config( + config.repo_root(), + &service_config_toml_for_config( + config, + token_env_var, + config.codex().internal_review_mode(), + config.codex().external_review_enabled(), + ), + ); + + load_service_config(config.repo_root()) +} + +fn service_config_with_external_review_enabled( + config: &ServiceConfig, + external_review_enabled: bool, +) -> ServiceConfig { + write_service_config( + config.repo_root(), + &service_config_toml_for_config( + config, + config.github().token_env_var(), + config.codex().internal_review_mode(), + external_review_enabled, + ), + ); + + load_service_config(config.repo_root()) +} + +fn service_config_with_internal_review_mode( + config: &ServiceConfig, + internal_review_mode: InternalReviewMode, +) -> ServiceConfig { + write_service_config( + config.repo_root(), + &service_config_toml_for_config( + config, + config.github().token_env_var(), + internal_review_mode, + config.codex().external_review_enabled(), + ), + ); + + load_service_config(config.repo_root()) +} + +#[allow(dead_code)] +fn temp_project_layout_with_tracker_project_slug( + _project_slug: &str, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + temp_project_layout_with_tracker_project_slug_and_read_first( + "pubfi", + &[], + "Follow the repository policy.\n", + ) +} + +fn temp_project_layout_with_read_first( + read_first_files: &[(&str, &str)], + workflow_body: &str, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + temp_project_layout_with_tracker_project_slug_and_read_first( + "pubfi", + read_first_files, + workflow_body, + ) +} + +fn temp_project_layout_with_max_turns( + max_turns: u32, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + temp_project_layout_with_tracker_project_slug_max_turns_and_read_first( + "pubfi", + max_turns, + &[], + "Follow the repository policy.\n", + ) +} + +fn temp_project_layout_with_tracker_project_slug_and_read_first( + _project_slug: &str, + read_first_files: &[(&str, &str)], + workflow_body: &str, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + temp_project_layout_with_tracker_project_slug_max_turns_and_read_first( + "pubfi", + 1, + read_first_files, + workflow_body, + ) +} + +fn temp_project_layout_with_tracker_project_slug_max_turns_and_read_first( + _project_slug: &str, + max_turns: u32, + read_first_files: &[(&str, &str)], + workflow_body: &str, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("target-repo"); + let read_first_paths = read_first_files.iter().map(|(path, _)| *path).collect::>(); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(repo_root.join(".worktrees")).expect("worktree root should exist"); + fs::create_dir_all(service_config_dir(&repo_root)).expect("service config dir should exist"); + + for (relative_path, contents) in read_first_files { + let absolute_path = repo_root.join(relative_path); + + if let Some(parent) = absolute_path.parent() { + fs::create_dir_all(parent).expect("read_first parent should exist"); + } + + fs::write(absolute_path, contents).expect("read_first file should exist"); + } + + fs::write( + service_workflow_path(&repo_root), + sample_workflow_markdown("pubfi", &read_first_paths, workflow_body, max_turns), + ) + .expect("workflow should exist"); + fs::write(repo_root.join("README.md"), "test repo\n").expect("tracked repo file should exist"); + + write_service_config( + &repo_root, + &sample_service_config_toml("pubfi", "HOME", "HOME", None, InternalReviewMode::Loop, true), + ); + git_status_success(&repo_root, &["init", "-b", "main"]); + git_status_success(&repo_root, &["config", "user.name", "Decodex Tests"]); + git_status_success(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + git_status_success(&repo_root, &["config", "commit.gpgsign", "false"]); + git_status_success(&repo_root, &["add", "."]); + git_status_success(&repo_root, &["commit", "-m", "bootstrap repo"]); + + let config = load_service_config(&repo_root); + let workflow = + WorkflowDocument::from_path(config.workflow_path()).expect("workflow should load"); + + (temp_dir, config, workflow) +} + +fn temp_project_layout_with_workflow_markdown( + workflow_markdown: &str, +) -> (TempDir, ServiceConfig, WorkflowDocument) { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("target-repo"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(repo_root.join(".worktrees")).expect("worktree root should exist"); + fs::create_dir_all(service_config_dir(&repo_root)).expect("service config dir should exist"); + fs::write(service_workflow_path(&repo_root), workflow_markdown).expect("workflow should exist"); + fs::write(repo_root.join("README.md"), "test repo\n").expect("tracked repo file should exist"); + + write_service_config( + &repo_root, + &sample_service_config_toml("pubfi", "HOME", "HOME", None, InternalReviewMode::Loop, true), + ); + git_status_success(&repo_root, &["init", "-b", "main"]); + git_status_success(&repo_root, &["config", "user.name", "Decodex Tests"]); + git_status_success(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + git_status_success(&repo_root, &["config", "commit.gpgsign", "false"]); + git_status_success(&repo_root, &["add", "."]); + git_status_success(&repo_root, &["commit", "-m", "bootstrap repo"]); + + let config = load_service_config(&repo_root); + let workflow = + WorkflowDocument::from_path(config.workflow_path()).expect("workflow should load"); + + (temp_dir, config, workflow) +} + +fn profile_scoped_workflow_markdown(project_slug: &str) -> String { + let _ = project_slug; + let markdown = r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +canonicalize_commands = ["cargo make fmt", "cargo make lint"] +verify_commands = ["cargo make check"] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Follow the repository policy. +"#; + + markdown.to_string() +} + +fn add_origin_remote(repo_root: &Path, remote_root: &Path) { + let remote_url = remote_root.display().to_string(); + + git_status_success( + remote_root.parent().expect("remote root should have parent"), + &[ + "init", + "--bare", + "-b", + "main", + remote_root.to_str().expect("remote path should be utf-8"), + ], + ); + git_status_success(repo_root, &["remote", "add", "origin", remote_url.as_str()]); + git_status_success(repo_root, &["push", "-u", "origin", "main"]); +} + +fn checkout_new_branch(repo_root: &Path, branch_name: &str) { + git_status_success(repo_root, &["checkout", "-b", branch_name]); +} + +fn sample_workflow_markdown( + _project_slug: &str, + read_first: &[&str], + workflow_body: &str, + max_turns: u32, +) -> String { + let read_first = + read_first.iter().map(|path| format!("\"{path}\"")).collect::>().join(", "); + let context = format!("[context]\nread_first = [{read_first}]"); + let markdown = format!( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = {max_turns} +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {{}} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +{context} ++++ + +{workflow_body}"# + ); + + markdown +} diff --git a/apps/decodex/src/orchestrator/tests/intake/candidate_selection.rs b/apps/decodex/src/orchestrator/tests/intake/candidate_selection.rs new file mode 100644 index 00000000..2ec270f2 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/intake/candidate_selection.rs @@ -0,0 +1,1473 @@ +fn candidate_selection_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn sample_handoff_summary(issue: &TrackerIssue, worktree_path: &Path) -> RunSummary { + RunSummary { + project_id: String::from("pubfi"), + issue_id: issue.id.clone(), + issue_identifier: issue.identifier.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + branch_name: String::from("main"), + worktree_path: worktree_path.to_path_buf(), + attempt_number: 1, + run_id: String::from("run-review-handoff"), + continuation_pending: false, + } +} + +#[test] +fn candidate_selection_sorts_by_priority_created_at_and_identifier() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let high_priority = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(1), + "2026-03-13T04:18:17.133Z", + ); + let oldest_same_priority = sample_issue_with_sort_fields( + "issue-3", + "PUB-103", + "Todo", + &[], + Some(2), + "2026-03-13T04:15:17.133Z", + ); + let newest_same_priority = sample_issue_with_sort_fields( + "issue-4", + "PUB-104", + "Todo", + &[], + Some(2), + "2026-03-13T04:19:17.133Z", + ); + let no_priority = sample_issue_with_sort_fields( + "issue-5", + "PUB-105", + "Todo", + &[], + None, + "2026-03-13T04:14:17.133Z", + ); + let tracker = FakeTracker::new(vec![ + no_priority.clone(), + newest_same_priority.clone(), + oldest_same_priority.clone(), + high_priority.clone(), + ]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![no_priority, newest_same_priority, oldest_same_priority, high_priority], + &workflow, + &state_store, + "pubfi", + ) + .expect("candidate selection should succeed") + .expect("one issue should be selected"); + + assert_eq!(selected.identifier, "PUB-102"); +} + +#[test] +fn candidate_selection_breaks_ties_by_identifier_after_created_at() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let later_identifier = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(2), + "2026-03-13T04:16:17.133Z", + ); + let earlier_identifier = sample_issue_with_sort_fields( + "issue-3", + "PUB-101", + "Todo", + &[], + Some(2), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![later_identifier.clone(), earlier_identifier.clone()]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![later_identifier, earlier_identifier], + &workflow, + &state_store, + "pubfi", + ) + .expect("candidate selection should succeed") + .expect("one issue should be selected"); + + assert_eq!(selected.identifier, "PUB-101"); +} + +#[test] +fn candidate_selection_does_not_requery_queue_label_for_truncated_candidates() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]) + .with_label_lookup_issues(&queue_label, vec![issue.clone()]); + let mut truncated_issue = issue.clone(); + + truncated_issue.labels_complete = false; + + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![truncated_issue], + &workflow, + &state_store, + TEST_SERVICE_ID, + ) + .expect("candidate selection should succeed") + .expect("queue candidate should remain selectable"); + + assert_eq!(selected.identifier, issue.identifier); + assert!(tracker.label_queries.borrow().is_empty()); +} + +#[test] +fn candidate_selection_skips_todo_issue_with_nonterminal_blockers() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut blocked_high_priority = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(1), + "2026-03-13T04:15:17.133Z", + ); + + blocked_high_priority.blockers = vec![sample_blocker("issue-9", "PUB-109", "In Progress")]; + + let unblocked_lower_priority = sample_issue_with_sort_fields( + "issue-3", + "PUB-103", + "Todo", + &[], + Some(2), + "2026-03-13T04:16:17.133Z", + ); + let tracker = + FakeTracker::new(vec![blocked_high_priority.clone(), unblocked_lower_priority.clone()]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![blocked_high_priority, unblocked_lower_priority], + &workflow, + &state_store, + "pubfi", + ) + .expect("candidate selection should succeed") + .expect("one issue should be selected"); + + assert_eq!(selected.identifier, "PUB-103"); +} + +#[test] +fn candidate_selection_respects_single_dispatch_slot() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .upsert_lease("pubfi", "issue-active", "run-1", "In Progress") + .expect("lease should record"); + + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![issue], + &workflow, + &state_store, + "pubfi", + ) + .expect("candidate selection should succeed"); + + assert!(selected.is_none(), "project-level dispatch slot should block new selection"); +} + +#[test] +fn plan_project_issue_run_prefers_post_review_repair_lane_over_normal_candidate() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let normal_issue = sample_issue_with_sort_fields( + "issue-1", + "PUB-101", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let repair_issue = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "In Review", + &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()], + Some(3), + "2026-03-13T04:18:17.133Z", + ); + let tracker = FakeTracker::with_refresh_snapshots( + vec![normal_issue.clone(), repair_issue.clone()], + vec![vec![repair_issue.clone()], vec![repair_issue.clone()], vec![repair_issue.clone()]], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&repair_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/174"; + + state_store + .upsert_worktree( + config.service_id(), + &repair_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &repair_issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("CHANGES_REQUESTED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]); + let selected = orchestrator::select_post_review_repair_issue_candidate_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &[], + &inspector, + ) + .expect("post-review repair selection should succeed") + .expect("repair lane should be selected"); + + assert_eq!(selected.identifier, repair_issue.identifier); + + let tracker = FakeTracker::with_refresh_snapshots( + vec![repair_issue.clone()], + vec![vec![repair_issue.clone()], vec![repair_issue.clone()]], + ); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &repair_issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::ReviewRepair, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("targeted review-repair planning should succeed") + .expect("review-repair issue run should plan"); + + assert_eq!(summary.issue_identifier, repair_issue.identifier); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::ReviewRepair); + assert_eq!(summary.issue_state, "In Review"); +} + +#[test] +fn post_review_repair_selection_skips_exhausted_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let repair_issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![repair_issue.clone()], + vec![vec![repair_issue.clone()]], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&repair_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/174"; + + state_store + .upsert_worktree( + config.service_id(), + &repair_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + for attempt in 1..=3 { + state_store + .record_run_attempt( + &format!("run-review-repair-{attempt}"), + &repair_issue.id, + attempt, + "failed", + ) + .expect("failed repair attempt should record"); + } + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &repair_issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("CHANGES_REQUESTED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]); + let selected = orchestrator::select_post_review_repair_issue_candidate_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &[], + &inspector, + ) + .expect("post-review repair selection should succeed"); + + assert!(selected.is_none(), "exhausted repair lanes should not be redispatched"); +} + +#[test] +fn targeted_post_review_repair_skips_persisted_exhausted_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let repair_issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![repair_issue.clone()], + vec![vec![repair_issue.clone()], vec![repair_issue.clone()]], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&repair_issue.identifier, false) + .expect("worktree should exist"); + + state_store + .upsert_worktree( + config.service_id(), + &repair_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_retry_budget_attempt_count(&worktree.path, "older-run", 3, 3) + .expect("retry budget marker should write"); + + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &repair_issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::ReviewRepair, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("targeted review-repair planning should succeed"); + + assert!(summary.is_none(), "persisted exhausted budget should block direct repair dispatch"); +} + +#[test] +fn candidate_selection_allows_multi_slot_dispatch_when_configured() { + let workflow_source = + sample_workflow_markdown("pubfi", &[], "Multi-slot workflow policy.\n", 1); + + assert!(workflow_source.contains("max_concurrent_agents = 1")); + + let workflow = WorkflowDocument::parse_markdown( + &workflow_source.replace("max_concurrent_agents = 1", "max_concurrent_agents = 2"), + ) + .expect("workflow should parse"); + + assert_eq!(workflow.frontmatter().execution().max_concurrent_agents(), 2); + + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .upsert_lease("pubfi", "issue-active", "run-1", "In Progress") + .expect("existing lease should record"); + + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![issue], + &workflow, + &state_store, + "pubfi", + ) + .expect("candidate selection should succeed") + .expect("one issue should be selected"); + + assert_eq!(selected.identifier, "PUB-101"); +} + +#[test] +fn plan_project_issue_run_prefers_post_review_closeout_lane_over_normal_candidate() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let normal_issue = sample_issue("Todo", &[]); + let closeout_issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![normal_issue.clone(), closeout_issue.clone()], + vec![ + vec![closeout_issue.clone()], + vec![closeout_issue.clone()], + vec![closeout_issue.clone()], + ], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + + state_store + .upsert_worktree( + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &closeout_issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_orchestration_marker( + &worktree.branch_name, + pr_url, + &head_oid, + "waiting_for_merge", + 1, + ), + ); + + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let mut merged_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + merged_review_state.state = String::from("MERGED"); + + let inspector = FakePullRequestReviewStateInspector::new(vec![ + Ok(merged_review_state.clone()), + Ok(merged_review_state), + ]); + let selected = orchestrator::select_post_review_issue_candidate_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &[], + &inspector, + ) + .expect("post-review closeout selection should succeed") + .expect("closeout lane should be selected"); + + assert_eq!(selected.issue.identifier, closeout_issue.identifier); + assert_eq!(selected.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); + assert_eq!( + selected + .preferred_run_identity + .as_ref() + .map(|identity| (identity.run_id.as_str(), identity.attempt_number)), + Some(("run-1", 1)) + ); + + let tracker = FakeTracker::with_refresh_snapshots( + vec![closeout_issue.clone()], + vec![vec![closeout_issue.clone()], vec![closeout_issue.clone()]], + ); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &closeout_issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("targeted closeout planning should succeed") + .expect("closeout issue run should plan"); + + assert_eq!(summary.issue_identifier, closeout_issue.identifier); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); + assert_eq!(summary.issue_state, "In Review"); +} + +#[test] +fn plan_project_issue_run_allows_merged_closeout_after_retry_budget() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let closeout_issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![closeout_issue.clone()], + vec![ + vec![closeout_issue.clone()], + vec![closeout_issue.clone()], + vec![closeout_issue.clone()], + ], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + + state_store + .upsert_worktree( + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &closeout_issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_orchestration_marker( + &worktree.branch_name, + pr_url, + &head_oid, + "waiting_for_merge", + 1, + ), + ); + + for attempt in 1..=3 { + state_store + .record_run_attempt( + &format!("run-closeout-{attempt}"), + &closeout_issue.id, + attempt, + "failed", + ) + .expect("failed attempt should record"); + } + + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let mut merged_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + merged_review_state.state = String::from("MERGED"); + + let inspector = FakePullRequestReviewStateInspector::new(vec![ + Ok(merged_review_state.clone()), + Ok(merged_review_state), + ]); + let selected = orchestrator::select_post_review_issue_candidate_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &[], + &inspector, + ) + .expect("post-review closeout selection should succeed") + .expect("closeout lane should be selected"); + + assert_eq!(selected.issue.identifier, closeout_issue.identifier); + assert_eq!(selected.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); + + let tracker = FakeTracker::with_refresh_snapshots( + vec![closeout_issue.clone()], + vec![vec![closeout_issue.clone()], vec![closeout_issue.clone()]], + ); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &closeout_issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("targeted closeout planning should succeed") + .expect("closeout issue run should plan"); + + assert_eq!(summary.issue_identifier, closeout_issue.identifier); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); +} + +#[test] +fn retained_closeout_identity_reuse_respects_attempt_history() { + { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = candidate_selection_service_owned_issue("In Review"); + let identity = RetainedReviewRunIdentity { + run_id: String::from("pub-101-attempt-1-111"), + attempt_number: 1, + }; + + assert!( + orchestrator::retained_closeout_run_identity_is_reusable( + &state_store, + &issue.id, + &identity, + ) + .expect("missing attempts should be reusable for recovered closeout") + ); + + state_store + .record_run_attempt(&identity.run_id, &issue.id, identity.attempt_number, "failed") + .expect("failed attempt should record"); + + assert!( + !orchestrator::retained_closeout_run_identity_is_reusable( + &state_store, + &issue.id, + &identity, + ) + .expect("failed attempts should not be reused for closeout") + ); + assert_eq!( + state_store.next_attempt_number(&issue.id).expect("next attempt should calculate"), + 2, + "actual failed closeout attempts should still allocate the next attempt" + ); + } + { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = candidate_selection_service_owned_issue("In Review"); + let identity = RetainedReviewRunIdentity { + run_id: String::from("pub-101-attempt-1-111"), + attempt_number: 1, + }; + + state_store + .record_run_attempt(&identity.run_id, &issue.id, identity.attempt_number, "succeeded") + .expect("completed handoff attempt should record"); + state_store + .record_run_attempt("pub-101-attempt-2-222", &issue.id, 2, "succeeded") + .expect("later non-retry attempt should record"); + + assert!( + orchestrator::retained_closeout_run_identity_is_reusable( + &state_store, + &issue.id, + &identity, + ) + .expect("later non-retry attempts should not block handoff identity reuse") + ); + assert_eq!( + state_store.next_attempt_number(&issue.id).expect("next attempt should calculate"), + 3, + "non-retry local history may still know about later attempts" + ); + } + + for status in ["failed", "interrupted", TERMINAL_GUARDED_RUN_STATUS] { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = candidate_selection_service_owned_issue("In Review"); + let identity = RetainedReviewRunIdentity { + run_id: String::from("pub-101-attempt-1-111"), + attempt_number: 1, + }; + let retry_run_id = format!("pub-101-attempt-2-{status}"); + + state_store + .record_run_attempt(&identity.run_id, &issue.id, identity.attempt_number, "succeeded") + .expect("completed handoff attempt should record"); + state_store + .record_run_attempt(&retry_run_id, &issue.id, 2, status) + .expect("later closeout retry should record"); + + assert!( + !orchestrator::retained_closeout_run_identity_is_reusable( + &state_store, + &issue.id, + &identity, + ) + .expect("later retry-budget attempts should block handoff identity reuse"), + "later `{status}` closeout retry should block handoff identity reuse" + ); + assert_eq!( + state_store.next_attempt_number(&issue.id).expect("next attempt should calculate"), + 3, + "real `{status}` closeout retries should keep incrementing" + ); + } +} + +#[test] +fn internal_review_only_retained_drain_handles_same_issue_closeout_after_merge_visibility() { + for closeout_available in [true, false] { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&config, "PATH"), + false, + ); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![ + vec![issue.clone()], + vec![issue.clone()], + vec![issue.clone()], + vec![issue.clone()], + ], + ); + let repo_root = config.repo_root().to_path_buf(); + let pr_url = "https://github.com/hack-ink/decodex/pull/176"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let landed_merge_subject = r#"{"schema":"decodex/commit/1","summary":"Land current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "main", + &repo_root.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let open_review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let mut merged_review_state = open_review_state.clone(); + + merged_review_state.state = String::from("MERGED"); + + let handoff_summary = sample_handoff_summary(&issue, &repo_root); + let closeout_summary = RunSummary { + dispatch_mode: IssueDispatchMode::Closeout, + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + ..handoff_summary.clone() + }; + let closeout_dispatches = RefCell::new(Vec::new()); + let drained = orchestrator::drain_internal_review_only_retained_tail_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &handoff_summary, + &FakePullRequestReviewStateInspector::new(vec![ + Ok(open_review_state.clone()), + Ok(open_review_state), + Ok(merged_review_state.clone()), + Ok(merged_review_state), + ]), + |source_summary| { + closeout_dispatches.borrow_mut().push(source_summary.issue_id.clone()); + + assert_eq!(source_summary.run_id, handoff_summary.run_id); + assert_eq!(source_summary.attempt_number, handoff_summary.attempt_number); + + if closeout_available { Ok(Some(closeout_summary.clone())) } else { Ok(None) } + }, + ) + .expect("internal-review-only retained drain should succeed"); + + if closeout_available { + assert_eq!( + drained.expect("merged same-issue closeout should dispatch"), + closeout_summary + ); + } else { + assert!( + drained.is_none(), + "retained drain should stop cleanly when closeout is unavailable" + ); + } + + assert_eq!(*closeout_dispatches.borrow(), vec![issue.id.clone()]); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "waiting_for_merge"); + + assert_admin_merge_invocation( + &invocation_log_path, + &head_oid, + landed_merge_subject, + pr_url, + ); + } +} + +fn assert_admin_merge_invocation( + invocation_log_path: &Path, + head_oid: &str, + landed_merge_subject: &str, + pr_url: &str, +) { + let gh_invocation = fs::read_to_string(invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + String::from(head_oid), + String::from("--subject"), + String::from(landed_merge_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + ] + ); +} + +#[test] +fn internal_review_only_retained_drain_stops_cleanly_when_checks_are_pending() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled(&config, false); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = candidate_selection_service_owned_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let repo_root = config.repo_root().to_path_buf(); + let head_oid = git_output(&repo_root, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/177"; + + state_store + .upsert_worktree(config.service_id(), &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let handoff_summary = sample_handoff_summary(&issue, &repo_root); + let closeout_dispatches = RefCell::new(Vec::new()); + let drained = orchestrator::drain_internal_review_only_retained_tail_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &handoff_summary, + &FakePullRequestReviewStateInspector::new(vec![ + Ok(sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("PENDING"), + 0, + )), + Ok(sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("PENDING"), + 0, + )), + ]), + |source_summary| { + closeout_dispatches.borrow_mut().push(source_summary.issue_id.clone()); + + Ok(None) + }, + ) + .expect("pending checks should stop the retained drain cleanly"); + + assert!(drained.is_none()); + assert!(closeout_dispatches.borrow().is_empty()); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "request_pending"); +} + +#[test] +fn post_review_closeout_selection_skips_completed_issue_with_open_pull_request() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let closeout_issue = candidate_selection_service_owned_issue("Done"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![closeout_issue.clone()], + vec![vec![closeout_issue.clone()]], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + + state_store + .upsert_worktree( + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &closeout_issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let open_pr_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let inspector = FakePullRequestReviewStateInspector::new(vec![ + Ok(open_pr_review_state.clone()), + Ok(open_pr_review_state), + ]); + let selected = orchestrator::select_post_review_closeout_issue_candidate_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &[], + &inspector, + ) + .expect("post-review closeout selection should succeed"); + + assert!( + selected.is_none(), + "completed issues should not auto-dispatch closeout until the PR is merged" + ); +} + +#[test] +fn closeout_dispatch_policy_rejects_open_pull_request() { + for state_name in ["Done", "In Review"] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let closeout_issue = candidate_selection_service_owned_issue(state_name); + let tracker = FakeTracker::new(vec![closeout_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/176"; + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let open_pr_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let dispatch_inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(open_pr_review_state.clone())]); + let block_reason_inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(open_pr_review_state)]); + + assert!( + !orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &dispatch_inspector, + ) + .expect("dispatch policy inspection should succeed"), + "{state_name} closeout issues must wait until the retained PR is merged", + ); + assert_eq!( + orchestrator::closeout_dispatch_block_reason_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &block_reason_inspector, + ) + .expect("block reason inspection should succeed"), + Some("pull_request_not_merged"), + "{state_name} closeout issues with open PRs should stay blocked, not ineligible", + ); + } +} + +#[test] +fn closeout_dispatch_policy_allows_completed_issue_after_pull_request_merges() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let closeout_issue = candidate_selection_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![closeout_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/177"; + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let inspector = FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]); + + assert!( + orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &inspector, + ) + .expect("dispatch policy inspection should succeed"), + "completed issues should pass closeout dispatch after the retained PR merges", + ); +} + +#[test] +fn closeout_dispatch_policy_blocks_completed_issue_with_missing_review_handoff_record() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let closeout_issue = candidate_selection_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![closeout_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let _worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + + assert!( + !orchestrator::issue_passes_closeout_dispatch_policy( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + ) + .expect("dispatch policy inspection should succeed"), + "completed issues with missing review handoff must remain non-dispatchable", + ); + assert_eq!( + orchestrator::closeout_dispatch_block_reason( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + ) + .expect("block reason inspection should succeed"), + Some("missing_review_handoff_record"), + "completed issues with retained worktrees but no review handoff should stay retained as blocked lanes", + ); +} + +#[test] +fn closeout_dispatch_policy_rejects_completed_issue_without_service_active_label() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let closeout_issue = sample_issue("Done", &[]); + let tracker = FakeTracker::new(vec![closeout_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/177"; + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let inspector = FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]); + + assert!( + !orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &inspector, + ) + .expect("dispatch policy inspection should succeed"), + "completed issues without service ownership must not pass closeout dispatch", + ); + assert_eq!( + orchestrator::closeout_dispatch_block_reason_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("block reason inspection should succeed"), + None, + "ownership-gated closeout issues should become ineligible rather than retained as blocked lanes", + ); +} + +#[test] +fn closeout_dispatch_policy_uses_matching_handoff_record_for_current_branch() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let closeout_issue = candidate_selection_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![closeout_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let current_pr_url = "https://github.com/hack-ink/decodex/pull/177"; + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + current_pr_url, + &head_oid, + ); + + state_store + .upsert_review_handoff_marker( + config.service_id(), + &closeout_issue.id, + &ReviewHandoffMarker::new( + String::from("run-review-handoff-newer"), + 2, + String::from("x/pubfi-pub-101-next"), + String::from("https://github.com/hack-ink/decodex/pull/999"), + String::from("release/9.x"), + String::from("x/pubfi-pub-101-next"), + String::from("feedface"), + ), + ) + .expect("unrelated branch handoff should persist"); + + let mut merged_review_state = sample_pull_request_review_state( + current_pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + merged_review_state.state = String::from("MERGED"); + + assert!( + orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &closeout_issue, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(merged_review_state)]), + ) + .expect("dispatch policy inspection should succeed"), + "matching branch handoff records should remain dispatchable even when newer tracker comments belong to another branch", + ); +} + +#[test] +fn non_dry_run_closeout_dispatch_errors_when_pr_state_read_fails() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_DIRECT_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = candidate_selection_service_owned_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/179"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let error = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: Some("In Review"), + dry_run: false, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect_err("non-dry-run closeout dispatch should surface GH state read failures"); + + assert!(error.to_string().contains("pull_request_state_read_failed")); +} + +#[test] +fn candidate_selection_skips_issue_claimed_by_another_process() { + let workflow = WorkflowDocument::parse_markdown( + &sample_workflow_markdown("pubfi", &[], "Claim-aware workflow policy.\n", 1) + .replace("max_concurrent_agents = 1", "max_concurrent_agents = 2"), + ) + .expect("workflow should parse"); + let (_temp_dir, config, _default_workflow) = temp_project_layout(); + let remote_store = StateStore::open_in_memory().expect("remote state store should open"); + let local_store = StateStore::open_in_memory().expect("local state store should open"); + let claimed_issue = sample_issue_with_sort_fields( + "issue-claimed", + "PUB-100", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let free_issue = sample_issue_with_sort_fields( + "issue-free", + "PUB-101", + "Todo", + &[], + Some(2), + "2026-03-13T04:16:18.133Z", + ); + + remote_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("remote dispatch-slot root should configure"); + local_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("local dispatch-slot root should configure"); + + assert!( + remote_store + .try_acquire_lease(config.service_id(), &claimed_issue.id, "run-claimed", "In Progress") + .expect("remote issue claim should succeed") + ); + + let tracker = FakeTracker::new(vec![claimed_issue.clone(), free_issue.clone()]); + let selected = orchestrator::select_issue_candidate( + &tracker, + vec![claimed_issue, free_issue.clone()], + &workflow, + &local_store, + config.service_id(), + ) + .expect("candidate selection should succeed") + .expect("the unclaimed issue should still be selected"); + + assert_eq!(selected.id, free_issue.id); +} diff --git a/apps/decodex/src/orchestrator/tests/intake/eligibility.rs b/apps/decodex/src/orchestrator/tests/intake/eligibility.rs new file mode 100644 index 00000000..f0350e7d --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/intake/eligibility.rs @@ -0,0 +1,263 @@ +#[test] +fn eligibility_uses_state_label_blocker_and_lease_rules() { + let (_, _, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let eligible_issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![eligible_issue.clone()]); + let opted_out_issue = sample_issue("Todo", &["decodex:manual-only"]); + let needs_attention_issue = sample_issue("Todo", &["decodex:needs-attention"]); + let mut blocked_issue = sample_issue("Todo", &[]); + + blocked_issue.blockers = vec![sample_blocker("issue-2", "PUB-102", "In Progress")]; + + let mut unblocked_issue = sample_issue("Todo", &[]); + + unblocked_issue.blockers = vec![sample_blocker("issue-3", "PUB-103", "Done")]; + + let wrong_state_issue = sample_issue("In Progress", &[]); + + assert!( + orchestrator::is_issue_eligible( + &tracker, + &eligible_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &opted_out_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &needs_attention_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &blocked_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + assert!( + orchestrator::is_issue_eligible( + &tracker, + &unblocked_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &wrong_state_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); + + state_store + .upsert_lease("pubfi", "issue-1", "run-1", "In Progress") + .expect("lease should record"); + + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &eligible_issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("eligibility should succeed") + ); +} + +#[test] +fn claimed_issue_still_passes_post_claim_dispatch_policy() { + let (_, _, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .try_acquire_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease acquisition should succeed"); + + assert!( + orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &tracker::automation_queue_label(TEST_SERVICE_ID), + false, + ) + .expect("dispatch policy should succeed"), + "post-claim policy should ignore the caller's own lease" + ); + assert!( + !orchestrator::is_issue_eligible( + &tracker, + &issue, + TEST_SERVICE_ID, + &workflow, + &state_store, + ) + .expect("pre-claim eligibility should still reject leased issues") + ); +} + +#[test] +fn machine_only_fenced_descriptions_fail_normal_dispatch_policy() { + let (_, _, workflow) = temp_project_layout(); + let cases = [ + ( + "single json fence", + "```json\n{\n \"schema\": \"opaque-pointer/1\",\n \"id\": \"ptr-1\"\n}\n```", + ), + ( + "multiple json fences", + "```json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n```\n\n```json\n{\n \"schema\": \"opaque-pointer/2\"\n}\n```", + ), + ("four backtick json fence", "````json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n````"), + ("tilde json fence", "~~~json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n~~~"), + ]; + + for (case_name, description) in cases { + let mut issue = sample_issue("Todo", &[]); + + issue.description = description.to_owned(); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + assert!( + !orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &tracker::automation_queue_label(TEST_SERVICE_ID), + false, + ) + .expect("dispatch policy should succeed"), + "normal dispatch should reject {case_name} without a human briefing surface" + ); + } +} + +#[test] +fn prose_plus_fenced_block_description_still_passes_normal_dispatch_policy() { + let (_, _, workflow) = temp_project_layout(); + let mut issue = sample_issue("Todo", &[]); + + issue.description = String::from( + "Implement the retained lane repair.\n\n```json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n```", + ); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + assert!( + orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &tracker::automation_queue_label(TEST_SERVICE_ID), + false, + ) + .expect("dispatch policy should succeed"), + "dispatch should remain allowed when a generic briefing exists outside the fenced block" + ); +} + +#[test] +fn truncated_label_pages_do_not_block_queue_label_dispatch() { + let (_, _, workflow) = temp_project_layout(); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut issue = sample_issue("Todo", &[]); + + issue.labels_complete = false; + + issue.labels.retain(|label| label.name != queue_label.as_str()); + + let tracker = FakeTracker::new(vec![issue.clone()]) + .with_label_lookup_issues(&queue_label, vec![issue.clone()]); + + assert!( + orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &queue_label, + false, + ) + .expect("dispatch policy should succeed"), + "server-filtered queue membership should remain authoritative when the local label page is truncated" + ); +} + +#[test] +fn truncated_label_pages_block_dispatch_when_queue_label_was_removed() { + let (_, _, workflow) = temp_project_layout(); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let mut issue = sample_issue("Todo", &[]); + + issue.labels_complete = false; + + issue.labels.retain(|label| label.name != queue_label.as_str()); + + let tracker = FakeTracker::new(vec![]); + + assert!( + !orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &queue_label, + false, + ) + .expect("dispatch policy should succeed"), + "dispatch should re-check queue membership server-side when the local label page is truncated" + ); +} + +#[test] +fn text_fenced_briefing_still_passes_normal_dispatch_policy() { + let (_, _, workflow) = temp_project_layout(); + let mut issue = sample_issue("Todo", &[]); + + issue.description = + String::from("```text\nImplement the retained lane repair and keep scope tight.\n```"); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + assert!( + orchestrator::issue_passes_dispatch_policy( + &tracker, + &issue, + &workflow, + &tracker::automation_queue_label(TEST_SERVICE_ID), + false, + ) + .expect("dispatch policy should succeed"), + "human-readable fenced text should still count as a generic briefing surface" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/intake/prepare_issue_run.rs b/apps/decodex/src/orchestrator/tests/intake/prepare_issue_run.rs new file mode 100644 index 00000000..1ba563b8 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/intake/prepare_issue_run.rs @@ -0,0 +1,691 @@ +use state::PreacquiredLeaseGuards; + +#[test] +fn prepare_issue_run_records_starting_attempt_before_execute() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = + sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Retry, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("issue preparation should succeed") + .expect("active retry issue should prepare"); + + assert_eq!( + state_store + .run_attempt(&issue_run.run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "starting" + ); + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("lease should exist") + .run_id(), + issue_run.run_id + ); + + let event_types = tracker + .comments + .borrow() + .iter() + .filter_map(|comment| records::parse_linear_execution_event_record(comment)) + .map(|record| record.event_type) + .collect::>(); + + assert_eq!( + event_types, + vec![ + String::from("intake"), + String::from("lease_acquired"), + String::from("worktree_prepared"), + ] + ); +} + +#[test] +fn prepare_issue_run_runs_after_create_workspace_hook() { + let workflow_markdown = r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = ["printf '%s\n' \"$DECODEX_BRANCH\" > \"$DECODEX_REPO_ROOT/after-create.log\""] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Follow the repository policy. + "#; + let (_temp_dir, base_config, workflow) = + temp_project_layout_with_workflow_markdown(workflow_markdown); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + issue, + ) + .expect("issue preparation should succeed") + .expect("startable issue should prepare"); + + assert_eq!( + fs::read_to_string(config.repo_root().join("after-create.log")) + .expect("after-create hook log should exist"), + format!("{}\n", issue_run.worktree.branch_name) + ); +} + +#[test] +fn prepare_issue_run_uses_persisted_retry_budget_marker_after_restart() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + + state::write_run_retry_budget_attempt_count(&worktree.path, "older-run", 4, 2) + .expect("retry budget marker should write"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + issue, + ) + .expect("issue preparation should succeed") + .expect("startable issue should prepare"); + + assert_eq!( + issue_run.retry_budget_base, 2, + "restart recovery should preserve retry budget from the retained worktree marker" + ); +} + +#[test] +fn prepare_issue_run_keeps_persisted_retry_budget_when_preferred_base_is_stale() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + + state::write_run_retry_budget_attempt_count(&worktree.path, "older-run", 4, 2) + .expect("retry budget marker should write"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: Some(0), + }, + issue, + ) + .expect("issue preparation should succeed") + .expect("startable issue should prepare"); + + assert_eq!( + issue_run.retry_budget_base, 2, + "preferred retry-budget base should not hide retained worktree state" + ); +} + +#[test] +fn prepare_issue_run_honors_preferred_identity_when_attempt_is_current() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("issue preparation should succeed") + .expect("targeted issue should prepare"); + + assert_eq!(issue_run.run_id, "planned-run"); + assert_eq!(issue_run.attempt_number, 1); + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("lease should exist") + .run_id(), + "planned-run" + ); +} + +#[cfg(unix)] +#[test] +fn prepare_issue_run_allows_preacquired_cross_process_slot() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let parent_store = StateStore::open_in_memory().expect("parent state store should open"); + let child_store = StateStore::open_in_memory().expect("child state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + parent_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("parent dispatch-slot root should configure"); + child_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("child dispatch-slot root should configure"); + + assert!( + parent_store + .try_acquire_lease(config.service_id(), &issue.id, "planned-run", "In Progress") + .expect("parent should acquire the shared dispatch slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child(&issue.id) + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child(&issue.id) + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + config.service_id(), + &issue.id, + "planned-run", + "In Progress", + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &child_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: true, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("preacquired issue preparation should succeed") + .expect("targeted issue should prepare under the preacquired lease"); + + assert_eq!(issue_run.run_id, "planned-run"); + assert_eq!( + child_store + .lease_for_issue(&issue.id) + .expect("child lease lookup should succeed") + .expect("child should retain the adopted local lease") + .run_id(), + "planned-run", + "preacquired child runs should keep the adopted local lease so cleanup can release the handoff guard" + ); +} + +#[cfg(unix)] +#[test] +fn prepare_issue_run_allows_preacquired_recovered_retry_attempt() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = + sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let parent_store = StateStore::open_in_memory().expect("parent state store should open"); + let child_store = StateStore::open_in_memory().expect("child state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + parent_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("parent dispatch-slot root should configure"); + child_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("child dispatch-slot root should configure"); + + assert!( + parent_store + .try_acquire_lease(config.service_id(), &issue.id, "planned-run", "In Progress") + .expect("parent should acquire the shared dispatch slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child(&issue.id) + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child(&issue.id) + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + config.service_id(), + &issue.id, + "planned-run", + "In Progress", + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + child_store + .record_run_attempt("planned-run", &issue.id, 1, "running") + .expect("recovered attempt should record before targeted execution"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &child_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: true, + dispatch_mode: IssueDispatchMode::Retry, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("recovered retry preparation should succeed") + .expect("planned retry attempt should still execute"); + + assert_eq!(issue_run.run_id, "planned-run"); + assert_eq!(issue_run.attempt_number, 1); + assert_eq!( + child_store + .lease_for_issue(&issue.id) + .expect("child lease lookup should succeed") + .expect("child should retain the adopted local lease") + .run_id(), + "planned-run" + ); +} + +#[cfg(unix)] +#[test] +fn prepare_issue_run_allows_preacquired_cross_process_slot_without_github_token_authority() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let parent_store = StateStore::open_in_memory().expect("parent state store should open"); + let child_store = StateStore::open_in_memory().expect("child state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained worktree should exist"); + let sentinel_path = worktree.path.join("dirty.txt"); + + fs::write(&sentinel_path, "uncommitted repair work\n") + .expect("retained worktree should keep local edits"); + + parent_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("parent dispatch-slot root should configure"); + child_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("child dispatch-slot root should configure"); + + assert!( + parent_store + .try_acquire_lease(config.service_id(), &issue.id, "planned-run", "In Progress") + .expect("parent should acquire the shared dispatch slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child(&issue.id) + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child(&issue.id) + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + config.service_id(), + &issue.id, + "planned-run", + "In Progress", + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &child_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: true, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("preacquired live runs should not require github token authority before review handoff") + .expect("preacquired live runs should still prepare a run"); + + assert_eq!(issue_run.run_id, "planned-run"); + assert_eq!(issue_run.attempt_number, 1); + assert!( + worktree.path.exists(), + "reused retained worktrees should remain available for preacquired child runs" + ); + assert!( + sentinel_path.exists(), + "prepare path must not discard retained local work for preacquired child runs" + ); + assert!( + child_store + .lease_for_issue(&issue.id) + .expect("child lease lookup should succeed") + .expect("preacquired child lease should remain adopted") + .run_id() == "planned-run", + "preacquired child lease should remain adopted after planning" + ); + assert!( + child_store + .latest_run_attempt_for_issue(&issue.id) + .expect("run attempt lookup should work") + .expect("starting attempt should record") + .status() == "starting", + "preacquired child planning should record a starting attempt" + ); +} + +#[cfg(unix)] +#[test] +fn run_target_issue_once_skips_reconciliation_for_preacquired_child_runs() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![Vec::new(), Vec::new()]); + let parent_store = StateStore::open_in_memory().expect("parent state store should open"); + let child_store = StateStore::open_in_memory().expect("child state store should open"); + + parent_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("parent dispatch-slot root should configure"); + child_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("child dispatch-slot root should configure"); + + assert!( + parent_store + .try_acquire_lease(config.service_id(), &issue.id, "planned-run", "In Progress") + .expect("parent should acquire the shared dispatch slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child(&issue.id) + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child(&issue.id) + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .record_run_attempt("planned-run", &issue.id, 1, "running") + .expect("adopted run attempt should record"); + + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &child_store, + issue_id: &issue.id, + preferred_issue_state: Some("In Progress"), + preferred_initial_issue_state: None, + dry_run: false, + lease_preacquired: true, + preferred_issue_claim_fd: Some(child_issue_claim.into_raw_fd()), + preferred_dispatch_slot_fd: Some(child_guard.into_raw_fd()), + preferred_dispatch_slot_index: Some(child_slot_index), + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }) + .expect("targeted child run should not error before refresh lookup"); + + assert!(summary.is_none(), "missing refreshed issue should stop before execution"); + assert_eq!( + child_store + .lease_for_issue(&issue.id) + .expect("child lease lookup should succeed") + .expect("preacquired child lease should remain adopted") + .run_id(), + "planned-run" + ); + assert_eq!( + child_store + .run_attempt("planned-run") + .expect("run lookup should succeed") + .expect("planned attempt should remain recorded") + .status(), + "running" + ); +} + +#[test] +fn prepare_issue_run_rejects_stale_preferred_identity_after_attempt_advance() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + state_store + .record_run_attempt("other-run", &issue.id, 1, "succeeded") + .expect("existing run attempt should record"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: Some(PreferredRunIdentity { + run_id: "planned-run", + attempt_number: 1, + }), + preferred_retry_budget_base: None, + }, + issue.clone(), + ) + .expect("stale targeted issue preparation should not error"); + + assert!(issue_run.is_none(), "stale preferred identity should be rejected"); + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); +} diff --git a/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs new file mode 100644 index 00000000..e3373582 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs @@ -0,0 +1,1912 @@ +use crate::agent::ReviewExecutionMode; +use crate::agent::ReviewHandoffContext; + +struct PromptSurfaces { + developer_instructions: String, + user_input: String, + continuation_input: String, +} +impl PromptSurfaces { + fn all(&self) -> [&str; 3] { + [ + self.developer_instructions.as_str(), + self.user_input.as_str(), + self.continuation_input.as_str(), + ] + } +} + +fn run_and_prompting_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn assert_prompt_orders_thread_replies_after_push(prompt: &str, push_requirement: &str) { + let push_index = + prompt.find(push_requirement).expect("prompt should require push before thread resolution"); + let reply_index = prompt + .find("After the repaired head is pushed, reply in-thread for every addressed comment") + .expect("prompt should place thread replies after push"); + + assert!(push_index < reply_index); +} + +fn build_normal_prompt_surfaces( + config: &ServiceConfig, + workflow: &WorkflowDocument, +) -> PromptSurfaces { + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let state_store = StateStore::open_in_memory().expect("state store should open"); + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + config, + workflow, + &issue_run, + &state_store, + None, + ) + .expect("developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + config, + &issue, + workflow, + &issue_run, + &state_store, + None, + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + workflow, + IssueDispatchMode::Normal, + None, + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + PromptSurfaces { developer_instructions, user_input, continuation_input } +} + +#[test] +fn dry_run_selects_one_issue_and_plans_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![sample_issue("Todo", &[])]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("run once should succeed") + .expect("one issue should be selected"); + + assert_eq!( + summary, + RunSummary { + project_id: String::from("pubfi"), + issue_id: String::from("issue-1"), + issue_identifier: String::from("PUB-101"), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + retry_project_slug: String::new(), + dispatch_mode: orchestrator::IssueDispatchMode::Normal, + branch_name: String::from("x/pubfi-pub-101"), + worktree_path: Path::new(&config.worktree_root().join("PUB-101")).to_path_buf(), + attempt_number: 1, + run_id: summary.run_id.clone(), + continuation_pending: false, + } + ); + assert!(tracker.comments.borrow().is_empty()); +} + +#[test] +fn targeted_identifier_dispatch_accepts_status_ready_queued_issue() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-ready", + "PUB-101", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("status snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == issue.identifier) + .expect("queued issue should appear in status"); + + assert_eq!(candidate.classification, "ready"); + assert_eq!(candidate.reason, "eligible_for_dispatch"); + + let summary = orchestrator::run_target_issue_once_with_inferred_dispatch( + TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.identifier, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + ) + .expect("targeted identifier run should succeed") + .expect("status-ready queued issue should dispatch by identifier"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.issue_identifier, issue.identifier); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Normal); +} + +#[test] +fn targeted_inferred_dispatch_keeps_retry_for_active_issue() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = run_and_prompting_service_owned_issue("In Progress"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let summary = orchestrator::run_target_issue_once_with_inferred_dispatch( + TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.identifier, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + ) + .expect("targeted active identifier run should succeed") + .expect("active target should fall back to retry dispatch"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.issue_identifier, issue.identifier); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Retry); +} + +#[test] +fn targeted_identifier_dispatch_accepts_status_visible_retained_closeout_lane() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = run_and_prompting_service_owned_issue("In Review"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/181"; + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("status snapshot should build"); + let lane = snapshot + .post_review_lanes + .iter() + .find(|lane| lane.issue_identifier == issue.identifier) + .expect("retained closeout lane should appear in status"); + + assert_eq!(lane.classification, "continue"); + assert_eq!(lane.reason, "pull_request_merged_closeout_pending"); + assert_eq!(lane.pr_state.as_deref(), Some("MERGED")); + + let summary = orchestrator::run_target_issue_once_with_inferred_dispatch( + TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.identifier, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + ) + .expect("targeted retained closeout identifier run should succeed") + .expect("status-visible retained closeout lane should dispatch by identifier"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.issue_identifier, issue.identifier); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, "run-1"); + assert_eq!(summary.attempt_number, 1); +} + +#[test] +fn targeted_identifier_dispatch_rejects_different_status_visible_closeout_lane() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let closeout_issue = sample_issue_with_sort_fields( + "issue-closeout", + "PUB-101", + "In Review", + &[active_label.as_str()], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let requested_issue = sample_issue_with_sort_fields( + "issue-requested", + "PUB-102", + "In Review", + &[active_label.as_str()], + Some(2), + "2026-03-13T04:17:17.133Z", + ); + let tracker = FakeTracker::new(vec![closeout_issue.clone(), requested_issue.clone()]); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&closeout_issue.identifier, false) + .expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/182"; + + state_store + .upsert_worktree( + config.service_id(), + &closeout_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("status snapshot should build"); + + assert_eq!(snapshot.post_review_lanes.len(), 1); + assert_eq!(snapshot.post_review_lanes[0].issue_identifier, closeout_issue.identifier); + assert_eq!(snapshot.post_review_lanes[0].classification, "continue"); + assert_eq!(snapshot.post_review_lanes[0].reason, "pull_request_merged_closeout_pending",); + + let error = orchestrator::run_target_issue_once_with_inferred_dispatch( + TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &requested_issue.identifier, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + ) + .expect_err("targeted closeout inference should reject a different visible lane"); + let message = error.to_string(); + + assert!(message.contains("targeted retained closeout mismatch")); + assert!(message.contains(&requested_issue.identifier)); + assert!(message.contains(&closeout_issue.identifier)); +} + +#[test] +fn format_run_once_summary_surfaces_continuation_boundaries() { + let summary = RunSummary { + project_id: String::from("pubfi"), + issue_id: String::from("issue-1"), + issue_identifier: String::from("PUB-101"), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("In Progress"), + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + branch_name: String::from("x/pubfi-pub-101"), + worktree_path: PathBuf::from(".worktrees/PUB-101"), + attempt_number: 1, + run_id: String::from("pub-101-attempt-1"), + continuation_pending: true, + }; + let message = orchestrator::format_run_once_summary(&summary, false); + + assert!(message.contains("run paused at continuation boundary")); + assert!(message.contains("next_action=rerun_or_use_daemon")); + assert!(!message.contains("run complete")); +} + +#[test] +fn dry_run_returns_none_when_intake_has_no_service_owned_candidate() { + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::with_refresh_snapshots_and_project(vec![], vec![vec![]], false); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = + orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("dry run without queued issues should succeed"); + + assert!(summary.is_none(), "empty intake should simply produce no dry-run selection"); + } + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue_with_project_slug_and_sort_fields( + "issue-1", + "PUB-101", + "other-service", + "Todo", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = + orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("dry run should succeed"); + + assert!(summary.is_none(), "service-scoped queue labels should isolate intake"); + } +} + +#[test] +fn no_eligible_issue_message_includes_operator_hint() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let message = orchestrator::format_no_eligible_issue_message(&config, &workflow); + + assert!(message.contains("No eligible issue found for the configured project.")); + assert!(message.contains("`Todo`")); + assert!(message.contains("`decodex:queued:`")); + assert!(message.contains("`decodex:queued:pubfi`")); + assert!(message.contains("`decodex:manual-only`/`decodex:needs-attention`")); + assert!(message.contains("non-terminal state")); + assert!(message.contains("dependency blockers")); + assert!(message.contains("available capacity")); +} + +#[test] +fn dry_run_falls_back_to_normal_issue_when_retained_retry_loses_ownership() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let normal_issue = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(1), + "2026-03-13T04:17:17.133Z", + ); + let retry_issue = run_and_prompting_service_owned_issue("In Progress"); + let retry_issue_without_ownership = sample_issue("In Progress", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![normal_issue.clone(), retry_issue.clone()], + vec![ + vec![retry_issue.clone()], + vec![retry_issue.clone()], + vec![retry_issue_without_ownership.clone()], + vec![retry_issue_without_ownership], + vec![sample_issue("In Progress", &[])], + vec![normal_issue.clone()], + ], + ); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&retry_issue.identifier, false) + .expect("retained retry worktree should exist"); + + state_store + .upsert_worktree( + config.service_id(), + &retry_issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("dry run should succeed") + .expect("normal queued issue should be selected after retained retry is excluded"); + + assert_eq!(summary.issue_identifier, normal_issue.identifier); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Normal); +} + +#[test] +fn developer_instructions_trim_workflow_body_and_preserve_required_guidance() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue, + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ) + .expect("developer instructions should build"); + + assert!(instructions.contains("Workflow policy\nFollow the repository policy.\n")); + assert!(instructions.contains("Keep pre-edit discovery bounded")); + assert!(instructions.contains("Do not browse upstream references")); + assert!(instructions.contains("Tracker tool contract")); + assert!(instructions.contains("You own issue-scoped tracker writes for `PUB-101`.")); + assert!( + instructions.contains("Do not speculate about capabilities you did not directly verify.") + ); + assert!(instructions.contains(ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME)); + assert!(instructions.contains(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME)); + assert!(instructions.contains(ISSUE_REVIEW_HANDOFF_TOOL_NAME)); + assert!(instructions.contains(ISSUE_TERMINAL_FINALIZE_TOOL_NAME)); + assert!(instructions.contains("treat `issue_progress_checkpoint` as terminal completion")); + assert!(!instructions.contains("you may end the turn without")); + assert!(!instructions.contains("WORKFLOW.md\n")); +} + +#[test] +fn review_pull_request_title_normalizes_issue_prefix() { + for title in [ + "Ensure Decodex-created PR titles include issue authority prefix", + "xy-381: Ensure Decodex-created PR titles include issue authority prefix", + ] { + let mut issue = sample_issue("Todo", &[]); + + issue.identifier = String::from("XY-381"); + issue.title = String::from(title); + + assert_eq!( + orchestrator::review_pull_request_title(&issue), + "XY-381: Ensure Decodex-created PR titles include issue authority prefix" + ); + } +} + +#[test] +fn normal_prompts_require_issue_prefixed_pull_request_title() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let mut issue = sample_issue("Todo", &[]); + + issue.identifier = String::from("XY-381"); + issue.title = String::from("Ensure Decodex-created PR titles include issue authority prefix"); + + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("y/decodex-xy-381"), + issue_identifier: String::from("XY-381"), + path: config.worktree_root().join("XY-381"), + reused_existing: false, + }, + retry_project_slug: String::from("decodex"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("xy-381-attempt-1-123"), + retry_budget_base: 0, + }; + let state_store = StateStore::open_in_memory().expect("state store should open"); + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &state_store, + None, + ) + .expect("developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &state_store, + None, + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::Normal, + None, + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + let expected_title = "XY-381: Ensure Decodex-created PR titles include issue authority prefix"; + let create_or_update_instruction = + format!("create or update a non-draft PR titled `{expected_title}`"); + + assert!(developer_instructions.contains(&create_or_update_instruction)); + assert!(user_input.contains(&create_or_update_instruction)); + assert!( + continuation_input + .contains(&format!("ensure the non-draft PR title is `{expected_title}`")) + ); + assert!(developer_instructions.contains("single-line `decodex/commit/1` JSON commit message")); +} + +#[test] +fn normal_prompts_respect_non_loop_internal_review_modes() { + for (mode, expected, forbidden_checkpoint) in [ + ( + InternalReviewMode::Off, + "do not call `issue_review_checkpoint`", + None, + ), + ( + InternalReviewMode::Prompt, + "Review your work repeatedly and fix any logic bugs until no new issues are found.", + Some(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME), + ), + ] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_internal_review_mode(&config, mode); + let prompts = build_normal_prompt_surfaces(&config, &workflow); + + for prompt in prompts.all() { + assert!(prompt.contains(expected), "{mode:?} prompt should contain `{expected}`"); + assert!(!prompt.contains("Follow the repo-native bounded review method")); + + if let Some(forbidden_checkpoint) = forbidden_checkpoint { + assert!(!prompt.contains(forbidden_checkpoint)); + } + + assert!(!prompt.contains("only after the latest `issue_review_checkpoint`")); + } + + assert!( + prompts + .developer_instructions + .contains("Call `issue_review_handoff` after the branch is pushed") + ); + assert!(prompts.user_input.contains("required validation has passed")); + assert!(prompts.continuation_input.contains("after required validation has passed")); + } +} + +#[test] +fn multi_turn_prompts_allow_nonterminal_yield_boundary() { + let (_temp_dir, config, workflow) = temp_project_layout_with_max_turns(4); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::Normal, + None, + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + assert!(user_input.contains("you may end the turn without")); + assert!(continuation_input.contains("you may end the turn without terminal finalization")); + assert!(!user_input.contains("Do not end the turn")); +} + +#[test] +fn closeout_prompts_forbid_clean_continuation_boundaries() { + let (_temp_dir, config, workflow) = temp_project_layout_with_max_turns(4); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Closeout, + attempt_number: 3, + run_id: String::from("pub-101-attempt-3-123"), + retry_budget_base: 0, + }; + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ) + .expect("closeout developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::Closeout, + Some(pr_url), + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + for prompt in [&developer_instructions, &user_input, &continuation_input] { + assert!(prompt.contains("short deterministic tail")); + assert!(prompt.contains("Do not end the turn without")); + assert!(!prompt.contains("you may end the turn without terminal finalization")); + } +} + +#[test] +fn review_repair_prompts_require_same_pr_repair_completion() { + let (_temp_dir, config, workflow) = temp_project_layout_with_max_turns(4); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2-123"), + retry_budget_base: 0, + }; + let pr_url = "https://github.com/hack-ink/decodex/pull/77"; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ) + .expect("review repair developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::ReviewRepair, + Some(pr_url), + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + assert!(developer_instructions.contains(ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME)); + assert!(developer_instructions.contains(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME)); + assert!(developer_instructions.contains("Do not move the issue back to `In Progress`")); + assert!(developer_instructions.contains("do not call `issue_review_handoff`")); + assert!( + developer_instructions + .contains("Follow the repo-native bounded review method from `WORKFLOW.md`") + ); + assert!(developer_instructions.contains( + "including non-thread review summaries, validate the claim against the codebase, tests, and requirements" + )); + assert!(developer_instructions.contains( + "After the repaired head is pushed, reply in-thread for every addressed comment" + )); + assert!(developer_instructions.contains("retained landing fallback")); + assert!(developer_instructions.contains("Do not merge or land the PR yourself")); + assert!(user_input.contains(pr_url)); + assert!(user_input.contains(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME)); + assert!(user_input.contains("Follow the repo-native bounded review method from `WORKFLOW.md`")); + assert!(user_input.contains( + "Read the current review feedback on `https://github.com/hack-ink/decodex/pull/77`, including non-thread review summaries" + )); + assert!( + user_input.contains( + "validate each actionable claim against the codebase, tests, and requirements" + ) + ); + assert!(user_input.contains("Leave pushback or clarification threads open")); + assert!(user_input.contains("because retained landing was not a deterministic clean path")); + assert!(user_input.contains("Do not merge or land the PR yourself")); + assert!(user_input.contains( + "resolve only the GitHub review threads whose fixes landed and verified on the repaired head" + )); + assert!(continuation_input.contains(ISSUE_REVIEW_CHECKPOINT_TOOL_NAME)); + assert!( + continuation_input + .contains("Resume the repo-native bounded review method from `WORKFLOW.md`") + ); + assert!(continuation_input.contains( + "Validate each actionable review claim against the codebase, tests, and requirements before changing code" + )); + assert!( + continuation_input.contains( + "keep pushback or clarification threads open until the repaired head is ready" + ) + ); + assert!(continuation_input.contains("retained landing fallback")); + assert!(continuation_input.contains("do not merge or land the PR yourself")); + assert!(continuation_input.contains("Do not request fresh external review yourself")); + assert!(continuation_input.contains("In Review")); + assert!(continuation_input.contains("review_repair")); + + assert_prompt_orders_thread_replies_after_push( + &developer_instructions, + "push the repaired head.", + ); + assert_prompt_orders_thread_replies_after_push( + &user_input, + "Commit the repair and push the same branch.", + ); + assert_prompt_orders_thread_replies_after_push( + &continuation_input, + "If the repaired head is ready, push it.", + ); +} + +#[test] +fn review_repair_prompts_skip_internal_review_checkpoint_when_disabled() { + let (_temp_dir, config, workflow) = temp_project_layout_with_max_turns(4); + let config = service_config_with_internal_review_mode(&config, InternalReviewMode::Off); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2-123"), + retry_budget_base: 0, + }; + let pr_url = "https://github.com/hack-ink/decodex/pull/77"; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ) + .expect("review repair developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::ReviewRepair, + Some(pr_url), + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + for prompt in [&developer_instructions, &user_input, &continuation_input] { + assert!(prompt.contains("codex.internal_review_mode = \"off\"")); + assert!(prompt.contains("do not call `issue_review_checkpoint`")); + assert!(!prompt.contains("Follow the repo-native bounded review method")); + assert!(!prompt.contains("only after the latest `issue_review_checkpoint`")); + assert!(prompt.contains(ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME)); + } + + assert!( + developer_instructions + .contains("Call `issue_review_repair_complete` after the repaired head is pushed") + ); + assert!(user_input.contains("required validation has passed")); + assert!(continuation_input.contains("required validation has passed")); + assert!(user_input.contains("validate each actionable claim against the codebase")); + assert!(continuation_input.contains("Do not request fresh external review yourself")); +} + +#[test] +fn review_repair_continuation_prompt_uses_configured_success_state() { + let workflow = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "Ready For QA" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 4 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Custom workflow. +"#, + ) + .expect("workflow should parse"); + let issue = sample_issue("Ready For QA", &[]); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::ReviewRepair, + Some("https://github.com/hack-ink/decodex/pull/77"), + workflow.frontmatter().tracker().success_state(), + InternalReviewMode::Loop, + ); + + assert!(continuation_input.contains("Ready For QA")); + assert!(!continuation_input.contains("do not move the issue out of `In Review`")); +} + +#[test] +fn review_repair_prompts_surface_architecture_check_on_fourth_external_round() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join(&issue.identifier); + let pr_url = "https://github.com/hack-ink/decodex/pull/77"; + let review_handoff = ReviewHandoffMarker::new( + "pub-101-attempt-4-123", + 4, + "x/pubfi-pub-101", + pr_url, + "main", + "x/pubfi-pub-101", + "abc123", + ); + + seed_review_handoff_marker_value(&state_store, config.service_id(), &issue.id, &review_handoff); + seed_review_orchestration_marker( + &state_store, + config.service_id(), + &issue.id, + &ReviewOrchestrationMarker::new( + "pub-101-attempt-4-123", + 4, + "x/pubfi-pub-101", + pr_url, + "abc123", + "repair_required", + None, + None, + None, + 0, + 4, + None, + ), + ); + + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: 4, + run_id: String::from("pub-101-attempt-4-123"), + retry_budget_base: 0, + }; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &state_store, + Some(pr_url), + ) + .expect("developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &state_store, + Some(pr_url), + ); + + assert!(developer_instructions.contains("external review round 4")); + assert!(developer_instructions.contains("architectural or root-cause defect")); + assert!(developer_instructions.contains("reset the external review-round budget")); + assert!(user_input.contains("external review round 4")); + assert!(user_input.contains("Do not request fresh external review yourself")); +} + +#[test] +fn review_repair_prompts_ignore_newer_unrelated_branch_orchestration_records() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join(&issue.identifier); + let pr_url = "https://github.com/hack-ink/decodex/pull/77"; + let current_handoff = ReviewHandoffMarker::new( + "pub-101-attempt-4-123", + 4, + "x/pubfi-pub-101", + pr_url, + "main", + "x/pubfi-pub-101", + "abc123", + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + ¤t_handoff, + ); + seed_review_orchestration_marker( + &state_store, + config.service_id(), + &issue.id, + &ReviewOrchestrationMarker::new( + "pub-101-attempt-4-123", + 4, + "x/pubfi-pub-101", + pr_url, + "abc123", + "repair_required", + None, + None, + None, + 0, + 3, + None, + ), + ); + + let unrelated_handoff = ReviewHandoffMarker::new( + "other-run", + 1, + "x/pubfi-pub-101-next", + "https://github.com/hack-ink/decodex/pull/88", + "main", + "x/pubfi-pub-101-next", + "def456", + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &unrelated_handoff, + ); + seed_review_orchestration_marker( + &state_store, + config.service_id(), + &issue.id, + &ReviewOrchestrationMarker::new( + "other-run", + 1, + "x/pubfi-pub-101-next", + "https://github.com/hack-ink/decodex/pull/88", + "def456", + "repair_required", + None, + None, + None, + 0, + 4, + None, + ), + ); + + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: 4, + run_id: String::from("pub-101-attempt-4-123"), + retry_budget_base: 0, + }; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &state_store, + Some(pr_url), + ) + .expect("review repair developer instructions should build"); + + assert!(!developer_instructions.contains("external review round 4")); + assert!(!developer_instructions.contains("architectural or root-cause defect")); +} + +#[test] +fn closeout_prompts_require_retained_pr_closeout_completion() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join(&issue.identifier), + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Closeout, + attempt_number: 3, + run_id: String::from("pub-101-attempt-3-123"), + retry_budget_base: 0, + }; + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ) + .expect("closeout developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + Some(pr_url), + ); + let continuation_input = orchestrator::build_continuation_user_input( + &issue, + &workflow, + IssueDispatchMode::Closeout, + Some(pr_url), + workflow.frontmatter().tracker().success_state(), + config.codex().internal_review_mode(), + ); + + assert!(developer_instructions.contains(ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME)); + assert!(developer_instructions.contains(ISSUE_TRANSITION_TOOL_NAME)); + assert!(developer_instructions.contains("Merge is already authoritative")); + assert!(developer_instructions.contains("Do not land, merge, or request review")); + assert!(developer_instructions.contains("single-line `decodex/commit/1` JSON commit message")); + assert!(developer_instructions.contains("do not call `issue_review_handoff`")); + assert!(developer_instructions.contains("may already be in `Done`")); + assert!(developer_instructions.contains( + "either omit `head_sha` and let `decodex` record the exact current lane HEAD automatically, or pass the exact full current `HEAD` SHA" + )); + assert!(developer_instructions.contains( + "If the issue is still in `In Review`, transition it once to `Done` with `issue_transition` before `issue_closeout_complete`" + )); + assert!(user_input.contains("merged PR lineage")); + assert!(user_input.contains("Merge is already authoritative")); + assert!(user_input.contains("Do not land, merge, or request review")); + assert!(user_input.contains("may already be in `Done`")); + assert!(user_input.contains( + "either omit `head_sha` and let `decodex` record the exact current lane HEAD automatically, or pass the exact full current `HEAD` SHA" + )); + assert!(user_input.contains( + "If the issue is still in `In Review`, move it once to `Done` with `issue_transition` before `issue_closeout_complete`" + )); + assert!(user_input.contains("closeout")); + assert!(continuation_input.contains("merged PR lineage")); + assert!(continuation_input.contains("Merge is already authoritative")); + assert!(continuation_input.contains("Do not land, merge, or request review")); + assert!(continuation_input.contains("may already be in `Done`")); + assert!( + continuation_input + .contains("either omit `head_sha` or pass the exact full current `HEAD` SHA") + ); + assert!(continuation_input.contains( + "If the issue is still in `In Review`, transition it once to `Done` with `issue_transition` before `issue_closeout_complete`" + )); + assert!(continuation_input.contains("closeout")); +} + +#[test] +fn single_turn_prompts_do_not_allow_nonterminal_yield_boundary() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue, + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ) + .expect("developer instructions should build"); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &sample_issue("Todo", &[]), + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ); + + assert!(!developer_instructions.contains("you may end the turn without")); + assert!(!user_input.contains("you may end the turn without")); +} + +#[test] +fn prompts_handle_machine_only_and_text_fenced_tracker_descriptions() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let cases: &[(&str, &str, &[&str])] = &[ + ( + "single json fence", + "```json\n{\n \"schema\": \"opaque-pointer/1\",\n \"id\": \"ptr-1\"\n}\n```", + &["\"schema\": \"opaque-pointer/1\""], + ), + ( + "multiple json fences", + "```json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n```\n\n```json\n{\n \"schema\": \"opaque-pointer/2\"\n}\n```", + &["\"schema\": \"opaque-pointer/1\"", "\"schema\": \"opaque-pointer/2\""], + ), + ( + "four backtick json fence", + "````json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n````", + &["\"schema\": \"opaque-pointer/1\""], + ), + ( + "tilde json fence", + "~~~json\n{\n \"schema\": \"opaque-pointer/1\"\n}\n~~~", + &["\"schema\": \"opaque-pointer/1\""], + ), + ]; + + for (case_name, description, forbidden_fragments) in cases { + let mut issue = sample_issue("Todo", &[]); + + issue.description = (*description).to_owned(); + + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = normal_prompt_issue_run(&config, issue.clone()); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ); + + assert!( + user_input.contains("machine-only tracker description omitted"), + "{case_name} should be redacted" + ); + + for forbidden in *forbidden_fragments { + assert!(!user_input.contains(forbidden), "{case_name} leaked {forbidden}"); + } + } + + let mut issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + + issue.description = + String::from("```text\nImplement the retained lane repair and keep scope tight.\n```"); + + let issue_run = normal_prompt_issue_run(&config, issue.clone()); + let user_input = orchestrator::build_user_input( + &tracker, + &config, + &issue, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ); + + assert!(!user_input.contains("machine-only tracker description omitted")); + assert!(user_input.contains("Implement the retained lane repair and keep scope tight.")); +} + +fn normal_prompt_issue_run( + config: &ServiceConfig, + issue: TrackerIssue, +) -> orchestrator::IssueRunPlan { + orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + } +} + +#[test] +fn developer_instructions_match_trimmed_prompt_shape() { + let read_first_files = [ + ("docs/index.md", "Use the documentation index.\n"), + ("docs/runbook/index.md", "Use the runbook index.\n"), + ]; + let (_temp_dir, config, workflow) = temp_project_layout_with_read_first( + &read_first_files, + "This workflow body should be appended.\n", + ); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue, + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &StateStore::open_in_memory().expect("state store should open"), + None, + ) + .expect("developer instructions should build"); + + assert_eq!( + instructions, + expected_developer_instructions(&read_first_files, &workflow, &issue_run) + ); +} + +#[test] +fn continuation_guard_rejects_first_turn_without_startup_transition() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: &issue.state.name, + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + review_state_inspector: None, + }; + let error = guard + .validate_continuation_boundary(1) + .expect_err("turn 1 should fail if the startup transition never happened"); + + assert!(error.to_string().contains("ended without moving the tracker issue to `In Progress`")); +} + +#[test] +fn continuation_guard_allows_local_startup_transition_on_stale_rereads() { + { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("Todo"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let transition_response = DynamicToolHandler::handle_call( + &tracker_tool_bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "In Progress" }), + ); + + assert!(transition_response.success); + + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: &issue.state.name, + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + review_state_inspector: None, + }; + + assert!( + guard + .should_continue_turn(1) + .expect("a stale pre-write reread should not block turn-one continuation") + ); + + guard.validate_continuation_boundary(1).expect( + "a stale pre-write reread should not hard-fail turn one after a local startup transition", + ); + } + { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("Todo"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let transition_response = DynamicToolHandler::handle_call( + &tracker_tool_bridge, + ISSUE_TRANSITION_TOOL_NAME, + serde_json::json!({ "state": "In Progress" }), + ); + + assert!(transition_response.success); + + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: &issue.state.name, + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + review_state_inspector: None, + }; + + assert!( + guard + .should_continue_turn(1) + .expect("a stale pre-write reread should not block turn-one continuation") + ); + assert!( + guard + .should_continue_turn(2) + .expect("a stale pre-write reread should remain tolerated after turn one") + ); + } +} + +#[test] +fn continuation_guard_allows_review_repair_continuation_while_issue_remains_in_review() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: &issue.state.name, + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + review_state_inspector: None, + }; + + assert!( + guard + .should_continue_turn(2) + .expect("retained review-repair lane should continue while issue remains in review") + ); + + guard.validate_continuation_boundary(2).expect( + "review-repair continuation boundary should stay valid while the issue remains in review", + ); +} + +#[test] +fn continuation_guard_allows_closeout_continuation_after_issue_reaches_completed_state() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let tracker_tool_bridge = TrackerToolBridge::with_run_context_and_state_store( + &tracker, + &issue, + &workflow, + ReviewHandoffContext { + attempt_number: 1, + branch_name: worktree.branch_name.clone(), + run_id: String::from("run-closeout-continuation"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: worktree.path.display().to_string(), + cwd: worktree.path.clone(), + github_token_env_var: None, + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Closeout, + recorded_pr_url: Some(String::from(pr_url)), + }, + &state_store, + ); + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let review_state_inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(review_state.clone()), Ok(review_state)]); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: "In Review", + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Closeout, + review_state_inspector: Some(&review_state_inspector), + }; + + assert!( + guard + .should_continue_turn(2) + .expect("retained closeout lane should continue while the issue remains completed") + ); + + guard + .validate_continuation_boundary(2) + .expect("closeout continuation boundary should stay valid after tracker completion"); +} + +#[test] +fn continuation_guard_blocks_closeout_continuation_when_completed_issue_pr_is_open() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("Done"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/176"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let tracker_tool_bridge = TrackerToolBridge::with_run_context_and_state_store( + &tracker, + &issue, + &workflow, + ReviewHandoffContext { + attempt_number: 1, + branch_name: worktree.branch_name.clone(), + run_id: String::from("run-closeout-open-pr"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: worktree.path.display().to_string(), + cwd: worktree.path.clone(), + github_token_env_var: None, + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Closeout, + recorded_pr_url: Some(String::from(pr_url)), + }, + &state_store, + ); + let review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let review_state_inspector = + FakePullRequestReviewStateInspector::new(vec![Ok(review_state.clone()), Ok(review_state)]); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: "In Review", + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Closeout, + review_state_inspector: Some(&review_state_inspector), + }; + + assert!( + !guard.should_continue_turn(2).expect( + "completed issues must not continue retained closeout while the PR is still open" + ) + ); + + let error = guard + .validate_continuation_boundary(2) + .expect_err("completed issues with open PRs must fail the retained closeout boundary"); + + assert!(error.to_string().contains("retained closeout continuation boundary")); +} + +#[test] +fn continuation_guard_errors_when_completed_issue_pr_state_cannot_be_read() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("Done"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/177"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let tracker_tool_bridge = TrackerToolBridge::with_run_context_and_state_store( + &tracker, + &issue, + &workflow, + ReviewHandoffContext { + attempt_number: 1, + branch_name: worktree.branch_name.clone(), + run_id: String::from("run-closeout-read-failed"), + service_id: String::from(TEST_SERVICE_ID), + worktree_path: worktree.path.display().to_string(), + cwd: worktree.path.clone(), + github_token_env_var: None, + internal_review_mode: InternalReviewMode::Loop, + mode: ReviewExecutionMode::Closeout, + recorded_pr_url: Some(String::from(pr_url)), + }, + &state_store, + ); + let review_state_inspector = FakePullRequestReviewStateInspector::new(vec![ + Err(color_eyre::eyre::eyre!("gh api failed")), + Err(color_eyre::eyre::eyre!("gh api failed")), + ]); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: "In Review", + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Closeout, + review_state_inspector: Some(&review_state_inspector), + }; + let continue_error = guard.should_continue_turn(2).expect_err( + "GH state read failures must not degrade to a silent completed-state closeout skip", + ); + + assert!(continue_error.to_string().contains("PR state read failed")); + + let boundary_error = guard + .validate_continuation_boundary(2) + .expect_err("GH state read failures must fail the retained closeout boundary explicitly"); + + assert!(boundary_error.to_string().contains("PR state read failed")); +} + +#[test] +fn continuation_guard_preserves_original_startable_state_across_continuation_retries() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("In Progress"); + let stale_issue = run_and_prompting_service_owned_issue("Todo"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![stale_issue.clone()]]); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: "Todo", + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + review_state_inspector: None, + }; + + assert!( + guard + .should_continue_turn(2) + .expect("continuation retries must preserve the original startable state even after a refreshed in-progress run plan") + ); +} + +#[test] +fn continuation_guard_stops_when_service_active_label_is_removed() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let issue = run_and_prompting_service_owned_issue("In Progress"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![sample_issue("In Progress", &[])]], + ); + let tracker_tool_bridge = TrackerToolBridge::new(&tracker, &issue, &workflow); + let guard = IssueTurnContinuationGuard { + tracker: &tracker, + tracker_tool_bridge: &tracker_tool_bridge, + workflow: &workflow, + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + initial_issue_state: &issue.state.name, + retry_project_slug: issue + .project_slug + .as_deref() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + review_state_inspector: None, + }; + + assert!( + !guard + .should_continue_turn(2) + .expect("continuation must stop once service ownership is removed"), + ); +} diff --git a/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs b/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs new file mode 100644 index 00000000..a6ee8680 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs @@ -0,0 +1,196 @@ +#[test] +fn daemon_workflow_reload_keeps_last_known_good_on_same_path_failure() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let mut workflow_cache = None; + let initial = orchestrator::load_daemon_tick_workflow(&config, &mut workflow_cache) + .expect("initial workflow load should succeed"); + + assert_eq!(initial, workflow); + + fs::write(config.workflow_path(), "not valid workflow markdown") + .expect("invalid workflow should be written"); + + let fallback = orchestrator::load_daemon_tick_workflow(&config, &mut workflow_cache) + .expect("invalid reload should keep the cached workflow"); + + assert_eq!(fallback, workflow); +} + +#[test] +fn daemon_workflow_reload_replaces_cached_document_after_valid_update() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let mut workflow_cache = None; + + orchestrator::load_daemon_tick_workflow(&config, &mut workflow_cache) + .expect("initial workflow load should succeed"); + + let updated_workflow = sample_workflow_markdown("pubfi", &[], "Updated workflow policy.\n", 1) + .replace("max_attempts = 3", "max_attempts = 5"); + + fs::write(config.workflow_path(), updated_workflow) + .expect("updated workflow should be written"); + + let reloaded = orchestrator::load_daemon_tick_workflow(&config, &mut workflow_cache) + .expect("valid reload should replace the cached workflow"); + + assert_ne!(reloaded, workflow); + assert_eq!(reloaded.frontmatter().execution().max_attempts(), 5); + assert_eq!(reloaded.body(), "Updated workflow policy."); +} + +#[test] +fn configured_cycle_workflow_snapshot_overrides_invalid_disk_workflow() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let workflow_snapshot = workflow.to_markdown().expect("workflow markdown should render"); + + fs::write(config.workflow_path(), "not valid workflow markdown") + .expect("invalid workflow should be written"); + + assert!( + orchestrator::load_configured_cycle_workflow(&config, None).is_err(), + "without an override the configured workflow load should fail" + ); + + let loaded = orchestrator::load_configured_cycle_workflow(&config, Some(&workflow_snapshot)) + .expect("configured workflow load should accept the supplied snapshot"); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &loaded, + state_store: &state_store, + issue_id: &issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("target issue dry run should succeed with the supplied snapshot"); + + assert!(summary.is_some(), "the child path should still run off the cached snapshot"); +} + +#[test] +fn active_child_reconciliation_keeps_spawn_time_workflow_until_exit() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let active_workflow = WorkflowDocument::parse_markdown( + &sample_workflow_markdown("pubfi", &[], "Spawn-time workflow policy.\n", 1) + .replace("max_attempts = 3", "max_attempts = 5"), + ) + .expect("workflow should parse"); + let current_workflow = WorkflowDocument::parse_markdown( + &sample_workflow_markdown("pubfi", &[], "Current workflow policy.\n", 1) + .replace("startable_states = [\"Todo\"]", "startable_states = [\"Backlog\"]"), + ) + .expect("workflow should parse"); + let child_issue = sample_issue("Todo", &[]); + let stale_issue = sample_issue_with_sort_fields( + "issue-stale", + "PUB-202", + "Todo", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![child_issue.clone(), stale_issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("run-child", &child_issue.id, 1, "running") + .expect("child run attempt should record"); + state_store + .upsert_lease("pubfi", &child_issue.id, "run-child", "In Progress") + .expect("child lease should record"); + state_store + .record_run_attempt("run-stale", &stale_issue.id, 1, "running") + .expect("stale run attempt should record"); + state_store + .upsert_lease("pubfi", &stale_issue.id, "run-stale", "In Progress") + .expect("stale lease should record"); + + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + ¤t_workflow, + &state_store, + Some(ActiveWorkflowOverride { + child: ChildRunRef { + issue_id: &child_issue.id, + run_id: "run-child", + attempt_number: 1, + }, + workflow: &active_workflow, + }), + OffsetDateTime::now_utc().unix_timestamp() + 1, + ) + .expect("active-run inspection should succeed"); + + assert!( + actions.iter().all(|action| action.issue.id != child_issue.id), + "the current child should keep its spawn-time workflow snapshot" + ); + assert!(actions.iter().any(|action| { + action.issue.id == stale_issue.id + && matches!(action.disposition, orchestrator::ActiveRunDisposition::NonActive) + })); +} + +fn expected_developer_instructions( + read_first_files: &[(&str, &str)], + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, +) -> String { + let continuation_guidance = if workflow.frontmatter().execution().max_turns() > 1 { + "\n- If more implementation work still remains at the current turn boundary, you may end the turn without `{terminal_finalize_tool}` and `decodex` may continue the same lane in a later turn." + } else { + "" + }; + let mut sections = Vec::new(); + + if !workflow.body().trim().is_empty() { + sections.push(format!("Workflow policy\n{}", workflow.body())); + } + + sections.extend( + read_first_files + .iter() + .map(|(relative_path, contents)| format!("File: {relative_path}\n{contents}")), + ); + sections.push(String::from( + "Execution discipline\n- Keep pre-edit discovery bounded to the smallest code surface that can satisfy the current issue.\n- Start with the implementation files directly implicated by the issue before reading broader docs or repo-wide guidance.\n- Do not browse upstream references or general repository documentation unless a concrete ambiguity blocks the change.\n- Once the relevant change surface is identified, patch code and run validation instead of continuing broad searches.", + )); + sections.push(String::from( + "Commit contract\n- When you create a local commit for this lane, use a single-line `decodex/commit/1` JSON commit message.\n- Required fields: `schema`, `summary`, and `authority`.\n- `authority` must be the authoritative Linear issue identifier for this lane.\n- Optional fields: `related` and `breaking`.\n- Do not encode landing mode, CI status, closeout state, or other process-state fields in the commit message.", + )); + + sections.push(format!( + "Tracker tool contract\n- You own issue-scoped tracker writes for `{issue}`.\n- At the start of execution, call `{transition_tool}` to move the issue to `{in_progress}` and add a brief `{comment_tool}` comment that you started work on run `{run_id}` attempt `{attempt}`.\n- Update `{progress_checkpoint_tool}` whenever the execution phase, focus, next action, blockers, evidence, or verification state changes materially.\n- Follow the repo-native bounded review method from `WORKFLOW.md`: review the actual current diff and branch state, run both the requirements pass and the adversarial reviewer pass, fix only the smallest coherent owned batch, rerun verification, and re-read `HEAD` before deciding the next normalized review status.\n- Every time the repo-native bounded review method produces a result for the current head, call `{review_checkpoint_tool}` with that normalized status, the exact current `HEAD` SHA, and any concise evidence items.\n- Treat failures from repo-native `canonicalize_commands`, `verify_commands`, or tracked rewrites left by that repo gate as continued repair by default: keep fixing the lane and rerun the gate instead of taking `manual_attention` unless the blocker is clearly toolchain, environment, or operator-owned.\n- When the implementation is ready, commit the lane, push branch `{branch}`, and create or update a non-draft PR titled `{pr_title}` for that branch.\n- Call `{review_handoff_tool}` only after the latest `{review_checkpoint_tool}` for this handoff phase and current `HEAD` is `clean`. Then call `{terminal_finalize_tool}` with path `review_handoff`.\n- If you determine the issue needs human attention, add label `{needs_attention}` with `{label_tool}`, explain the exact observed blocker in a comment, including the failed command and raw error when available, and then call `{terminal_finalize_tool}` with path `manual_attention`. Do not speculate about capabilities you did not directly verify. Do not call `{review_handoff_tool}` in that case; `decodex` will stop the lane as a human-required failure without automatic retry.\n- Do not move the issue directly to `{success}` with `{transition_tool}`. `decodex` will complete the success writeback only after its own validation passes.\n- Do not report the run as complete or treat `{progress_checkpoint_tool}` as terminal completion until `{terminal_finalize_tool}` succeeds.{continuation_guidance}\n- Never write to any other issue.", + issue = issue_run.issue.identifier, + transition_tool = ISSUE_TRANSITION_TOOL_NAME, + comment_tool = ISSUE_COMMENT_TOOL_NAME, + label_tool = ISSUE_LABEL_ADD_TOOL_NAME, + progress_checkpoint_tool = ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, + review_checkpoint_tool = ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, + review_handoff_tool = ISSUE_REVIEW_HANDOFF_TOOL_NAME, + terminal_finalize_tool = ISSUE_TERMINAL_FINALIZE_TOOL_NAME, + in_progress = workflow.frontmatter().tracker().in_progress_state(), + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + branch = issue_run.worktree.branch_name, + pr_title = orchestrator::review_pull_request_title(&issue_run.issue), + success = workflow.frontmatter().tracker().success_state(), + needs_attention = workflow.frontmatter().tracker().needs_attention_label(), + continuation_guidance = continuation_guidance, + )); + + sections.join("\n\n") +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs b/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs new file mode 100644 index 00000000..44eaeb79 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs @@ -0,0 +1,225 @@ +#[test] +fn control_plane_snapshot_lists_disabled_registered_projects() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let registration = ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + false, + "test-fingerprint", + ); + + state_store.upsert_project(®istration).expect("project should register"); + + let mut project_runtimes = HashMap::new(); + let snapshot = orchestrator::run_control_plane_tick(&state_store, &mut project_runtimes) + .expect("control-plane snapshot should build"); + let project = snapshot.projects.first().expect("disabled project should be listed"); + + assert_eq!(snapshot.project_id, "all"); + assert_eq!(snapshot.projects.len(), 1); + assert_eq!(project.project_id, "pubfi"); + assert!(!project.enabled); + assert_eq!(project.connector_state, "disabled"); + assert_eq!(project.active_run_count, 0); + assert_eq!(project.retained_worktree_count, 0); + assert!(snapshot.warnings.contains(&String::from("no_enabled_projects"))); + assert!(project_runtimes.is_empty(), "disabled projects should not be ticked"); +} + +#[test] +fn control_plane_snapshot_aggregates_top_level_lanes_for_all_registered_projects() { + let (_active_temp_dir, active_config, _active_workflow) = temp_project_layout(); + let (_idle_temp_dir, idle_base_config, _idle_workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-active", + "PUB-101", + "Todo", + &[], + Some(3), + "2026-04-30T03:01:00Z", + ); + + write_service_config( + idle_base_config.repo_root(), + &sample_service_config_toml("rsnap", "HOME", "HOME", None, InternalReviewMode::Loop, true), + ); + + let idle_config = load_service_config(idle_base_config.repo_root()); + let active_registration = ProjectRegistration::from_config( + active_config.service_id(), + &service_config_path(active_config.repo_root()), + &active_config, + true, + "active-fingerprint", + ); + let idle_registration = ProjectRegistration::from_config( + idle_config.service_id(), + &service_config_path(idle_config.repo_root()), + &idle_config, + true, + "idle-fingerprint", + ); + + state_store + .record_run_attempt("run-active", &issue.id, 1, "running") + .expect("active run should record"); + state_store + .upsert_lease(active_config.service_id(), &issue.id, "run-active", "In Progress") + .expect("active lease should record"); + + let active_snapshot = + orchestrator::build_operator_status_snapshot(&active_config, &state_store, 10) + .expect("active project snapshot should build"); + let idle_snapshot = + orchestrator::build_operator_status_snapshot(&idle_config, &state_store, 10) + .expect("idle project snapshot should build"); + let mut active_snapshot = Some(active_snapshot); + let mut idle_snapshot = Some(idle_snapshot); + let snapshot = orchestrator::collect_control_plane_snapshot( + vec![active_registration, idle_registration], + |project, _project_warnings| { + let project_snapshot = match project.service_id() { + "pubfi" => active_snapshot.take().expect("active snapshot should be used once"), + "rsnap" => idle_snapshot.take().expect("idle snapshot should be used once"), + service_id => panic!("unexpected project {service_id}"), + }; + let project_status = project_snapshot + .projects + .first() + .cloned() + .map(|status| orchestrator::complete_project_status(project, status)); + + ControlPlaneProjectTick { + snapshot: Some(project_snapshot), + project_status, + } + }, + ); + let project_by_id = snapshot + .projects + .iter() + .map(|project| (project.project_id.as_str(), project)) + .collect::>(); + + assert_eq!(snapshot.project_id, "all"); + assert_eq!(snapshot.projects.len(), 2); + assert_eq!( + project_by_id.get("pubfi").expect("active project summary should exist").active_run_count, + 1, + ); + assert_eq!( + project_by_id.get("rsnap").expect("idle project summary should exist").active_run_count, + 0, + ); + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.active_runs[0].run_id, "run-active"); + assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); + assert_eq!(snapshot.active_runs[0].phase, "executing"); +} + +#[test] +fn control_plane_snapshot_keeps_queued_project_summaries_service_scoped() { + let (_decodex_temp_dir, decodex_base_config, _decodex_workflow) = temp_project_layout(); + let (_rsnap_temp_dir, rsnap_base_config, _rsnap_workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let decodex_registration = service_scoped_project_registration(&decodex_base_config, "decodex"); + let rsnap_registration = service_scoped_project_registration(&rsnap_base_config, "rsnap"); + let queued_issue = sample_issue_with_project_slug_and_sort_fields( + "issue-decodex", + "XY-403", + "decodex", + "Todo", + &[], + Some(1), + "2026-05-01T03:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![queued_issue]); + + state_store.upsert_project(&decodex_registration).expect("decodex project should register"); + state_store.upsert_project(&rsnap_registration).expect("rsnap project should register"); + + let registered_projects = state_store.list_projects().expect("registered projects should list"); + let snapshot = orchestrator::collect_control_plane_snapshot( + registered_projects, + |project, _project_warnings| { + let config = ServiceConfig::from_path(project.config_path()) + .expect("project config should load"); + let workflow = WorkflowDocument::from_path(config.workflow_path()) + .expect("project workflow should load"); + let project_snapshot = orchestrator::build_operator_state_snapshot_for_publish( + &tracker, + &config, + &workflow, + &state_store, + 10, + &[], + &[], + ) + .expect("project snapshot should build"); + let project_status = project_snapshot + .projects + .first() + .cloned() + .map(|status| orchestrator::complete_project_status(project, status)); + + ControlPlaneProjectTick { + snapshot: Some(project_snapshot), + project_status, + } + }, + ); + let project_by_id = snapshot + .projects + .iter() + .map(|project| (project.project_id.as_str(), project)) + .collect::>(); + let decodex_project = + project_by_id.get("decodex").expect("decodex project summary should exist"); + let rsnap_project = project_by_id.get("rsnap").expect("rsnap project summary should exist"); + + assert_eq!(snapshot.project_id, "all"); + assert_eq!(snapshot.projects.len(), 2); + assert_eq!(snapshot.queued_candidates.len(), 1); + assert_eq!(snapshot.queued_candidates[0].issue_identifier, "XY-403"); + assert_eq!(decodex_project.queued_candidate_count, 1); + assert_eq!(rsnap_project.queued_candidate_count, 0); + assert_eq!(rsnap_project.waiting_lane_count, 0); + assert_eq!(rsnap_project.attention_count, 0); + + let label_queries = tracker.label_queries.borrow().clone(); + + assert_eq!( + label_queries, + vec![String::from("decodex:queued:decodex"), String::from("decodex:queued:rsnap"),], + ); +} + +fn service_scoped_project_registration( + base_config: &ServiceConfig, + service_id: &str, +) -> ProjectRegistration { + write_service_config( + base_config.repo_root(), + &sample_service_config_toml( + service_id, + "HOME", + "HOME", + None, + InternalReviewMode::Loop, + true, + ), + ); + + let config = load_service_config(base_config.repo_root()); + + ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + true, + &format!("{service_id}-fingerprint"), + ) +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs b/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs new file mode 100644 index 00000000..1846be25 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs @@ -0,0 +1,727 @@ +fn dashboard_response() -> String { + String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8") +} + +#[test] +fn operator_dashboard_background_wash_stays_viewport_fixed() { + let response = dashboard_response(); + + assert!(response.contains("background-attachment: fixed, fixed, fixed, scroll;")); + assert!( + response.contains("background-size: 100vw 100vh, 100vw 100vh, 100vw 100vh, auto;") + ); +} + +#[test] +fn operator_dashboard_child_bucket_rows_split_time_bars_from_event_diagnostics() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.contains("childBucketIsSubsecond")); + assert!(response.contains("childBucketIsEventOnly")); + assert!(response.contains("childBucketEventSignals")); + assert!(response.contains("childBucketEventSummary")); + assert!(response.contains("childBucketDiagnosticSignals")); + assert!(response.contains("childBucketDiagnosticSummary")); + assert!(response.contains("renderChildBucketDiagnosticSignals")); + assert!(response.contains("childBucketHasMeaningfulWallShare")); + assert!(response.contains("childAgentLargeOutputWarnings")); + assert!(response.contains("childAgentLargeOutputSummary")); + assert!(response.contains("childBucketShareLabel")); + assert!(response.contains("childBucketWidth")); + assert!(response.contains("function setPanelMeta(node, text, tone = \"\")")); + assert!(response.contains("function pluralLabel(count, singular, plural = `${singular}s`)")); + assert!(response.contains("return `${count} ${pluralLabel(count, singular, plural)}`;")); + assert!(response.contains("pluralLabel(notices.length, \"alert\")")); + assert!(response.contains("pluralLabel(notices.length, \"warning\")")); + assert!(response.contains("child-bucket is-share")); + assert!(response.contains("child-bucket is-diagnostic")); + assert!(response.contains("child-bucket is-event-only")); + assert!(response.contains("child-bucket-signals")); + assert!(response.contains("child-bucket-signal")); + assert!(response.contains("data-duration=\"wall-share\"")); + assert!(response.contains("data-duration=\"event-diagnostics\"")); + assert!(response.contains("data-duration=\"diagnostic\"")); + assert!(response.contains("function childDiagnosticBucketRank(bucket)")); + assert!(response.contains("summary.current_detail")); + assert!(response.contains("return `${label} · ${formatDuration(summary.current_elapsed_seconds)}`;")); + assert!(response.contains("tool calls")); + assert!(response.contains("output bytes")); + assert!(response.contains("Largest single tool output. Open Debug details for tool attribution.")); + assert!(response.contains("field(\"Large outputs\", childAgentLargeOutputSummary(childAgentActivity(run)))")); + assert!(!response.contains("events only")); + assert!(!response.contains("child-warning")); + assert!(!response.contains("${warnings.length ? `
")); + assert!(!response.contains("warnings.join(\" · \")")); + assert!(!response.contains("summary.largest_tool_output_tool || \"tool\"")); + assert!(!response.contains("child-bucket is-subsecond")); + assert!(!response.contains("data-duration=\"events-only\"")); + assert!(!response.contains("child-bucket.is-event-only .child-bucket-bar::before")); + assert!(response.contains("--child-bucket-value-column: clamp(190px, 18vw, 230px);")); + assert!(response.contains("grid-template-columns: 96px minmax(64px, 1fr) var(--child-bucket-value-column);")); + assert!(response.contains("width: var(--child-bucket-value-column);")); + assert!(response.contains("runNeedsAttention")); + assert!(response.contains("runCountsAsRunning")); + assert!(response.contains("runOperationRequiresLiveAgent")); + assert!(response.contains("runProcessStoppedWithoutAttention")); + assert!(response.contains("runStageLabel")); + assert!(response.contains("return \"Stopped\";")); + assert!(response.contains("Finalizing")); + assert!( + response.contains( + "Agent process stopped before the lane finished; operator recovery is required." + ) + ); + assert!(response.contains("Stopped agent process")); + assert!(response.contains("attention stopped")); + assert!(response.contains("recovery needed")); + assert!(response.contains("agent done")); + assert!(!response.contains("process stopped")); + assert!(response.contains("runningLaneMetaText")); + assert!(response.contains("return count === 1 ? \"1 needs attention\" : `${count} need attention`;")); + assert!(response.contains("nodes.activeRunsMeta,")); + assert!(!response.contains("const parts = [`${derived.liveRuns} live`];")); + assert!(!response.contains("parts.push(`${derived.runningAttentionCount} stalled`)")); + assert!(response.contains("runStaleWithoutKnownProcessNeedsAttention")); + assert!(response.contains("runExecutionLivenessSummary")); + assert!(response.contains("runQueueLeaseSummary")); + assert!(response.contains("lease not held")); + assert!(response.contains("field(\"Attempt status\", run.attempt_status || run.status)")); + assert!(response.contains("field(\"Queue lease\", runQueueLeaseSummary(run))")); + assert!(response.contains("field(\"Execution liveness\", runExecutionLivenessSummary(run))")); + assert!(response.contains("Live, no queue lease")); + assert!(response.contains("queue lease not held; live process keeps lane visible")); + assert!(!response.contains("Queue ownership")); + assert!(response.contains("attention.worktree_path")); + assert!(response.contains("candidate.attention?.attention_error_class")); + assert!(response.contains("facts.push([\"Cause\", humanizeToken(attention.attention_error_class)]);")); + assert!(response.contains("queued attention")); + assert!(response.contains("worktree.ownership_reason")); + assert!(response.contains("const hygiene = worktree.hygiene;")); + assert!(response.contains("hygiene.classification === \"merged_dirty_worktree\"")); + assert!(response.contains("post-land dirty worktree")); + assert!(response.contains("post-land cleanup")); + assert!(response.contains("hygiene.reason ||")); + assert!(response.contains("function renderWorktreeHygieneFields(worktree)")); + assert!(response.contains("field(\"Cleanup state\", humanizeToken(hygiene.classification || \"cleanup_pending\"))")); + assert!(response.contains("field(\"Default branch\", hygiene.default_branch || \"unknown\")")); + assert!(response.contains("field(\"Uncommitted changes\", hygiene.dirty ? \"yes\" : \"no\")")); + assert!(response.contains("local cleanup")); + assert!(response.contains( + "Owned by an intake issue that needs attention; recover it from Intake Queue instead of cleaning it up." + )); + assert!(response.contains( + "No active lane, queued recovery, or PR lane owns this worktree; safe for local cleanup." + )); +} + +#[test] +fn operator_dashboard_renders_account_usage_controls() { + let response = dashboard_response(); + + assert!(response.contains("function codexAccount(run)")); + assert!(response.contains("function codexAccounts(run)")); + assert!(response.contains("function codexAccountDisplayName(account)")); + assert!(response.contains("function codexAccountTokenLabel(refreshStatus)")); + assert!(response.contains("function codexAccountWindowLabel(seconds)")); + assert!(response.contains("function codexAccountStatusTone(account)")); + assert!(response.contains("function renderRunCodexAccountInline(run)")); + assert!(response.contains("function renderAccountPool(snapshot)")); + assert!(response.contains("function codexAccountPoolAccounts(snapshot)")); + assert!(response.contains("function codexAccountPoolRank(account)")); + assert!(response.contains("function codexAccountPoolSortKey(account)")); + assert!(response.contains("function renderCodexAccountPool(accounts)")); + assert!(!response.contains("function renderCodexAccountPoolHeader(accounts)")); + assert!(response.contains("function renderCodexAccountPoolRow(account)")); + assert!(response.contains("function codexAccountDebugSummary(account)")); + assert!(response.contains("function codexAccountPoolDebugSummary(accounts)")); + assert!(response.contains("function codexAccountHistorySummary(account)")); + assert!(response.contains("snapshot?.accounts")); + assert!(response.contains("account?.email")); + assert!(response.contains("run?.account || null")); + assert!(response.contains("run?.accounts")); + assert!(response.contains("account-pool-panel")); + assert!(response.contains("

Accounts

")); + assert!(!response.contains("

Codex Accounts

")); + assert!(!response.contains(".stack > .panel + .panel")); + assert!(response.contains("panel section-control\" id=\"account-pool-panel\"")); + assert!(response.contains("section-marker section-marker-control")); + assert!(response.contains("section-marker section-marker-execution")); + assert!(response.contains("section-marker section-marker-aftercare")); + assert!(response.contains("Control Plane")); + assert!(response.contains("Execution")); + assert!(response.contains("Closeout")); + assert!(response.contains("Accounts · Projects")); + assert!(response.contains("Running · Intake")); + assert!(response.contains("Review · Recovery · History")); + assert!(response.contains("panel section-execution\" id=\"active-panel\"")); + assert!(response.contains("panel section-aftercare\" id=\"review-panel\"")); + assert!(!response.contains("section-group-start")); + assert!(response.contains("#queue-panel .panel-head")); + assert!(response.contains(".queue-group > .action-card:last-child")); + assert!(response.contains("nodes.projectTitle.textContent = \"Decodex\"")); + assert!(!response.contains("Decodex Operator")); + assert!(response.contains("primary: [\"accountPool\", \"projects\", \"active\", \"queue\", \"review\", \"worktrees\", \"recent\"]")); + assert!(response.contains("#account-pool-panel {")); + assert!(response.contains("#active-panel {\n\t\t\t\tbackground: transparent;")); + assert!(response.contains("account-pool-title")); + assert!(response.contains("account-privacy-toggle")); + assert!(response.contains("account-eye-open")); + assert!(response.contains("account-eye-off")); + assert!(response.contains("const ACCOUNT_PRIVACY_STORAGE_KEY = \"decodex.operator.accountPrivacy\";")); + assert!(response.contains("const ACCOUNT_NAME_OFFSET_STORAGE_KEY = \"decodex.operator.accountNameOffsets\";")); + assert!(response.contains("const ACCOUNT_IDENTITY_EDGE_CHARS = 6;")); + assert!(response.contains("const ACCOUNT_IDENTITY_MIN_EDGE_CHARS = 3;")); + assert!(!response.contains("const ACCOUNT_EMAIL_LOCAL_HEAD_CHARS = 5;")); + assert!(!response.contains("const ACCOUNT_EMAIL_LOCAL_TAIL_CHARS = 4;")); + assert!(response.contains("const ACCOUNT_RANDOM_NAMES = [")); + assert!(!response.contains("const ACCOUNT_RANDOM_NAME_PREFIXES = [")); + assert!(!response.contains("const ACCOUNT_RANDOM_NAME_SUFFIXES = [")); + assert!(response.contains("function trimLeadingEllipsis(value)")); + assert!(response.contains("function compactAccountIdentity(value)")); + assert!(!response.contains("function compactAccountEmailIdentity(value)")); + assert!(response.contains("function codexAccountIdentityHash(value)")); + assert!(response.contains("function codexAccountRandomName(account)")); + assert!(response.contains("function codexAccountEmail(account)")); + assert!(response.contains("function compactAccountEmail(email)")); + assert!(response.contains("function loadAccountPrivacy()")); + assert!(response.contains("function loadAccountNameOffsets()")); + assert!(response.contains("function persistAccountPrivacy(hidden)")); + assert!(response.contains("function persistAccountNameOffsets()")); + assert!(response.contains("function renderAccountPrivacyToggle()")); + assert!(response.contains("function codexAccountRandomNameKey(account)")); + assert!(response.contains("function codexAccountRandomNameOffset(account)")); + assert!(response.contains("function renderCodexAccountRandomNameButton(account)")); + assert!(response.contains("function codexAccountShowsEmail(account)")); + assert!(response.contains("function codexAccountVisibleName(account)")); + assert!(response.contains("function codexAccountDisplayTitle(account)")); + assert!(response.contains("return codexAccountShowsEmail(account) ? email : codexAccountRandomName(account);")); + assert!(response.contains("? compactAccountEmail(email)")); + assert!(response.contains("account-name-reroll")); + assert!(response.contains("data-account-name-reroll")); + assert!(response.contains("aria-label=\"Change account name\"")); + assert!(response.contains("return `${local.slice(0, 3)}...${local.slice(-3)}${domain}`;")); + assert!(response.contains("return ACCOUNT_RANDOM_NAMES[index];")); + assert!(response.contains("return status === \"selected\" ? 1 : 0;")); + assert!(response.contains("return codexAccountPoolSortKey(left).localeCompare(codexAccountPoolSortKey(right));")); + assert!(response.contains("const hash = codexAccountIdentityHash(identity).toString(16).padStart(8, \"0\");")); + assert!(response.contains("return hash;")); + assert!(!response.contains("const checkedAt = codexAccountNumber(account?.checked_at_unix_epoch) || 0;")); + assert!(!response.contains("localeCompare(codexAccountDisplayName(right))")); + assert!(!response.contains("return account.account_fingerprint;")); + assert!(!response.contains("`fingerprint ${account.account_fingerprint || \"unknown\"}`")); + assert!(!response.contains("const fingerprint = account.account_fingerprint || \"unknown\";")); + assert!(!response.contains("account.account_fingerprint || \"unknown\",\n")); + assert!(response.contains("renderAccountPrivacyToggle();")); + assert!(response.contains("persistAccountPrivacy(accountEmailsHidden);")); + assert!(response.contains("persistAccountNameOffsets();")); + assert!(response.contains("let lastDashboardRender = null;")); + assert!(response.contains("lastDashboardRender = {")); + assert!(response.contains("function renderDashboardState({")); + assert!(response.contains("renderDashboardState(lastDashboardRender);")); + assert!(response.contains(".table-meta[data-tone=\"active\"]")); + assert!(response.contains(".table-meta[data-tone=\"attention\"]")); + assert!(response.contains("font-size: 11px;")); + assert!(response.contains("letter-spacing: 0.06em;")); + assert!(response.contains("text-transform: uppercase;")); + assert!(response.contains("setPanelMeta(nodes.accountPoolMeta, meta, activeCount > 0 ? \"active\" : \"\")")); + assert!(response.contains("${pluralize(accounts.length, \"account\")} · ${activeCount} active")); +} + +#[test] +fn operator_dashboard_renders_account_sort_controls() { + let response = dashboard_response(); + + assert!(response.contains("const ACCOUNT_POOL_SORT_STORAGE_KEY = \"decodex.operator.accountSort\";")); + assert!(response.contains("const ACCOUNT_POOL_SORT_COLUMNS = [")); + assert!(response.contains("function loadAccountPoolSort()")); + assert!(response.contains("function persistAccountPoolSort()")); + assert!(response.contains("function isAccountPoolSortKey(value)")); + assert!(response.contains("function renderCodexAccountPoolSortButton([key, label])")); + assert!(response.contains("account-pool-sort")); + assert!(response.contains("data-account-sort-key")); + assert!(response.contains("aria-label=\"Sort accounts by ${escapeHtml(label)}; ${escapeHtml(current)}\"")); + assert!(response.contains("account-sort-up")); + assert!(response.contains("account-sort-down")); + assert!(response.contains("function codexAccountPoolColumnSortValue(account, key)")); + assert!(response.contains("function compareCodexAccountPoolColumn(left, right, key, direction)")); + assert!(response.contains("function compareCodexAccountPoolStable(left, right)")); + assert!(response.contains("function sortCodexAccountPoolAccounts(accounts)")); + assert!(response.contains("codexAccountWindowData(account, \"primary\").remainingPercent")); + assert!(response.contains("codexAccountWindowData(account, \"secondary\").remainingPercent")); + assert!(response.contains("codexAccountCreditsSortValue(account)")); + assert!(response.contains("persistAccountPoolSort();")); + assert!(response.contains("accountPoolSort.key === key && accountPoolSort.direction === \"asc\"")); +} + +#[test] +fn operator_dashboard_accounts_keeps_compact_table_layout() { + let response = dashboard_response(); + + assert!(response.contains("account-use-line")); + assert!(response.contains("account-pool-list")); + assert!(response.contains("account-pool-guide")); + assert!(!response.contains("account-pool-window-heads")); + assert!(response.contains("
")); + assert!(response.contains("[\"account\", \"Account\"]")); + assert!(response.contains("[\"plan\", \"Plan\"]")); + assert!(response.contains("[\"primary\", \"5h\"]")); + assert!(response.contains("[\"secondary\", \"7d\"]")); + assert!(response.contains("[\"credits\", \"Credits\"]")); + assert!(response.contains("[\"status\", \"Status\"]")); + assert!(response.contains("ACCOUNT_POOL_SORT_COLUMNS.map(renderCodexAccountPoolSortButton).join(\"\")")); + assert!(!response.contains("account-table-head")); + assert!(response.contains( + "--account-grid: minmax(220px, 1.12fr) minmax(56px, 0.42fr) repeat(4, minmax(0, 1fr));" + )); + assert!(response.contains( + "--account-grid: minmax(150px, 1fr) minmax(44px, 0.44fr) repeat(4, minmax(0, 1fr));" + )); + assert!(!response.contains("--account-grid: repeat(6, minmax(0, 1fr));")); + assert!(!response.contains("--account-grid: minmax(112px, 1fr)")); + assert!(response.contains(".account-pool-list {\n\t\t\t\t--account-grid:")); + assert!(response.contains(".account-pool {\n\t\t\t\tdisplay: grid;")); + assert!(response.contains("\n\t\t\t\toverflow-x: auto;")); + assert!(response.contains("\n\t\t\t\tdisplay: grid;\n\t\t\t\tmin-width: 760px;\n\t\t\t\tbackground: transparent;")); + assert!(response.contains(".account-pool-guide {\n\t\t\t\tdisplay: grid;")); + assert!(response.contains("grid-template-columns: var(--account-grid);")); + assert!(response.contains(".account-pool-sort {\n\t\t\t\tjustify-self: center;")); + assert!(response.contains(".account-pool-sort-icon")); + assert!(response.contains("background: transparent;")); + assert!(!response.contains("box-shadow: 0 8px 24px color-mix(in srgb, var(--account-accent) 7%, transparent);")); + assert!(!response.contains("account-quota-line")); + assert!(!response.contains("account-window-value")); + assert!(response.contains("account-window-reset")); + assert!(!response.contains(".account-window-reset strong")); + assert!(!response.contains(".account-window-reset span")); + assert!(response.contains("account-status")); + assert!(!response.contains("account-status-pill")); + assert!(response.contains("account-window-label")); + assert!(response.contains(".account-window {\n\t\t\t\tdisplay: inline-grid;")); + assert!(response.contains("grid-template-columns: max-content max-content;")); + assert!(response.contains("justify-content: center;")); + assert!(response.contains("justify-items: center;")); + assert!(response.contains("width: 100%;")); + assert!(response.contains("text-align: center;")); + assert!(response.contains(".account-window-label {\n\t\t\t\tdisplay: none;")); + assert!(response.contains("${escapeHtml(label)}")); + assert!(response.contains("aria-label=\"${escapeHtml(label)} remaining ${escapeHtml(remaining)}, ${escapeHtml(reset.aria)}\"")); + assert!(response.contains("title=\"${escapeHtml(resetTitle)}\"")); + assert!(response.contains("account-window-date")); + assert!(!response.contains("Reset")); + assert!(response.contains("is-selected")); + assert!(response.contains("--account-accent: var(--tone-muted);")); + assert!(response.contains("grid-template-areas:")); + assert!(response.contains("\"id plan primary secondary credit state\"")); + assert!(response.contains("\"meta meta meta meta meta meta\"")); + assert!(!response.contains("\"account status\"")); + assert!(!response.contains("\"windows windows\"")); + assert!(response.contains(".account-row-id {\n\t\t\t\tgrid-area: id;")); + assert!(response.contains("justify-content: center;")); + assert!(response.contains("text-align: center;")); + assert!(response.contains("function codexAccountPlanLabel(account)")); + assert!(response.contains( + "return account?.plan_type ? humanizeToken(account.plan_type) : \"not reported\";" + )); + assert!(response.contains("const plan = codexAccountPlanLabel(account);")); + assert!(response.contains(".account-row-plan {\n\t\t\t\tgrid-area: plan;")); + assert!(response.contains("
${escapeHtml(plan)}
")); + assert!(!response.contains("function codexAccountSecondaryLabel(account)")); + assert!(response.contains("const visibleName = codexAccountVisibleName(account);")); + assert!(response.contains("const displayTitle = codexAccountDisplayTitle(account);")); + assert!(response.contains("title=\"${escapeHtml(displayTitle)}\">${escapeHtml(visibleName)}")); + assert!(response.contains("text.startsWith(\"...\") && text.indexOf(\"...\", 3) === -1")); +} + +#[test] +fn operator_dashboard_accounts_keeps_window_status_and_credit_copy_compact() { + let response = dashboard_response(); + + assert!(response.contains("ACCOUNT_IDENTITY_MIN_EDGE_CHARS,")); + assert!(response.contains("Math.min(ACCOUNT_IDENTITY_EDGE_CHARS, Math.floor(text.length / 2)),")); + assert!(response.contains("return `${text.slice(0, headLength)}...${text.slice(-tailLength)}`;")); + assert!(response.contains("grid-area: primary;")); + assert!(response.contains("grid-area: secondary;")); + assert!(response.contains("justify-self: stretch;")); + assert!(!response.contains("max-width: 190px;")); + assert!(!response.contains("max-width: 142px;")); + assert!(response.contains("min-height: 42px;")); + assert!(response.contains("padding: 10px 0 10px 18px;")); + assert!(response.contains("border-bottom: 1px solid var(--line);")); + assert!(response.contains(".account-pool-list > .account-row:last-child")); + assert!(response.contains("account-row-credit")); + assert!(response.contains(".account-row-credit {\n\t\t\t\tgrid-area: credit;")); + assert!(response.contains(".account-row-credit {\n\t\t\t\tgrid-area: credit;\n\t\t\t\tjustify-self: center;")); + assert!(response.contains(".account-row-credit.is-danger strong")); + assert!(!response.contains("grid-template-columns: minmax(116px, 0.58fr) minmax(190px, 1fr);")); + assert!(!response.contains("grid-template-columns: minmax(34px, max-content) minmax(0, 1fr);")); + assert!(response.contains(".account-window-reset {\n\t\t\t\tdisplay: inline;")); + assert!(response.contains(".account-row::after")); + assert!(response.contains(".account-row::before")); + assert!(response.contains(".account-row:hover::before")); + assert!(response.contains(".account-row:focus-within::before")); + assert!(response.contains(".account-row:hover::after")); + assert!(response.contains(".account-row:focus-within::after")); + assert!(response.contains("background: linear-gradient(90deg, var(--hover), transparent 78%);")); + assert!(response.contains("box-shadow: 0 0 18px color-mix(in srgb, var(--account-accent) 42%, transparent);")); + assert!(response.contains(".account-row:hover .account-window")); + assert!(response.contains(".account-row:focus-within .account-window")); + assert!(response.contains(".account-status::before")); + assert!(response.contains(".account-row.is-selected .account-status")); + assert!(response.contains(".account-row.is-warn .account-status")); + assert!(response.contains(".account-row.is-danger .account-status")); + assert!(response.contains(".account-row:hover .account-status::before")); + assert!(response.contains(".account-row:focus-within .account-status::before")); + assert!(!response.contains("@keyframes account-active")); + assert!(!response.contains("account-active-glow")); + assert!(!response.contains("account-active-sweep")); + assert!(!response.contains("account-active-dot")); + assert!(response.contains("aria-label=\"Account used by this lane\"")); + assert!(response.contains("Account")); + assert!(!response.contains("Codex account")); + assert!(response.contains("aria-label=\"Accounts\"")); + assert!(response.contains("ACCOUNT_PRIVACY_STORAGE_KEY")); + assert!(response.contains("function codexAccountWindowData(account, prefix)")); + assert!(response.contains("renderCodexAccountPoolWindow(account, \"primary\")")); + assert!(response.contains("renderCodexAccountPoolWindow(account, \"secondary\")")); + assert!(!response.contains("
")); + assert!(response.contains("
")); + assert!(response.contains("credits_unlimited")); + assert!(response.contains("function formatCodexAccountCreditsBalance(value)")); + assert!(response.contains("const balance = formatCodexAccountCreditsBalance(account.credits_balance);")); + assert!(response.contains("return number.toFixed(2);")); + assert!(!response.contains(".replace(/\\.00$/, \"\")")); + assert!(!response.contains(".replace(/(\\.\\d)0$/, \"$1\")")); + assert!(response.contains("function codexAccountCreditsTone(account)")); + assert!(response.contains("const credits = codexAccountCreditsSummary(account);")); + assert!(response.contains("const creditTone = codexAccountCreditsTone(account);")); + assert!(response.contains("credits")); + assert!(response.contains("${escapeHtml(credits || \"not reported\")}")); + + let account_credit_index = response + .find("
") + .expect("account credit cell render"); + let account_status_index = response + .find("
") + .expect("account status cell render"); + + assert!(account_credit_index < account_status_index); + assert!(response.contains("return \"0.00\";")); + assert!(!response.contains("return \"No Credits\";")); + assert!(response.contains("return \"Unlimited\";")); + assert!(response.contains("return \"Ready\";")); + assert!(response.contains("return \"not reported\";")); + assert!(!response.contains("depleted")); + assert!(response.contains("rate_limit_reached_type")); + assert!(response.contains("if (normalizedStatus === \"available\")")); + assert!(response.contains("if (normalizedStatus.includes(\"limit\"))")); + assert!(response.contains("return \"Limited\";")); + assert!(response.contains("cooldown_until_unix_epoch")); + assert!(response.contains("`${prefix}_remaining_percent`")); + assert!(response.contains("`${prefix}_resets_at_unix_epoch`")); + assert!(response.contains("value === 18_000")); + assert!(response.contains("value === 604_800")); + assert!(response.contains("function formatCodexAccountResetDuration(seconds)")); + assert!(response.contains("function codexAccountResetDistance(value)")); + assert!(response.contains("function codexAccountResetDisplay(data)")); + assert!(!response.contains("const shortWindow = windowSeconds === 18_000;")); + assert!(response.contains("return { short: \"0m\", phrase: \"reset due now\", isPast: true };")); + assert!(response.contains("return { short, phrase: `resets in ${short}`, isPast: false };")); + assert!(response.contains("date: \"\",")); + assert!(response.contains("date: resetAt,")); + assert!(response.contains("aria: \"reset not reported\",")); + assert!(response.contains("reset at ${resetAt}, ${distance.phrase}")); + assert!(response.contains("data.remainingPercent == null ? \"not reported\" : `${data.remainingPercent}%`;")); + assert!(response.contains("aria-label=\"${escapeHtml(label)} usage not reported\"")); + assert!(response.contains("const resetTitle = `${label} ${remaining}, ${reset.aria}`;")); + assert!(response.contains("${escapeHtml(reset.short)}")); + assert!(response.contains("${reset.date ? `${escapeHtml(reset.date)}` : \"\"}")); + assert!(!response.contains("${escapeHtml(reset.main)}")); + assert!(!response.contains("${escapeHtml(reset.detail)}")); + assert!(response.contains("class=\"account-status\"")); + assert!(response.contains("function codexAccountWindowTone(percent)")); + assert!(response.contains(".account-window.is-warn > strong")); + assert!(response.contains(".account-window.is-danger > strong")); + assert!(!response.contains("account-meter")); + assert!(!response.contains("lowestRemaining}%")); + assert!(response.contains("nodes.accountPool.innerHTML = renderCodexAccountPool(accounts)")); + assert!(response.contains("setPanelMeta(nodes.accountPoolMeta, meta, activeCount > 0 ? \"active\" : \"\")")); + assert!(!response.contains("nodes.accountPoolMeta.textContent = snapshot")); + assert!(!response.contains("account-row-windows")); + assert!(!response.contains("account-mini-window")); + assert!(!response.contains("account-mini-label")); + assert!(!response.contains("grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));")); + assert!(!response.contains("grid-template-columns: minmax(170px, 1fr) minmax(360px, 1.7fr) 118px;")); + assert!(!response.contains("border-right: 1px solid var(--line);")); + assert!(!response.contains("box-shadow: inset 3px 0 0 var(--success)")); + assert!(!response.contains(">Emails")); + assert!(!response.contains("[\"checked\"")); +} + +#[test] +fn operator_dashboard_projects_keep_status_summary_compact() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.contains("function projectCapacitySummary(project)")); + assert!(response.contains("function renderProjectStats(project)")); + assert!(response.contains("function projectsMetaCopy(snapshot, projects)")); + assert!(response.contains("function renderEmptyState(title, copy = \"\")")); + assert!(response.contains("nodes.projectOverview.innerHTML = renderEmptyState(COPY.waitingSnapshot);")); + assert!(response.contains("snapshot ? \"No running lanes\" : COPY.waitingSnapshot")); + assert!(response.contains("return pluralize(1, \"project\");")); + assert!(response.contains("return pluralize(0, \"project\");")); + assert!(response.contains("return projects.length === 1 ? \"Current\" : \"Selected\";")); + assert!(response.contains("return \"\";")); + assert!(response.contains("return pluralize(project.warning_count, \"warning\");")); + assert!(response.contains("return `${pluralize(project.retained_worktree_count, \"worktree\")} retained`;")); + assert!(response.contains("return { label: \"needs attention\", tone: \"tone-blocked\"")); + assert!(response.contains("label: \"sync backoff\"")); + assert!(response.contains("label: \"sync degraded\"")); + assert!(response.contains("return { label: \"ok\", tone: \"tone-land\"")); + assert!(response.contains("function projectSyncMeta(project, health)")); + assert!(response.contains("const connectorCopy = projectSyncMeta(project, health);")); + assert!(response.contains("if (connector === \"ok\")")); + assert!(response.contains("return copy === health.label ? \"\" : copy;")); + assert!(response.contains("return \"ok\";")); + assert!(response.contains("${kicker ? `${escapeHtml(kicker)}` : \"\"}")); + assert!(!response.contains("const connectorCopy = `connector ${connector}`;")); + assert!(!response.contains("const connectorCopy = `sync ${connector}`;")); + assert!(!response.contains("? pluralize(project.warning_count, \"warning\")")); + assert!(!response.contains("explicitly registered")); + assert!(!response.contains("Current registration")); + assert!(!response.contains("Selected registration")); + assert!(!response.contains("Registry snapshot pending")); + assert!(!response.contains("Registered projects appear after the first operator state snapshot.")); + assert!(!response.contains("return \"Registered project\";")); + assert!(!response.contains("Disabled registration")); + assert!(response.contains("aria-label=\"Project status summary\"")); + assert!(response.contains("[project.active_run_count ?? 0, \"running\"]")); + assert!(response.contains("[project.waiting_lane_count ?? 0, \"waiting\"]")); + assert!(response.contains("[project.attention_count ?? 0, \"attention\"]")); + assert!(response.contains("`${project.active_run_count ?? 0} running`")); + assert!(response.contains("`${project.waiting_lane_count ?? 0} waiting`")); + assert!(response.contains("`${project.attention_count ?? 0} attention`")); + assert!(!response.contains("[project.post_review_lane_count ?? 0, \"review/land\"]")); + assert!(!response.contains("[project.retained_worktree_count, \"recovery\"]")); + assert!(!response.contains("aria-label=\"Project capacity\"")); +} + +#[test] +fn operator_dashboard_flow_counts_distinguish_intake_attention() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.contains("queuedCandidateNeedsAttention")); + assert!(response.contains("intakeAttentionCount")); + assert!(response.contains("queuedBlockedWithoutAttention")); + assert!(response.contains("candidate.classification !== \"claimed\"")); + assert!(response.contains("queueBacklogCandidates.filter(queuedCandidateNeedsAttention).length")); + assert!(response.contains( + "${pluralize(derived.postReviewLanes.length, \"PR\")} · ${pluralize(derived.reviewBlockerCount, \"needs attention\", \"need attention\")}" + )); + assert!(response.contains("${pluralize(retainedWorktrees.length, \"worktree\")} · retained or cleanup")); + assert!( + response.contains("Ready, capacity-limited, or blocked issues appear here before they start.") + ); + assert!(!response.contains("claimed without local lane")); + assert!(!response.contains("const repairCount = attentionItems.length;")); +} + +#[test] +fn operator_dashboard_prioritizes_needs_attention_reason_over_retry_count() { + let response = dashboard_response(); + let reason_text = response + .split("function queuedCandidateReasonText(candidate)") + .nth(1) + .expect("queued candidate reason function should exist") + .split("function queuedCandidateNeedsAttention(candidate)") + .next() + .expect("queued candidate reason function should have an end"); + + assert!(reason_text.contains("return \"Needs attention\";")); + assert!( + response.contains("facts.push([\"Attempt status\", humanizeToken(attention.attempt_status)]);") + ); + assert!(response.contains( + "facts.push([\"Failed attempts\", `${attention.retry_budget_attempt_count}${retryMax}`]);" + )); + assert!(response.contains( + "facts.push([\"Auto retry\", autoRetryBlockedReasonText(attention.auto_retry_blocked_reason)]);" + )); + assert!(response.contains("return \"needs-attention label set\";")); + assert!(reason_text.contains("return \"Auto retry paused\";")); + assert!(!response.contains("return \"blocked by needs-attention\";")); + assert!(!reason_text.contains("return \"Retry budget held\";")); + assert!(!response.contains( + "facts.push([\"Retry\", String(attention.retry_budget_attempt_count)]);" + )); + assert!( + reason_text + .find("return \"Needs attention\";") + .expect("needs-attention reason should exist") + < reason_text + .find("return \"Auto retry paused\";") + .expect("retry-budget reason should exist") + ); +} + +#[test] +fn operator_dashboard_header_shows_endpoint_and_snapshot_freshness() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.contains("SNAPSHOT_PUBLISHED_HEADER")); + assert!(response.contains("x-decodex-snapshot-unix-epoch")); + assert!(!response.contains("function dashboardEndpointMeta(path)")); + assert!(response.contains("function dashboardSocketUrl()")); + assert!(response.contains("function snapshotPublishedAtFromResponse(response)")); + assert!(response.contains("function snapshotAgeSeconds(snapshotPublishedAt)")); + assert!(response.contains("function snapshotFreshnessMeta(")); + assert!(response.contains("window.location.protocol === \"https:\" ? \"wss:\" : \"ws:\"")); + assert!(response.contains("transport")); + assert!(response.contains("${escapeHtml(dashboardSocketUrl())} · ${escapeHtml(stream.label)}")); + assert!(response.contains("Poll fallback: ${escapeHtml(ENDPOINTS.state)}")); + assert!(response.contains("snapshot")); + assert!(response.contains("Dashboard WebSocket connected.")); + assert!(response.contains("const snapshotFreshnessRow = snapshotFreshness")); + assert!(response.contains("return null;")); + assert!(response.contains("const staleByAge = ageSeconds != null && ageSeconds >= 30;")); + assert!(response.contains("const staleByReadiness = readiness.label === \"Snapshot stale\";")); + assert!(response.contains("data-tone=\"${escapeHtml(snapshotFreshness.tone)}\"")); + assert!(response.contains("Published ${formatTimestamp(snapshotPublishedAt)}")); + assert!(response.contains("formatRelativeTimestamp(snapshotPublishedAt)")); + assert!(response.contains("snapshotPublishedAt = stateResult.value.snapshotPublishedAt")); + assert!( + response.contains("renderHeader(snapshot, readiness, notices, snapshotPublishedAt, snapshotError)") + ); + assert!(response.contains(".transport-meta")); + assert!(response.contains("max-width: min(42vw, 320px);")); + assert!(!response.contains("Auto-refresh")); + assert!(!response.contains("Diagnostics")); +} + +#[test] +fn operator_dashboard_active_freshness_prefers_live_activity_source() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.contains("function activeRunFreshness(run)")); + assert!(response.contains("source: \"last_run_activity_at\"")); + assert!(response.contains("source: \"none\"")); + assert!(!response.contains("source: \"updated_at\"")); + assert!(response.contains("function formatRelativeTimestamp(value)")); + assert!(response.contains("function activeRunTelemetryFacts(run)")); + assert!(response.contains("function renderActiveTelemetryLine(run)")); + assert!(response.contains("activity-line")); + assert!( + response + .contains("freshness.timestamp ? formatter(freshness.timestamp) : \"not captured\"") + ); + assert!(!response.contains("Last ${freshness.sourceLabel}")); + assert!(!response.contains("Latest ${freshness.sourceLabel}")); + assert!(!response.contains("renderTimingStrip(run)")); + assert!(response.contains("field(\"Freshness source\", activeRunFreshnessSource(run))")); + assert!( + response.contains("field(\"Lane activity\", formatTimestamp(run.last_run_activity_at))") + ); + assert!(response.contains("field(\"Updated\", formatTimestamp(run.updated_at))")); +} + +#[test] +fn operator_dashboard_uses_shared_protocol_activity_summary() { + let response = dashboard_response(); + + assert!(response.contains("function protocolActivity(run)")); + assert!(response.contains("function protocolActivityFocus(run)")); + assert!(response.contains("function protocolActivityRecentSummary(run)")); + assert!(response.contains("function protocolActivityDebugSummary(run)")); + assert!(response.contains("facts.push([\"time going to\", focus]);")); + assert!(response.contains("return \"approval/user input\";")); + assert!(response.contains("return \"protocol idleness\";")); + assert!(response.contains("field(\"Protocol activity\", protocolActivityDebugSummary(run))")); + assert!(response.contains("field(\"Rate limit\", protocolActivityRateLimit(run))")); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/history.rs b/apps/decodex/src/orchestrator/tests/operator/status/history.rs new file mode 100644 index 00000000..1cfa1d02 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/history.rs @@ -0,0 +1,360 @@ +#[test] +fn operator_status_history_limit_applies_after_active_runs_are_split_out() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let active_issue = sample_issue_with_sort_fields( + "issue-1", + "PUB-101", + "Todo", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let failed_issue = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(3), + "2026-03-13T04:17:17.133Z", + ); + + state_store + .record_run_attempt("run-active", &active_issue.id, 1, "running") + .expect("active run should record"); + state_store + .upsert_lease("pubfi", &active_issue.id, "run-active", "In Progress") + .expect("active lease should record"); + state_store + .upsert_worktree( + "pubfi", + &active_issue.id, + "x/pubfi-pub-101", + &config.worktree_root().join(&active_issue.identifier).display().to_string(), + ) + .expect("active worktree should record"); + state_store + .record_run_attempt("run-failed", &failed_issue.id, 1, "failed") + .expect("failed run should record"); + state_store + .upsert_worktree( + "pubfi", + &failed_issue.id, + "x/pubfi-pub-102", + &config.worktree_root().join(&failed_issue.identifier).display().to_string(), + ) + .expect("failed worktree should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 1) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(snapshot.run_limit, 1); + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.recent_runs.len(), 2); + assert_eq!(snapshot.history_lanes.len(), 1); + assert_eq!(snapshot.history_lanes[0].attempt_count, 1); + assert!(rendered.contains( + "Run ledger shown: 1 issue lanes from 1 history attempts (running lanes inline)" + )); + assert_eq!(rendered.matches("run_id: run-active").count(), 1); + assert_eq!(rendered.matches("run_id: run-failed").count(), 1); + + let history_index = rendered.find("Run Ledger").expect("history section should render"); + let failed_index = rendered.find("run_id: run-failed").expect("failed run should render"); + + assert!( + failed_index > history_index, + "non-running history run should remain visible after running lane overlap is hidden" + ); +} + +#[test] +fn operator_status_history_lanes_group_attempts_by_issue() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let first_issue = sample_issue_with_sort_fields( + "issue-1", + "XY-323", + "Done", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let second_issue = sample_issue_with_sort_fields( + "issue-2", + "XY-330", + "Done", + &[], + Some(3), + "2026-03-13T04:17:17.133Z", + ); + + state_store + .record_run_attempt("xy-323-attempt-1-1777361523", &first_issue.id, 1, "failed") + .expect("first attempt should record"); + state_store + .record_run_attempt("xy-323-attempt-2-1777361550", &first_issue.id, 2, "succeeded") + .expect("second attempt should record"); + state_store + .record_run_attempt("xy-330-attempt-1-1777361600", &second_issue.id, 1, "succeeded") + .expect("other issue attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &first_issue.id, + "x/decodex-xy-323", + &config.worktree_root().join(&first_issue.identifier).display().to_string(), + ) + .expect("first issue worktree should record"); + state_store + .upsert_worktree( + "pubfi", + &second_issue.id, + "x/decodex-xy-330", + &config.worktree_root().join(&second_issue.identifier).display().to_string(), + ) + .expect("second issue worktree should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + let grouped_lane = snapshot + .history_lanes + .iter() + .find(|lane| lane.issue_key == "XY-323") + .expect("first issue should have a grouped history lane"); + + assert_eq!(snapshot.recent_runs.len(), 3); + assert_eq!(snapshot.history_lanes.len(), 2); + assert_eq!(grouped_lane.attempt_count, 2); + assert_eq!(grouped_lane.latest_run.run_id, "xy-323-attempt-2-1777361550"); + assert!(rendered.contains("Run ledger shown: 2 issue lanes from 3 history attempts")); + assert!(rendered.contains("issue: XY-323")); + assert!(rendered.contains("attempts: 2")); + assert!(rendered.contains("attempt_timeline")); +} + +#[test] +fn operator_status_project_waiting_count_ignores_superseded_waiting_attempts() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "XY-451", + "Done", + &[], + Some(3), + "2026-05-03T11:48:16Z", + ); + + state_store + .record_run_attempt("xy-451-attempt-1-1777791228", &issue.id, 1, "stalled") + .expect("stalled attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/decodex-xy-451", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should record"); + state_store + .record_run_attempt("xy-451-attempt-4-1777808209", &issue.id, 4, "succeeded") + .expect("successful attempt should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let grouped_lane = snapshot.history_lanes.first().expect("history lane should exist"); + + assert_eq!(grouped_lane.attempt_count, 2); + assert_eq!(grouped_lane.latest_run.run_id, "xy-451-attempt-4-1777808209"); + assert_eq!(snapshot.projects[0].waiting_lane_count, 0); +} + +#[test] +fn operator_status_project_connector_state_ignores_superseded_retry_backoff_attempts() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "XY-452", + "Done", + &[], + Some(3), + "2026-05-03T11:49:16Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + + state_store + .record_run_attempt("xy-452-attempt-1-1777791228", &issue.id, 1, "failed") + .expect("failed attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/decodex-xy-452", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_retry_schedule( + &worktree_path, + "xy-452-attempt-1-1777791228", + 1, + "failure", + OffsetDateTime::now_utc().unix_timestamp() + 60, + ) + .expect("retry schedule marker should write"); + + state_store + .record_run_attempt("xy-452-attempt-2-1777808209", &issue.id, 2, "succeeded") + .expect("successful attempt should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let grouped_lane = snapshot.history_lanes.first().expect("history lane should exist"); + + assert_eq!(grouped_lane.latest_run.run_id, "xy-452-attempt-2-1777808209"); + assert_eq!(snapshot.projects[0].waiting_lane_count, 0); + assert_eq!(snapshot.projects[0].connector_state, "ok"); +} + +#[test] +fn live_operator_history_lanes_prefer_linear_ledger_outcome() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut issue = sample_issue_with_sort_fields( + "issue-1", + "XY-355", + "Done", + &[], + Some(3), + "2026-04-29T10:11:00Z", + ); + + issue.title = String::from("Keep completed run rows self describing"); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "y/decodex-xy-355", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember project ownership"); + state_store + .record_run_attempt("xy-355-attempt-1-1777527013", &issue.id, 1, "failed") + .expect("failed attempt should record"); + state_store + .record_run_attempt("xy-355-attempt-2-1777527613", &issue.id, 2, "succeeded") + .expect("successful attempt should record"); + state_store + .clear_worktree(&issue.id) + .expect("completed lane cleanup should clear local worktree"); + tracker + .issue_comments + .borrow_mut() + .insert(issue.id.clone(), successful_linear_execution_history_comments(&issue)); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let rendered = orchestrator::render_operator_status(&snapshot); + let outcome_index = rendered.find("outcome: closeout").expect("ledger outcome should render"); + let local_index = rendered.find("latest_run_id:").expect("local attempt debug should render"); + + assert_eq!(snapshot.recent_runs.len(), 2); + assert_eq!(snapshot.history_lanes.len(), 1); + assert!(snapshot.recent_runs.iter().all(|run| run.project_id == TEST_SERVICE_ID)); + assert!(snapshot.recent_runs.iter().all(|run| { + run.issue_identifier.as_deref() == Some("XY-355") + && run.title.as_deref() == Some("Keep completed run rows self describing") + })); + assert_eq!(lane.project_id, TEST_SERVICE_ID); + assert_eq!(lane.issue_identifier.as_deref(), Some("XY-355")); + assert_eq!(lane.title.as_deref(), Some("Keep completed run rows self describing")); + assert_eq!(lane.latest_run.issue_identifier.as_deref(), Some("XY-355")); + assert_eq!(lane.latest_run.title.as_deref(), Some("Keep completed run rows self describing")); + assert_eq!(lane.ledger_outcome.ledger_status, "present"); + assert_eq!(lane.ledger_outcome.final_outcome, "closeout"); + assert_eq!( + lane.ledger_outcome.pr_url.as_deref(), + Some("https://github.com/hack-ink/decodex/pull/355") + ); + assert_eq!( + lane.ledger_outcome.commit_sha.as_deref(), + Some("2222222222222222222222222222222222222222") + ); + assert_eq!(lane.ledger_outcome.closeout_status.as_deref(), Some("Done")); + assert_eq!(lane.ledger_outcome.needs_attention_reason, None); + assert_eq!(lane.ledger_outcome.lifecycle_elapsed_seconds, Some(600)); + assert!( + outcome_index < local_index, + "durable ledger outcome should be primary before local attempt details" + ); + assert!(rendered.contains("ledger_status: present")); + assert!(rendered.contains("pr_url: https://github.com/hack-ink/decodex/pull/355")); + assert!(rendered.contains("commit_sha: 2222222222222222222222222222222222222222")); + assert!(rendered.contains("closeout_status: Done")); + assert!(rendered.contains("lifecycle_elapsed_seconds: 600")); + assert!(!rendered.contains("pr_url: none")); +} + +#[test] +fn live_operator_history_lanes_require_linear_execution_ledger_records() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "XY-356", + "Done", + &[], + Some(3), + "2026-04-29T10:11:00Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "y/decodex-xy-356", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember project ownership"); + state_store + .record_run_attempt("xy-356-attempt-1", &issue.id, 1, "succeeded") + .expect("successful attempt should record"); + state_store.clear_worktree(&issue.id).expect("completed lane cleanup should clear local worktree"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(tracker.comment_queries.borrow().as_slice(), slice::from_ref(&issue.id)); + assert_eq!(lane.ledger_outcome.ledger_status, "missing"); + assert_eq!(lane.ledger_outcome.final_outcome, "execution_ledger_missing"); + assert_eq!(lane.ledger_outcome.record_count, 0); + assert_eq!( + lane.ledger_outcome.summary.as_deref(), + Some("No decodex.linear_execution_event records are available for this history lane.") + ); + assert!(rendered.contains("ledger_status: missing")); + assert!(rendered.contains("outcome: execution_ledger_missing")); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/http.rs b/apps/decodex/src/orchestrator/tests/operator/status/http.rs new file mode 100644 index 00000000..c770bab6 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/http.rs @@ -0,0 +1,1170 @@ +use std::net::SocketAddr; + +#[test] +fn operator_state_endpoint_serves_snapshot_json() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 4, + last_event_type: "item/tool/call/response", + child_agent_activity: Some(&ChildAgentActivitySummary { + buckets: vec![state::ChildAgentActivityBucket { + name: String::from("Browser/Image"), + wall_seconds: 41, + event_count: 4, + tool_call_count: 2, + input_tokens: 0, + output_tokens: 0, + output_bytes: 240_000, + }], + current_bucket: Some(String::from("Model")), + current_detail: Some(String::from("waiting after tool output")), + current_started_unix_epoch: None, + current_elapsed_seconds: Some(0), + wall_seconds: 693, + event_count: 4, + tool_call_count: 2, + input_tokens_current: Some(135_000), + input_tokens_max: Some(135_000), + input_tokens_cumulative: 6_510_000, + output_tokens_cumulative: 18_000, + largest_tool_output_bytes: Some(240_000), + largest_tool_output_tool: Some(String::from("view_image")), + large_output_warnings: vec![String::from( + "view_image repeated 2 large outputs; largest 240000 bytes", + )], + }), + protocol_activity: None, + }, + ) + .expect("child activity marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let snapshot_json = serde_json::to_vec(&snapshot).expect("snapshot json should serialize"); + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_STATE_ENDPOINT_PATH + ) + .as_bytes(), + Some(snapshot_json.as_slice()), + OperatorSnapshotReadiness::Ready, + ) + .expect("response build should succeed"), + ) + .expect("response should be utf-8"); + let (status_line, body) = + response.split_once("\r\n").expect("response should contain a status line"); + let body = body.split_once("\r\n\r\n").expect("response should contain a body").1; + let served_snapshot: Value = serde_json::from_str(body).expect("body should be valid json"); + + assert_eq!(status_line, "HTTP/1.1 200 OK"); + assert_eq!(served_snapshot["project_id"], "pubfi"); + assert_eq!(served_snapshot["run_limit"], 10); + assert_eq!(served_snapshot["active_runs"][0]["run_id"], "run-1"); + assert_eq!(served_snapshot["active_runs"][0]["status"], "running"); + assert_eq!(served_snapshot["active_runs"][0]["attempt_status"], "running"); + assert_eq!(served_snapshot["active_runs"][0]["phase"], "executing"); + assert_eq!(served_snapshot["active_runs"][0]["queue_lease_state"], "held"); + assert_eq!(served_snapshot["active_runs"][0]["execution_liveness"], "process_alive"); + assert_eq!( + served_snapshot["active_runs"][0]["child_agent_activity"]["buckets"][0]["name"], + "Browser/Image" + ); + assert_eq!( + served_snapshot["active_runs"][0]["child_agent_activity"]["input_tokens_max"], + 135_000 + ); + assert_eq!(served_snapshot["queued_candidates"], Value::Array(Vec::new())); + assert_eq!(served_snapshot["worktrees"][0]["worktree_path"], ".worktrees/PUB-101"); +} + +#[test] +fn operator_state_endpoint_serializes_closed_queue_classification() { + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: vec![], + recent_runs: vec![], + history_lanes: vec![], + queued_candidates: vec![orchestrator::OperatorQueuedIssueStatus { + issue_id: String::from("issue-closed"), + issue_identifier: String::from("PUB-104"), + title: String::from("Retire closed queue residue"), + state: String::from("Done"), + priority: Some(1), + created_at: String::from("2026-03-14T09:58:00Z"), + classification: String::from("closed"), + reason: String::from("terminal_state"), + attention: None, + blocker_identifiers: vec![], + }], + worktrees: vec![], + post_review_lanes: vec![], + }; + let snapshot_json = serde_json::to_vec(&snapshot).expect("snapshot json should serialize"); + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_STATE_ENDPOINT_PATH + ) + .as_bytes(), + Some(snapshot_json.as_slice()), + OperatorSnapshotReadiness::Ready, + ) + .expect("response build should succeed"), + ) + .expect("response should be utf-8"); + let body = response.split_once("\r\n\r\n").expect("response should contain a body").1; + let served_snapshot: Value = serde_json::from_str(body).expect("body should be valid json"); + + assert_eq!(served_snapshot["queued_candidates"][0]["classification"], "closed"); + assert_eq!(served_snapshot["queued_candidates"][0]["reason"], "terminal_state"); +} + +#[test] +fn operator_state_endpoint_serves_dashboard_html_from_root_and_dashboard_route() { + for path in [ + OPERATOR_DASHBOARD_ENDPOINT_PATH, + OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH, + ] { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard response should build"), + ) + .expect("dashboard response should be utf-8"); + + assert!(response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(response.contains("Content-Type: text/html; charset=utf-8")); + assert!(response.contains("Decodex")); + assert!(response.contains("

Decodex

")); + assert!(response.contains("Delivery flow")); + assert!(response.contains("flow-queue")); + assert!(response.contains("Intake")); + assert!(response.contains("Landing")); + assert!(response.contains("

Projects

")); + assert!(response.contains("Registered projects")); + assert!(response.contains("projectRegistrationCommand")); + assert!( + response.contains("decodex project add ~/.codex/decodex/projects/") + ); + assert!(response.contains("Projects must be registered explicitly")); + assert!(response.contains("does not discover ~/.codex history or repo-local files")); + assert!(response.contains("data-detail-key")); + assert!(response.contains("notice-dock")); + assert!(response.contains("Notices")); + assert!(response.contains("notice-panel")); + assert!(response.contains("State fetch")); + assert!(response.contains("Snapshot warning")); + assert!(response.contains("Tracker sync paused")); + assert!(response.contains("connector_backoffs")); + assert!(response.contains("Sync backoff")); + assert!(response.contains("project_id")); + assert!(response.contains("retry_after_seconds")); + assert!(response.contains("reset_at")); + assert!(response.contains("sync_phase")); + assert!(!response.contains("error-banner")); + assert!(!response.contains("metric-active")); + assert!(!response.contains("Queued issue -> reviewed change -> landed branch")); + assert!(response.contains("Running Lanes")); + assert!(response.contains("Intake Queue")); + assert!(response.contains("Waiting for capacity")); + assert!(response.contains("Review & Landing")); + assert!(response.contains("Run History")); + assert!(response.contains("historyLedgerOutcome")); + assert!(response.contains("Run history unavailable")); + assert!(response.contains("renderHistoryLedgerFacts")); + assert!(response.contains("Recovery Worktrees")); + assert!(response.contains("Lane activity telemetry")); + assert!(response.contains("agent idle")); + assert!(response.contains("Child agent")); + assert!(response.contains("Agent now")); + assert!(response.contains("Current window")); + assert!(response.contains("Peak window")); + assert!(!response.contains("same as current")); + assert!(response.contains("Cumulative input")); + assert!(response.contains("Current context window from the latest child-agent event.")); + assert!(response.contains("Total input tokens processed across child-agent events.")); + assert!(response.contains("child_agent_activity")); + assert!(response.contains("renderChildAgentBreakdown")); + assert!(response.contains("Debug details")); + assert!(response.contains("already running")); + assert!(!response.contains("running laness")); + assert!(!response.contains("active-echo")); + assert!(response.contains("fold-panel")); + assert!(response.contains("data-fold-key=\"panel:worktrees\"")); + assert!(response.contains("data-fold-key=\"panel:recent\"")); + assert!(response.contains("cursor: pointer;")); + assert!(response.contains("animateDetail(details, !details.open)")); + assert!(response.contains("width: min(380px, calc(100vw - 36px));")); + assert!(response.contains(".notice-item p")); + assert!(response.contains("font-size: 13px;")); + assert!(!response.contains(".fold-panel.is-empty .fold-indicator")); + assert!(!response.contains("details.classList.contains(\"is-empty\")")); + assert!(!response.contains("Operator views")); + assert!(!response.contains("Command Brief")); + assert!(!response.contains("Intake Pressure")); + assert!(!response.contains("Landing Readiness")); + assert!(response.contains("/state")); + assert!(response.contains("/readyz")); + assert!(response.contains("/dashboard/control")); + assert!(response.contains("WebSocket")); + assert!(response.contains("applyDashboardRunActivity")); + assert!(response.contains("sendDashboardControl")); + assert!(response.contains("data-dashboard-control=\"focusProject\"")); + assert!(response.contains("data-dashboard-control=\"retryRun\"")); + assert!(response.contains("controlAck")); + assert!(!response.contains("Last updated: none")); + assert!(!response.contains("Auto-refresh")); + assert!(!response.contains("

Project Scope

")); + assert!(!response.contains("Projects appear on the first state update")); + assert!(!response.contains("Diagnostics")); + assert!(!response.contains("State JSON")); + assert!(!response.contains("Ready probe")); + assert!(!response.contains("Live probe")); + assert!(!response.contains("/livez")); + } +} + +#[test] +fn operator_state_endpoint_rejects_dashboard_websocket_without_upgrade() { + let response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_WS_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("dashboard websocket response should build"), + ) + .expect("dashboard websocket response should be utf-8"); + + assert!(response.starts_with("HTTP/1.1 426 Upgrade Required\r\n")); + assert!(response.contains("Upgrade: websocket")); + assert!(response.ends_with("websocket upgrade required")); +} + +#[test] +fn operator_dashboard_websocket_pushes_broadcast_events() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + let mut buffer = [0_u8; 2_048]; + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + assert!(response.contains("Upgrade: websocket")); + assert!(response.contains("Connection: Upgrade")); + assert!(response.contains("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=")); + + dashboard_events.broadcast( + "snapshot", + serde_json::json!({ + "snapshotPublishedAtUnixEpoch": 1_774_000_000_i64, + "snapshot": { "project_id": "pubfi" }, + }), + ); + + let deadline = Instant::now() + Duration::from_secs(1); + let payload = loop { + assert!(Instant::now() < deadline, "websocket should send broadcast events"); + + if frame.is_empty() { + let event_bytes = client.read(&mut buffer).expect("client should read broadcast event"); + + frame.extend_from_slice(&buffer[..event_bytes]); + } + + if let Some((payload, consumed)) = websocket_text_payload(&frame) { + let payload: Value = + serde_json::from_slice(payload).expect("event payload should be json"); + + frame.drain(..consumed); + + if payload["type"] == "snapshot" { + break payload; + } + } + }; + + assert_eq!(payload["type"], "snapshot"); + assert_eq!(payload["payload"]["snapshot"]["project_id"], "pubfi"); + + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +#[test] +fn operator_dashboard_websocket_accepts_subscription_and_project_pause_control() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let registration = ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + true, + "test-fingerprint", + ); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + + state_store.upsert_project(®istration).expect("project should register"); + + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"subscribe","requestId":"sub-1","projectId":"pubfi"}"#, + )) + .expect("client should send subscribe"); + + let subscribe_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "sub-1" + }); + + assert_eq!(subscribe_ack["payload"]["accepted"], true); + assert_eq!(subscribe_ack["payload"]["subscription"]["projectId"], "pubfi"); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"pause-1","action":"pauseProject","projectId":"pubfi"}"#, + )) + .expect("client should send pause"); + + let pause_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "pause-1" + }); + + assert_eq!(pause_ack["payload"]["accepted"], true); + assert_eq!(pause_ack["payload"]["status"], "paused"); + assert!( + !state_store + .list_projects() + .expect("projects should list") + .into_iter() + .find(|project| project.service_id() == "pubfi") + .expect("project should remain registered") + .enabled() + ); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"resume-1","action":"resumeProject","projectId":"pubfi"}"#, + )) + .expect("client should send resume"); + + let resume_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "resume-1" + }); + + assert_eq!(resume_ack["payload"]["accepted"], true); + assert_eq!(resume_ack["payload"]["status"], "resumed"); + assert!( + state_store + .list_projects() + .expect("projects should list") + .into_iter() + .find(|project| project.service_id() == "pubfi") + .expect("project should remain registered") + .enabled() + ); + + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +#[test] +fn operator_dashboard_websocket_controls_focus_and_clear_subscription() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"focus-1","action":"focus","projectId":"pubfi","issueId":"PUB-101","runId":"run-1"}"#, + )) + .expect("client should send focus"); + + let focus_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "focus-1" + }); + + assert_eq!(focus_ack["payload"]["accepted"], true); + assert_eq!(focus_ack["payload"]["status"], "focused"); + assert_eq!(focus_ack["payload"]["subscription"]["projectId"], "pubfi"); + assert_eq!(focus_ack["payload"]["subscription"]["issueId"], "PUB-101"); + assert_eq!(focus_ack["payload"]["subscription"]["runId"], "run-1"); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"clear-1","action":"clearFocus"}"#, + )) + .expect("client should send clear focus"); + + let clear_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "clear-1" + }); + + assert_eq!(clear_ack["payload"]["accepted"], true); + assert_eq!(clear_ack["payload"]["status"], "focused"); + assert_eq!(clear_ack["payload"]["subscription"]["projectId"], Value::Null); + assert_eq!(clear_ack["payload"]["subscription"]["issueId"], Value::Null); + assert_eq!(clear_ack["payload"]["subscription"]["runId"], Value::Null); + + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +#[test] +fn operator_dashboard_websocket_filters_run_activity_by_subscription() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"subscribe","requestId":"sub-filter","projectId":"pubfi","runId":"run-2"}"#, + )) + .expect("client should send subscription"); + + let _subscribe_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "sub-filter" + }); + + dashboard_events.broadcast( + "runActivity", + serde_json::json!({ + "emittedAtUnixEpoch": 1_774_000_010_i64, + "activeRuns": [ + { "project_id": "pubfi", "issue_id": "PUB-101", "run_id": "run-1" }, + { "project_id": "pubfi", "issue_id": "PUB-102", "run_id": "run-2" }, + { "project_id": "rsnap", "issue_id": "RS-1", "run_id": "run-2" } + ] + }), + ); + + let activity = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "runActivity" + }); + let active_runs = activity["payload"]["activeRuns"] + .as_array() + .expect("active runs should list"); + + assert_eq!(active_runs.len(), 1); + assert_eq!(active_runs[0]["project_id"], "pubfi"); + assert_eq!(active_runs[0]["issue_id"], "PUB-102"); + assert_eq!(active_runs[0]["run_id"], "run-2"); + + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +#[test] +fn operator_dashboard_websocket_retry_control_uses_registered_project_config() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let registration = ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + true, + "test-fingerprint", + ); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + let launcher_calls = dashboard_retry_launcher_calls_for_test(); + + state_store.upsert_project(®istration).expect("project should register"); + launcher_calls + .lock() + .expect("dashboard retry launcher calls should not be poisoned") + .clear(); + + let _launcher_guard = + orchestrator::install_dashboard_retry_launcher_for_test(fake_dashboard_retry_launcher); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"retry-1","action":"retryRun","projectId":"pubfi","issueId":"PUB-101","runId":"run-1"}"#, + )) + .expect("client should send retry"); + + let retry_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "retry-1" + }); + + assert_eq!(retry_ack["payload"]["accepted"], true); + assert_eq!(retry_ack["payload"]["status"], "retry_started"); + assert_eq!(retry_ack["payload"]["projectId"], "pubfi"); + assert_eq!(retry_ack["payload"]["issueId"], "PUB-101"); + assert_eq!(retry_ack["payload"]["runId"], "run-1"); + assert!( + retry_ack["payload"]["message"] + .as_str() + .expect("retry ack message should be text") + .contains("process 4242") + ); + + let calls = launcher_calls + .lock() + .expect("dashboard retry launcher calls should not be poisoned"); + + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, service_config_path(config.repo_root())); + assert_eq!(calls[0].1, "PUB-101"); + + drop(calls); + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +#[test] +fn operator_dashboard_websocket_retry_control_reports_validation_errors() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let dashboard_events = DashboardEventHub::default(); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server_dashboard_events = dashboard_events.clone(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &server_dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("websocket handler should complete after client disconnect"); + }); + let (mut client, response, mut frame) = open_dashboard_websocket_client(address); + + assert!(response.starts_with("HTTP/1.1 101 Switching Protocols\r\n")); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"missing-project","action":"pauseProject"}"#, + )) + .expect("client should send missing-project control"); + + let missing_project_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "missing-project" + }); + + assert_eq!(missing_project_ack["payload"]["accepted"], false); + assert_eq!(missing_project_ack["payload"]["status"], "missing_project"); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"missing-issue","action":"retryRun","projectId":"pubfi"}"#, + )) + .expect("client should send missing-issue control"); + + let missing_issue_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "missing-issue" + }); + + assert_eq!(missing_issue_ack["payload"]["accepted"], false); + assert_eq!(missing_issue_ack["payload"]["status"], "missing_issue"); + + client + .write_all(&websocket_client_text_frame( + r#"{"type":"control","requestId":"unknown-project","action":"retryRun","projectId":"missing","issueId":"PUB-101"}"#, + )) + .expect("client should send unknown-project control"); + + let unknown_project_ack = read_websocket_json_until(&mut client, &mut frame, |payload| { + payload["type"] == "controlAck" && payload["payload"]["requestId"] == "unknown-project" + }); + + assert_eq!(unknown_project_ack["payload"]["accepted"], false); + assert_eq!(unknown_project_ack["payload"]["status"], "failed"); + assert!( + unknown_project_ack["payload"]["message"] + .as_str() + .expect("retry ack message should be text") + .contains("not registered") + ); + + drop(client); + + dashboard_events.close_clients_for_test(); + server.join().expect("server thread should complete"); +} + +fn websocket_text_payload(frame: &[u8]) -> Option<(&[u8], usize)> { + if frame.len() < 2 || frame[0] != 0x81 { + return None; + } + + let payload_length_marker = frame[1] & 0x7f; + let (payload_offset, payload_length): (usize, usize) = match payload_length_marker { + length @ 0..=125 => (2_usize, usize::from(length)), + 126 => { + if frame.len() < 4 { + return None; + } + + (4_usize, usize::from(u16::from_be_bytes([frame[2], frame[3]]))) + }, + 127 => { + if frame.len() < 10 { + return None; + } + + let length = u64::from_be_bytes([ + frame[2], frame[3], frame[4], frame[5], frame[6], frame[7], frame[8], frame[9], + ]); + let Ok(length) = usize::try_from(length) else { + return None; + }; + + (10_usize, length) + }, + _ => return None, + }; + let payload_end = payload_offset.checked_add(payload_length)?; + + (frame.len() >= payload_end).then(|| (&frame[payload_offset..payload_end], payload_end)) +} + +fn open_dashboard_websocket_client(address: SocketAddr) -> (TcpStream, String, Vec) { + let mut client = TcpStream::connect(address).expect("client should connect"); + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 2_048]; + + client + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("client timeout should configure"); + client + .write_all( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n", + orchestrator::OPERATOR_DASHBOARD_WS_ENDPOINT_PATH + ) + .as_bytes(), + ) + .expect("client should write request"); + + let header_end = loop { + let header_bytes = client.read(&mut buffer).expect("client should read stream headers"); + + bytes.extend_from_slice(&buffer[..header_bytes]); + + if let Some(index) = bytes.windows(4).position(|window| window == b"\r\n\r\n") { + break index + 4; + } + }; + let response = String::from_utf8(bytes[..header_end].to_vec()) + .expect("headers should be utf-8"); + let frame = bytes[header_end..].to_vec(); + + (client, response, frame) +} + +fn websocket_client_text_frame(payload: &str) -> Vec { + let payload = payload.as_bytes(); + let mask = [0x11_u8, 0x22, 0x33, 0x44]; + let mut frame = Vec::new(); + + frame.push(0x81); + + match payload.len() { + length @ 0..=125 => frame.push(0x80 | length as u8), + length @ 126..=65_535 => { + frame.push(0x80 | 126); + frame.extend_from_slice(&(length as u16).to_be_bytes()); + }, + length => { + frame.push(0x80 | 127); + frame.extend_from_slice(&(length as u64).to_be_bytes()); + }, + } + + frame.extend_from_slice(&mask); + frame.extend(payload.iter().enumerate().map(|(index, byte)| byte ^ mask[index % mask.len()])); + + frame +} + +fn dashboard_retry_launcher_calls_for_test() -> &'static Mutex> { + static CALLS: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + + CALLS.get_or_init(|| Mutex::new(Vec::new())) +} + +fn fake_dashboard_retry_launcher(config_path: &Path, issue_id: &str) -> Result { + dashboard_retry_launcher_calls_for_test() + .lock() + .expect("dashboard retry launcher calls should not be poisoned") + .push((config_path.to_path_buf(), issue_id.to_owned())); + + Ok(4_242) +} + +fn read_websocket_json_until( + client: &mut TcpStream, + frame: &mut Vec, + matches: impl Fn(&Value) -> bool, +) -> Value { + let deadline = Instant::now() + Duration::from_secs(1); + let mut buffer = [0_u8; 2_048]; + + loop { + assert!(Instant::now() < deadline, "websocket should send expected event"); + + if frame.is_empty() { + let event_bytes = client.read(&mut buffer).expect("client should read websocket event"); + + frame.extend_from_slice(&buffer[..event_bytes]); + } + + if let Some((payload, consumed)) = websocket_text_payload(frame) { + let payload: Value = + serde_json::from_slice(payload).expect("event payload should be json"); + + frame.drain(..consumed); + + if matches(&payload) { + return payload; + } + } else { + let event_bytes = client.read(&mut buffer).expect("client should continue websocket event"); + + frame.extend_from_slice(&buffer[..event_bytes]); + } + } +} + +#[test] +fn operator_dashboard_run_activity_event_summarizes_active_runs() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let registration = ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + true, + "test-fingerprint", + ); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + let protocol_activity = ProtocolActivitySummary { + turn_status: Some(String::from("running")), + waiting_reason: Some(String::from("model")), + rate_limit_status: None, + recent_events: vec![state::ProtocolActivityEventSummary { + event_type: String::from("item/model/delta"), + category: String::from("model"), + detail: Some(String::from("received model delta")), + }], + }; + let account = CodexAccountActivitySummary { + account_fingerprint: String::from("acct-1"), + status: String::from("available"), + refresh_status: String::from("ok"), + ..Default::default() + }; + + state_store.upsert_project(®istration).expect("project should register"); + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 7, + last_event_type: "item/model/delta", + child_agent_activity: None, + protocol_activity: Some(&protocol_activity), + }, + ) + .expect("protocol activity marker should write"); + state::write_run_account_marker( + &worktree_path, + &CodexAccountMarker { + run_id: "run-1", + attempt_number: 1, + account: &account, + accounts: slice::from_ref(&account), + }, + ) + .expect("account marker should write"); + + let event = + orchestrator::build_operator_run_activity_event(&state_store).expect("event should build"); + let message = orchestrator::dashboard_websocket_message( + event.event.event_type, + &event.event.payload, + ) + .expect("event should serialize"); + let (payload, _consumed) = websocket_text_payload(&message).expect("event should be a text frame"); + let payload: Value = serde_json::from_slice(payload).expect("event data should be json"); + let data = &payload["payload"]; + + assert_eq!(payload["type"], "runActivity"); + assert_eq!(data["activeRuns"][0]["run_id"], "run-1"); + assert_eq!(data["activeRuns"][0]["project_id"], "pubfi"); + assert_eq!(data["activeRuns"][0]["protocol_activity"]["waiting_reason"], "model"); + assert_eq!(data["activeRuns"][0]["account"]["account_fingerprint"], "acct-1"); + assert_eq!(data["activeRuns"][0]["accounts"][0]["account_fingerprint"], "acct-1"); +} + +#[test] +fn operator_state_endpoint_reads_complete_headers_before_parsing() { + const SNAPSHOT_UNIX_EPOCH: i64 = 1_774_000_000; + + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot { + snapshot_json: Some(br#"{"status":"ok"}"#.to_vec()), + last_publish_unix_epoch: Some(SNAPSHOT_UNIX_EPOCH), + })); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + let dashboard_events = DashboardEventHub::default(); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("handler should accept segmented headers"); + }); + let mut client = TcpStream::connect(address).expect("client should connect"); + let mut response = String::new(); + + client.write_all(b"GET /st").expect("client should write first request fragment"); + + thread::sleep(Duration::from_millis(10)); + + client + .write_all(b"ate HTTP/1.1\r\nHost: localhost\r\n\r\n") + .expect("client should write second request fragment"); + client.shutdown(Shutdown::Write).expect("client should close the request body stream"); + client.read_to_string(&mut response).expect("client should read response"); + server.join().expect("server thread should complete"); + + assert!(response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(response.contains(&format!( + "X-Decodex-Snapshot-Unix-Epoch: {SNAPSHOT_UNIX_EPOCH}\r\n" + ))); + assert!(response.ends_with("{\"status\":\"ok\"}")); +} + +#[test] +fn operator_state_endpoint_livez_ignores_poisoned_snapshot_lock() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address should resolve"); + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot { + snapshot_json: Some(br#"{"status":"ok"}"#.to_vec()), + last_publish_unix_epoch: Some(OffsetDateTime::now_utc().unix_timestamp()), + })); + let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); + let poisoned_snapshot = Arc::clone(&snapshot); + let _ = panic::catch_unwind(move || { + let _guard = poisoned_snapshot.lock().expect("snapshot lock should acquire"); + + panic!("poison snapshot lock"); + }); + let server_snapshot = Arc::clone(&snapshot); + let server_state_store = Arc::clone(&state_store); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().expect("listener should accept a connection"); + let dashboard_events = DashboardEventHub::default(); + + orchestrator::handle_operator_state_endpoint_connection( + stream, + &server_snapshot, + &dashboard_events, + &server_state_store, + Duration::from_secs(30), + ) + .expect("live probe should not require snapshot lock"); + }); + let mut client = TcpStream::connect(address).expect("client should connect"); + let mut response = String::new(); + + client + .write_all( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_LIVE_ENDPOINT_PATH + ) + .as_bytes(), + ) + .expect("client should write request"); + client.shutdown(Shutdown::Write).expect("client should close the request body stream"); + client.read_to_string(&mut response).expect("client should read response"); + server.join().expect("server thread should complete"); + + assert!(response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(response.ends_with("ok")); +} + +#[test] +fn operator_state_endpoint_serves_liveness_and_readiness_probes() { + let live_response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_LIVE_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("live response should build"), + ) + .expect("live response should be utf-8"); + + assert!(live_response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(live_response.ends_with("ok")); + + let ready_unavailable = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_READY_ENDPOINT_PATH + ) + .as_bytes(), + None, + OperatorSnapshotReadiness::SnapshotUnavailable, + ) + .expect("ready response should build"), + ) + .expect("ready response should be utf-8"); + + assert!(ready_unavailable.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); + assert!(ready_unavailable.ends_with("snapshot_unavailable")); + + let ready_response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_READY_ENDPOINT_PATH + ) + .as_bytes(), + Some(br#"{"status":"ok"}"#), + OperatorSnapshotReadiness::Ready, + ) + .expect("ready response should build"), + ) + .expect("ready response should be utf-8"); + + assert!(ready_response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(ready_response.ends_with("ready")); +} + +#[test] +fn operator_state_endpoint_reports_stale_snapshots_as_not_ready() { + let stale_response = String::from_utf8( + orchestrator::build_operator_state_http_response( + format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + orchestrator::OPERATOR_READY_ENDPOINT_PATH + ) + .as_bytes(), + Some(br#"{"status":"ok"}"#), + OperatorSnapshotReadiness::SnapshotStale, + ) + .expect("stale ready response should build"), + ) + .expect("stale ready response should be utf-8"); + + assert!(stale_response.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); + assert!(stale_response.ends_with("snapshot_stale")); +} + +#[test] +fn operator_snapshot_readiness_handles_timestamp_edges() { + for (last_publish, now, threshold, expected) in [ + ( + Some(200), + 100, + Duration::from_secs(30), + OperatorSnapshotReadiness::SnapshotStale, + ), + ( + Some(100), + 101, + Duration::from_secs(u64::MAX), + OperatorSnapshotReadiness::Ready, + ), + ] { + assert_eq!( + orchestrator::operator_snapshot_readiness(last_publish, now, threshold), + expected + ); + } +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs b/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs new file mode 100644 index 00000000..d876a79f --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs @@ -0,0 +1,293 @@ +#[test] +fn live_operator_status_snapshot_degrades_when_post_review_status_refresh_fails() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let tracker = FakeTracker::with_refresh_error(vec![issue.clone()], "rate limited"); + let worktree_path = config.worktree_root().join(&issue.identifier); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should degrade instead of failing"); + + assert_eq!(snapshot.warnings, vec![String::from("post_review_lane_status_unavailable")]); + assert_eq!(snapshot.worktrees.len(), 1); + assert!(snapshot.post_review_lanes.is_empty()); +} + +#[test] +fn operator_state_snapshot_publish_skips_external_observers_after_tick_failure() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let tracker = FakeTracker::new(vec![sample_issue("Todo", &[])]); + let snapshot = orchestrator::build_operator_state_snapshot_for_publish( + &tracker, + &config, + &workflow, + &state_store, + 10, + &["control_plane_tick_failed"], + &[], + ) + .expect("snapshot should build from local state"); + + assert_eq!( + snapshot.warnings, + vec![ + String::from("control_plane_tick_failed"), + String::from("external_observer_status_skipped"), + ] + ); + assert_eq!(snapshot.projects[0].warning_count, 2); + assert_eq!(snapshot.projects[0].connector_state, "degraded"); + assert!( + tracker.label_queries.borrow().is_empty(), + "degraded publish should not query queued labels" + ); +} + +#[test] +fn operator_state_snapshot_reports_tracker_rate_limit_as_backoff() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let tracker = FakeTracker::new(vec![sample_issue("Todo", &[])]); + let reset_unix_epoch = OffsetDateTime::now_utc().unix_timestamp() + 60; + let error = eyre::eyre!( + "Linear connector is rate limited until `{reset_unix_epoch}`: API rate limit exceeded" + ); + let connector_backoff = orchestrator::tracker_rate_limit_backoff( + &error, + Instant::now(), + "operator_snapshot_refresh", + ) + .expect("rate limit should create backoff") + .to_operator_status(config.service_id(), reset_unix_epoch - 15); + let snapshot = orchestrator::build_operator_state_snapshot_for_publish( + &tracker, + &config, + &workflow, + &state_store, + 10, + &[TRACKER_RATE_LIMIT_WARNING], + slice::from_ref(&connector_backoff), + ) + .expect("snapshot should build from local state"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!( + snapshot.warnings, + vec![ + String::from(orchestrator::TRACKER_RATE_LIMIT_WARNING), + String::from("external_observer_status_skipped"), + ] + ); + assert_eq!(snapshot.connector_backoffs, vec![connector_backoff]); + assert_eq!(snapshot.connector_backoffs[0].project_id, config.service_id()); + assert_eq!(snapshot.connector_backoffs[0].connector, "linear"); + assert_eq!(snapshot.connector_backoffs[0].sync_phase, "operator_snapshot_refresh"); + assert_eq!(snapshot.connector_backoffs[0].quota_class, "linear_graphql_api"); + assert_eq!(snapshot.connector_backoffs[0].reset_unix_epoch, reset_unix_epoch); + assert_eq!(snapshot.connector_backoffs[0].reset_source, "linear"); + assert_eq!(snapshot.connector_backoffs[0].retry_after_seconds, 15); + assert_eq!(snapshot.connector_backoffs[0].warning, orchestrator::TRACKER_RATE_LIMIT_WARNING); + assert_eq!(snapshot_json["connector_backoffs"][0]["connector"], "linear"); + assert_eq!( + snapshot_json["connector_backoffs"][0]["sync_phase"], + "operator_snapshot_refresh" + ); + assert_eq!(snapshot_json["connector_backoffs"][0]["reset_unix_epoch"], reset_unix_epoch); + assert_eq!(snapshot_json["connector_backoffs"][0]["retry_after_seconds"], 15); + assert_ne!(snapshot_json["connector_backoffs"][0]["reset_at"], Value::Null); + assert_ne!(snapshot_json["connector_backoffs"][0]["next_action"], Value::Null); + assert_eq!(snapshot.projects[0].connector_state, "backoff"); + assert!( + tracker.label_queries.borrow().is_empty(), + "rate-limited publish should not query queued labels" + ); +} + +#[test] +fn operator_state_snapshot_publish_does_not_derive_history_outcome_without_execution_ledger() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "XY-355", + "Done", + &[], + Some(3), + "2026-04-29T10:11:00Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "y/decodex-xy-355", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember project ownership"); + state_store + .record_run_attempt("xy-355-attempt-1", &issue.id, 1, "succeeded") + .expect("successful attempt should record"); + state_store + .clear_worktree(&issue.id) + .expect("completed lane cleanup should clear local worktree"); + + let snapshot = orchestrator::build_operator_state_snapshot_for_publish( + &tracker, + &config, + &workflow, + &state_store, + 10, + &[], + &[], + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(lane.ledger_outcome.ledger_status, "missing"); + assert_eq!(lane.ledger_outcome.final_outcome, "execution_ledger_missing"); + assert_eq!(lane.ledger_outcome.record_count, 0); + assert_eq!( + lane.ledger_outcome.summary.as_deref(), + Some("No decodex.linear_execution_event records are available for this history lane.") + ); + assert_eq!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["ledger_status"], + "missing" + ); + assert_eq!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["final_outcome"], + "execution_ledger_missing" + ); + assert_ne!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["ledger_status"], + Value::Null + ); + assert_ne!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["final_outcome"], + Value::Null + ); + assert!( + tracker.comment_queries.borrow().is_empty(), + "control-plane publish should not replay Linear history comments every tick" + ); +} + +#[test] +fn operator_state_snapshot_publish_reads_local_completed_ledger_details_without_comment_replay() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "XY-355", + "Done", + &[], + Some(3), + "2026-04-29T10:11:00Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let local_comments = successful_linear_execution_history_comments_with_cleanup(&issue); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "y/decodex-xy-355", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember project ownership"); + state_store + .record_run_attempt("xy-355-attempt-1", &issue.id, 1, "succeeded") + .expect("successful attempt should record"); + state_store + .clear_worktree(&issue.id) + .expect("completed lane cleanup should clear local worktree"); + + seed_local_linear_execution_events(&state_store, &local_comments); + + let snapshot = orchestrator::build_operator_state_snapshot_for_publish( + &tracker, + &config, + &workflow, + &state_store, + 10, + &[], + &[], + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(lane.ledger_outcome.ledger_status, "present"); + assert_eq!(lane.ledger_outcome.final_outcome, "cleanup_complete"); + assert_eq!( + lane.ledger_outcome.pr_url.as_deref(), + Some("https://github.com/hack-ink/decodex/pull/355") + ); + assert_eq!( + lane.ledger_outcome.commit_sha.as_deref(), + Some("2222222222222222222222222222222222222222") + ); + assert_eq!(lane.ledger_outcome.closeout_status.as_deref(), Some("completed")); + assert_eq!(lane.ledger_outcome.lifecycle_elapsed_seconds, Some(660)); + assert_eq!(lane.ledger_outcome.record_count, 6); + assert_eq!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["ledger_status"], + "present" + ); + assert_eq!( + snapshot_json["history_lanes"][0]["ledger_outcome"]["pr_url"], + "https://github.com/hack-ink/decodex/pull/355" + ); + assert!( + tracker.comment_queries.borrow().is_empty(), + "control-plane publish should use local execution events instead of replaying Linear comments" + ); +} + +#[test] +fn tracker_rate_limit_error_enters_control_plane_backoff() { + let now = Instant::now(); + let error = eyre::eyre!( + "Linear connector is rate limited: Rate limit exceeded. Only 2500 requests are allowed per 1 hour." + ); + let backoff_until = orchestrator::tracker_rate_limit_backoff(&error, now, "control_plane_tick") + .expect("rate limit should create backoff"); + + assert!(backoff_until.until > now); +} + +#[test] +fn tracker_rate_limit_error_uses_reset_timestamp_when_available() { + let now = Instant::now(); + let reset_unix_epoch = OffsetDateTime::now_utc().unix_timestamp() + 30; + let error = eyre::eyre!( + "Linear connector is rate limited until `{reset_unix_epoch}`: API rate limit exceeded" + ); + let backoff_until = orchestrator::tracker_rate_limit_backoff(&error, now, "control_plane_tick") + .expect("rate limit reset should create backoff"); + + assert!(backoff_until.until >= now + Duration::from_secs(29)); + assert!(backoff_until.until <= now + Duration::from_secs(31)); + assert_eq!(backoff_until.reset_unix_epoch, reset_unix_epoch); + assert_eq!(backoff_until.reset_source, "linear"); + assert_eq!(backoff_until.sync_phase, "control_plane_tick"); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs new file mode 100644 index 00000000..1be4d054 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs @@ -0,0 +1,635 @@ +#[test] +fn live_operator_status_snapshot_includes_queued_candidates_with_dispatch_classification() { + let workflow_markdown = + sample_workflow_markdown("pubfi", &[], "Follow the repository policy.", 1) + .replace("max_concurrent_agents = 1", "max_concurrent_agents = 2"); + let (_temp_dir, config, workflow) = + temp_project_layout_with_workflow_markdown(&workflow_markdown); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let ready_issue = sample_issue_with_sort_fields( + "issue-ready", + "PUB-101", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let mut blocked_issue = sample_issue_with_sort_fields( + "issue-blocked", + "PUB-102", + "Todo", + &[], + Some(2), + "2026-03-13T05:16:17.133Z", + ); + + blocked_issue.description = String::from("```json\n{}\n```"); + + let claimed_issue = sample_issue_with_sort_fields( + "issue-claimed", + "PUB-103", + "Todo", + &[], + Some(3), + "2026-03-13T06:16:17.133Z", + ); + let closed_issue = sample_issue_with_sort_fields( + "issue-closed", + "PUB-104", + "Done", + &[], + Some(4), + "2026-03-13T07:16:17.133Z", + ); + let canceled_issue = sample_issue_with_sort_fields( + "issue-canceled", + "PUB-105", + "Canceled", + &[], + Some(5), + "2026-03-13T08:16:17.133Z", + ); + + state_store + .upsert_lease(config.service_id(), &claimed_issue.id, "run-claimed", "In Progress") + .expect("lease should record"); + + let tracker = FakeTracker::new(vec![ + claimed_issue.clone(), + blocked_issue.clone(), + closed_issue.clone(), + canceled_issue.clone(), + ready_issue.clone(), + ]); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + + assert_eq!(snapshot.queued_candidates.len(), 3); + + let queued_by_issue = snapshot + .queued_candidates + .iter() + .map(|candidate| (candidate.issue_identifier.as_str(), candidate)) + .collect::>(); + + assert_eq!( + queued_by_issue.get("PUB-101").expect("ready queued issue should exist").classification, + "ready" + ); + assert_eq!( + queued_by_issue.get("PUB-101").expect("ready queued issue should exist").reason, + "eligible_for_dispatch" + ); + assert_eq!( + queued_by_issue.get("PUB-102").expect("blocked queued issue should exist").classification, + "blocked" + ); + assert_eq!( + queued_by_issue.get("PUB-102").expect("blocked queued issue should exist").reason, + "missing_dispatch_briefing" + ); + assert_eq!( + queued_by_issue.get("PUB-103").expect("claimed queued issue should exist").classification, + "claimed" + ); + assert_eq!( + queued_by_issue.get("PUB-103").expect("claimed queued issue should exist").reason, + "shared_claim_present" + ); + assert!( + !queued_by_issue.contains_key("PUB-104"), + "terminal queued echoes should not appear in operator intake candidates" + ); + assert!( + !queued_by_issue.contains_key("PUB-105"), + "canceled queued echoes should not appear in operator intake candidates" + ); +} + +#[test] +fn live_operator_status_snapshot_excludes_claimed_candidates_from_waiting_intake_count() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let claimed_issue = sample_issue_with_sort_fields( + "issue-claimed", + "PUB-103", + "Todo", + &[], + Some(3), + "2026-03-13T06:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![claimed_issue.clone()]); + + state_store + .record_run_attempt("run-claimed", &claimed_issue.id, 1, "running") + .expect("active run should record"); + state_store + .upsert_lease(config.service_id(), &claimed_issue.id, "run-claimed", "In Progress") + .expect("active lease should record"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let project = snapshot.projects.first().expect("project summary should exist"); + let candidate = + snapshot.queued_candidates.first().expect("claimed queue echo should remain raw-visible"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.active_runs[0].run_id, "run-claimed"); + assert_eq!(project.active_run_count, 1); + assert_eq!(candidate.issue_identifier, "PUB-103"); + assert_eq!(candidate.classification, "claimed"); + assert_eq!(candidate.reason, "shared_claim_present"); + assert_eq!( + project.queued_candidate_count, 0, + "claimed queue echoes are raw state, not waiting intake" + ); + assert_eq!( + project.waiting_lane_count, 0, + "claimed queue echoes must not inflate project waiting counts" + ); + assert!(rendered.contains("Backlog: 0")); + assert!(rendered.contains("Active queue echoes: 1")); +} + +#[test] +fn live_operator_status_snapshot_reports_capacity_waiting_separately_from_blocked() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let waiting_issue = sample_issue_with_sort_fields( + "issue-waiting", + "PUB-101", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![waiting_issue]); + + state_store + .upsert_lease(config.service_id(), "issue-running", "run-active", "In Progress") + .expect("active lease should consume the single global slot"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot.queued_candidates.first().expect("waiting queued issue should exist"); + + assert_eq!(candidate.issue_identifier, "PUB-101"); + assert_eq!(candidate.classification, "waiting"); + assert_eq!(candidate.reason, "global_concurrency_exhausted"); + assert_eq!(candidate.attention, None); +} + +#[test] +fn live_operator_status_snapshot_includes_needs_attention_run_context() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-105", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state::write_run_thread_status_marker( + &worktree_path, + "run-needs-attention", + 3, + Some("thread-1"), + Some("turn-1"), + "systemError", + &[], + ) + .expect("thread status marker should write"); + state::write_run_retry_budget_attempt_count(&worktree_path, "run-needs-attention", 3, 3) + .expect("retry budget marker should write"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-105") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert_eq!(attention.run_id.as_deref(), Some("run-needs-attention")); + assert_eq!(attention.attempt_number, Some(3)); + assert_eq!(attention.current_operation.as_deref(), Some(state::RUN_OPERATION_AGENT_RUN)); + assert_eq!(attention.thread_status.as_deref(), Some("systemError")); + assert_eq!(attention.attempt_status, None); + assert_eq!(attention.retry_budget_attempt_count, Some(3)); + assert_eq!(attention.retry_budget_max_attempts, 3); + assert_eq!(attention.worktree_path.as_deref(), Some(".worktrees/PUB-105")); + assert!(attention.summary.contains("systemError")); + assert!( + snapshot.worktrees.iter().any(|worktree| worktree.worktree_path == ".worktrees/PUB-105"), + "needs-attention worktree should still be reported in raw snapshot state" + ); + assert_eq!( + snapshot.projects[0].retained_worktree_count, 0, + "needs-attention queue ownership should keep the worktree out of recovery cleanup counts" + ); + + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("attention_worktree: .worktrees/PUB-105")); + assert!(rendered.contains("Recovery worktrees: 0")); + assert!(rendered.contains("- none (owned worktrees are shown in their lane sections above)")); + assert!(!rendered.contains("role: cleanup_only")); +} + +#[test] +fn live_operator_status_snapshot_explains_needs_attention_before_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-107", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state::write_run_operation_marker( + &worktree_path, + "run-needs-attention", + 1, + RUN_OPERATION_AGENT_RUN, + ) + .expect("operation marker should write"); + state::write_run_retry_budget_attempt_count(&worktree_path, "run-needs-attention", 1, 1) + .expect("retry budget marker should write"); + + state_store + .record_run_attempt("run-needs-attention", &issue.id, 1, "interrupted") + .expect("interrupted attempt should record"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-107") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert_eq!(attention.attempt_status.as_deref(), Some("interrupted")); + assert_eq!(attention.auto_retry_blocked_reason.as_deref(), Some("needs_attention_label")); + assert_eq!(attention.retry_budget_attempt_count, Some(1)); + assert_eq!(attention.retry_budget_max_attempts, 3); + assert_eq!( + attention.summary, + "Previous attempt was interrupted during agent execution; operator recovery required." + ); +} + +#[test] +fn live_operator_status_snapshot_surfaces_needs_attention_event_cause() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-108", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + + tracker.issue_comments.borrow_mut().insert( + issue.id.clone(), + vec![linear_execution_history_comment( + &issue, + "terminal_failure", + "2026-03-13T09:20:00Z", + "retained-review-head-mismatch", + |record| { + record.error_class = Some(String::from("review_orchestration_head_mismatch")); + record.next_action = Some(String::from( + "inspect retained review orchestration reason `review_orchestration_head_mismatch`, resolve the blocker manually", + )); + record.summary = Some(String::from( + "Retained review orchestration requires operator attention.", + )); + record.blockers = Some(vec![String::from( + "retained review orchestration head mismatch", + )]); + record.evidence = Some(vec![String::from( + "review orchestration marker head differs from local worktree HEAD", + )]); + record.terminal_path = Some(String::from("manual_attention")); + }, + )], + ); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-108") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert_eq!( + attention.attention_error_class.as_deref(), + Some("review_orchestration_head_mismatch") + ); + assert_eq!( + attention.attention_next_action.as_deref(), + Some( + "inspect retained review orchestration reason `review_orchestration_head_mismatch`, resolve the blocker manually" + ) + ); + assert!(rendered.contains("attention_cause: review_orchestration_head_mismatch")); + assert!(rendered.contains("attention_next_action: inspect retained review orchestration")); +} + +#[test] +fn live_operator_status_snapshot_surfaces_retained_partial_progress() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-106", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + let tracker = FakeTracker::new(vec![issue.clone()]); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-106", ".worktrees/PUB-106", "main"], + ); + + fs::write(worktree_path.join("README.md"), "changed repo file\n") + .expect("tracked worktree file should change"); + state::write_run_retry_budget_attempt_count(&worktree_path, "run-partial-progress", 3, 3) + .expect("retry budget marker should write"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-106") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert!(attention.worktree_has_tracked_changes); + assert_eq!(attention.retry_budget_attempt_count, Some(3)); + assert_eq!(attention.retry_budget_max_attempts, 3); + assert!( + attention.summary.contains("Partial worktree changes are retained"), + "summary should explain retained patch recovery, got {:?}", + attention.summary + ); +} + +#[test] +fn live_operator_status_snapshot_surfaces_git_credential_failures() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-105", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state::write_run_operation_marker( + &worktree_path, + "run-missing-credentials", + 1, + RUN_OPERATION_GIT_CREDENTIALS, + ) + .expect("credential preflight marker should write"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-105") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + + assert!(snapshot.active_runs.is_empty()); + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert_eq!(attention.current_operation.as_deref(), Some(state::RUN_OPERATION_GIT_CREDENTIALS)); + assert_eq!(attention.summary, "Git credential preflight failed; operator recovery required."); +} + +#[test] +fn live_operator_status_snapshot_recovers_shared_claims_for_fresh_status_store_instances() { + let workflow_markdown = + sample_workflow_markdown("pubfi", &[], "Follow the repository policy.", 1) + .replace("max_concurrent_agents = 1", "max_concurrent_agents = 2"); + let (_temp_dir, config, workflow) = + temp_project_layout_with_workflow_markdown(&workflow_markdown); + let remote_store = StateStore::open_in_memory().expect("remote state store should open"); + let observer_store = StateStore::open_in_memory().expect("observer state store should open"); + let claimed_issue = sample_issue_with_sort_fields( + "issue-claimed", + "PUB-103", + "Todo", + &[], + Some(3), + "2026-03-13T06:16:17.133Z", + ); + let ready_issue = sample_issue_with_sort_fields( + "issue-ready", + "PUB-101", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![claimed_issue.clone(), ready_issue]); + + remote_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("remote store should configure dispatch-slot root"); + + assert!( + remote_store + .try_acquire_lease( + config.service_id(), + &claimed_issue.id, + "run-claimed", + workflow.frontmatter().tracker().in_progress_state(), + ) + .expect("remote store should acquire the shared issue claim") + ); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &observer_store, + 10, + ) + .expect("snapshot should build"); + let queued_by_issue = snapshot + .queued_candidates + .iter() + .map(|candidate| (candidate.issue_identifier.as_str(), candidate)) + .collect::>(); + + assert_eq!( + queued_by_issue.get("PUB-103").expect("claimed queued issue should exist").classification, + "claimed" + ); + assert_eq!( + queued_by_issue.get("PUB-103").expect("claimed queued issue should exist").reason, + "shared_claim_present" + ); + assert!( + snapshot.active_runs.is_empty(), + "fresh observer stores should not invent local running lanes while reconstructing the shared claim view" + ); +} + +#[test] +fn live_operator_status_snapshot_reconstructs_same_shared_view_for_fresh_state_stores() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_issue = sample_active_issue("In Progress"); + let closed_issue = sample_issue_with_sort_fields( + "issue-closed", + "PUB-104", + "Done", + &[], + Some(4), + "2026-03-13T07:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![active_issue.clone(), closed_issue]); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&active_issue.identifier, false) + .expect("retained worktree should exist"); + + state::write_run_activity_marker(&worktree.path, "run-1", 1) + .expect("activity marker should write"); + + let build_view = |state_store: &StateStore| -> Value { + let recovered = orchestrator::recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + state_store, + ) + .expect("runtime recovery should succeed"); + + orchestrator::hydrate_status_snapshot_state(&config, state_store, recovered) + .expect("status hydration should succeed"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + state_store, + 10, + ) + .expect("snapshot should build"); + + serde_json::json!({ + "active_runs": snapshot.active_runs.iter().map(|run| { + serde_json::json!({ + "run_id": run.run_id, + "issue_id": run.issue_id, + "phase": run.phase, + "current_operation": run.current_operation, + "active_lease": run.active_lease, + "branch_name": run.branch_name, + "worktree_path": run.worktree_path, + }) + }).collect::>(), + "queued_candidates": snapshot.queued_candidates, + "worktrees": snapshot.worktrees, + "post_review_lanes": snapshot.post_review_lanes, + }) + }; + let first_store = StateStore::open_in_memory().expect("first state store should open"); + let second_store = StateStore::open_in_memory().expect("second state store should open"); + + assert_eq!(build_view(&first_store), build_view(&second_store)); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs new file mode 100644 index 00000000..060847db --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs @@ -0,0 +1,1347 @@ +#[test] +fn failure_comments_use_repo_relative_worktree_paths() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let worktree = WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.repo_root().join(".worktrees/PUB-101"), + reused_existing: true, + }; + + assert_eq!(orchestrator::relative_worktree_path(&config, &worktree), ".worktrees/PUB-101"); +} + +#[test] +fn operator_status_snapshot_includes_active_runs_and_repo_relative_paths() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store.update_run_thread("run-1", "thread-1").expect("thread id should attach"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + state_store + .append_event("run-1", 1, "turn/completed", "{\"turn\":\"1\"}") + .expect("event should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + + assert_eq!(snapshot.project_id, "pubfi"); + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.recent_runs.len(), 1); + assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); + assert_eq!(snapshot.active_runs[0].run_id, "run-1"); + assert_eq!(snapshot.active_runs[0].phase, "executing"); + assert_eq!(snapshot.active_runs[0].current_operation, state::RUN_OPERATION_AGENT_RUN); + assert_eq!(snapshot.active_runs[0].thread_id.as_deref(), Some("thread-1")); + assert_eq!(snapshot.active_runs[0].branch_name.as_deref(), Some("x/pubfi-pub-101")); + assert_eq!(snapshot.active_runs[0].worktree_path.as_deref(), Some(".worktrees/PUB-101")); + assert!(snapshot.active_runs[0].last_run_activity_at.is_some()); + assert!(snapshot.active_runs[0].last_progress_at.is_some()); + assert!(!snapshot.active_runs[0].suspected_stall); + assert_eq!(snapshot.active_runs[0].last_event_type.as_deref(), Some("turn/completed")); + assert_eq!(snapshot.worktrees[0].worktree_path, ".worktrees/PUB-101"); + assert_eq!(snapshot.worktrees[0].ownership, "active_lane"); + assert!(snapshot.worktrees[0].ownership_reason.contains("Active lane `run-1`")); + + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(project.active_run_count, 1); + assert_eq!( + project.retained_worktree_count, 0, + "active running lane worktrees must not inflate project recovery counts" + ); + assert_eq!(project.connector_state, "ok"); + assert!(project.last_activity_at.is_some()); +} + +#[test] +fn operator_status_snapshot_surfaces_merged_dirty_ad_hoc_worktree() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("accounts-column-format"); + + git_status_success( + config.repo_root(), + &[ + "worktree", + "add", + "-b", + "xy/accounts-column-format", + worktree_path.to_str().expect("worktree path should be UTF-8"), + "main", + ], + ); + commit_worktree_change(&worktree_path, "README.md", "feature work\n", "feature work"); + git_status_success( + config.repo_root(), + &["merge", "--no-ff", "xy/accounts-column-format", "-m", "land feature"], + ); + + fs::write(worktree_path.join("README.md"), "dirty after land\n") + .expect("worktree file should become dirty"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let worktree = snapshot + .worktrees + .iter() + .find(|worktree| worktree.worktree_path == ".worktrees/accounts-column-format") + .expect("ad-hoc merged dirty worktree should be surfaced"); + + assert!(snapshot.warnings.contains(&String::from("merged_worktree_cleanup_pending"))); + assert!(snapshot.warnings.contains(&String::from("merged_dirty_worktree"))); + assert_eq!(worktree.branch_name, "xy/accounts-column-format"); + assert_eq!(worktree.ownership, "post_land_cleanup"); + assert!( + worktree + .ownership_reason + .contains("already merged into `main`"), + "ownership reason should explain why the worktree is no longer usable" + ); + assert!( + worktree.hygiene.as_ref().is_some_and(|hygiene| hygiene.dirty), + "hygiene state should mark the local changes" + ); + + let error = orchestrator::ensure_project_has_no_merged_worktree_cleanup_debt(&config) + .expect_err("normal automation should stop while merged dirty worktrees remain"); + + assert!(error.to_string().contains("Post-land worktree cleanup is pending")); +} + +#[test] +fn live_operator_status_snapshot_hydrates_active_run_issue_display_metadata() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "xy-392-attempt-1-1777551056"; + let mut issue = sample_issue_with_sort_fields( + "issue-active", + "XY-392", + "In Progress", + &[], + Some(3), + "2026-04-30T03:01:00Z", + ); + + issue.title = String::from("Hydrate issue display metadata on run rows"); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("active run should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "In Progress") + .expect("active lease should record"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let active_run = snapshot.active_runs.first().expect("active run should exist"); + let recent_run = snapshot.recent_runs.first().expect("recent run should exist"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(active_run.project_id, config.service_id()); + assert_eq!(active_run.issue_identifier.as_deref(), Some("XY-392")); + assert_eq!(active_run.title.as_deref(), Some("Hydrate issue display metadata on run rows")); + assert_eq!(recent_run.issue_identifier.as_deref(), Some("XY-392")); + assert_eq!(recent_run.title.as_deref(), Some("Hydrate issue display metadata on run rows")); + assert_eq!(snapshot_json["active_runs"][0]["project_id"], "pubfi"); + assert_eq!(snapshot_json["active_runs"][0]["issue_identifier"], "XY-392"); + assert_eq!( + snapshot_json["active_runs"][0]["title"], + "Hydrate issue display metadata on run rows" + ); +} + +#[test] +fn idle_operator_status_snapshot_has_no_runtime_or_recovery_noise() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let tracker = FakeTracker::new(Vec::new()); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("idle snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(snapshot.project_id, "pubfi"); + assert_eq!(snapshot.run_limit, 10); + assert!(snapshot.warnings.is_empty(), "idle snapshot warnings: {:?}", snapshot.warnings); + assert!(snapshot.active_runs.is_empty(), "idle snapshot should have no active runs"); + assert!(snapshot.recent_runs.is_empty(), "idle snapshot should have no run history"); + assert!(snapshot.history_lanes.is_empty(), "idle snapshot should have no run ledger lanes"); + assert!( + snapshot.queued_candidates.is_empty(), + "idle snapshot should have no queued candidates" + ); + assert!(snapshot.worktrees.is_empty(), "idle snapshot should have no recovery worktrees"); + assert!( + snapshot.post_review_lanes.is_empty(), + "idle snapshot should have no retained post-review lanes" + ); + + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(project.retained_worktree_count, 0); + assert_eq!(project.waiting_lane_count, 0); + assert_eq!(project.attention_count, 0); + assert_eq!(project.connector_state, "ok"); + assert_eq!(project.last_activity_at, None); + + for field in [ + "warnings", + "active_runs", + "recent_runs", + "history_lanes", + "queued_candidates", + "worktrees", + "post_review_lanes", + ] { + assert_eq!( + snapshot_json[field], + serde_json::json!([]), + "idle /state field {field} should serialize as an empty array", + ); + } + + assert!(rendered.contains("Warnings: 0")); + assert!(rendered.contains("Running lanes: 0")); + assert!(rendered.contains("Run ledger shown: 0 issue lanes from 0 history attempts")); + assert!(rendered.contains("Backlog: 0")); + assert!(rendered.contains("Active queue echoes: 0")); + assert!(rendered.contains("Stale closed queue labels: 0")); + assert!(rendered.contains("Recovery worktrees: 0")); + assert!(rendered.contains("Post-review lanes: 0")); + assert!(rendered.contains("\nRunning Lanes\n- none\n")); + assert!(rendered.contains("\nRun Ledger\n- none\n")); + assert!(rendered.contains("\nBacklog\n- none\n")); + assert!(rendered.contains("\nActive Queue Echoes\n- none\n")); + assert!(rendered.contains("\nStale Closed Queue Labels\n- none\n")); + assert!(rendered.contains("\nRecovery Worktrees\n- none\n")); + assert!(rendered.contains("\nPost-Review Lanes\n- none\n")); + assert!(!rendered.contains("Warning details:")); + assert!(!rendered.contains("run_id:")); + assert!(!rendered.contains("active_lease: true")); + assert!(!rendered.contains("role: post_review_lane")); + assert!(!rendered.contains("role: cleanup_only")); +} + +#[test] +fn idle_operator_status_snapshot_includes_configured_codex_accounts() { + let (_temp_dir, base_config, _workflow) = temp_project_layout(); + let accounts_path = service_config_dir(base_config.repo_root()).join("codex-auth.jsonl"); + let usage_endpoint = start_codex_usage_fixture_server(vec![ + r#"{"plan_type":"pro","rate_limit":{"primary_window":{"used_percent":7,"limit_window_seconds":18000,"reset_at":1800018000},"secondary_window":{"used_percent":11,"limit_window_seconds":604800,"reset_at":1800604800}},"credits":{"has_credits":true,"unlimited":false,"balance":"12.34"}}"#, + r#"{"plan_type":"plus","rate_limit":{"primary_window":{"used_percent":22,"limit_window_seconds":18000,"reset_at":1800019000},"secondary_window":{"used_percent":33,"limit_window_seconds":604800,"reset_at":1800605800}},"credits":{"has_credits":false,"unlimited":false,"balance":"0"}}"#, + ]); + + std::fs::write( + &accounts_path, + r#"{"email":"default@example.com","auth_mode":"chatgpt","tokens":{"access_token":"access-default","refresh_token":"refresh-default","account_id":"acct_default"}} +{"email":"copy@example.com","auth_mode":"chatgpt","tokens":{"access_token":"access-copy","refresh_token":"refresh-copy","account_id":"acct_copy"}} +"#, + ) + .expect("accounts fixture should write"); + + let mut config_toml = service_config_toml_for_config( + &base_config, + base_config.github().token_env_var(), + base_config.codex().internal_review_mode(), + base_config.codex().external_review_enabled(), + ); + + config_toml.push_str(&format!( + "\n[codex.accounts]\npath = \"{}\"\nusage_endpoint = \"{}\"\n", + accounts_path.display(), + usage_endpoint + )); + + write_service_config(base_config.repo_root(), &config_toml); + + let config = load_service_config(base_config.repo_root()); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + let accounts = + snapshot_json["accounts"].as_array().expect("snapshot should expose configured accounts"); + + assert!(snapshot.active_runs.is_empty()); + assert_eq!(accounts.len(), 2); + assert_eq!(accounts[0]["email"], "default@example.com"); + assert_eq!(accounts[0]["status"], "available"); + assert_eq!(accounts[0]["refresh_status"], "not_needed"); + assert_eq!(accounts[0]["plan_type"], "pro"); + assert_eq!(accounts[0]["primary_remaining_percent"], 93); + assert_eq!(accounts[0]["credits_balance"], "12.34"); + assert_eq!(accounts[1]["email"], "copy@example.com"); + assert_eq!(accounts[1]["status"], "available"); + assert_eq!(accounts[1]["refresh_status"], "not_needed"); + assert_eq!(accounts[1]["plan_type"], "plus"); + assert_eq!(accounts[1]["primary_remaining_percent"], 78); + assert_eq!(accounts[1]["credits_balance"], "0"); +} + +fn start_codex_usage_fixture_server(responses: Vec<&'static str>) -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("usage fixture server should bind"); + let address = listener.local_addr().expect("usage fixture address should resolve"); + + thread::spawn(move || { + for body in responses { + let (mut stream, _peer) = + listener.accept().expect("usage fixture request should arrive"); + let mut request = [0_u8; 4_096]; + let _ = stream.read(&mut request); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + + stream + .write_all(response.as_bytes()) + .expect("usage fixture response should write"); + + let _ = stream.shutdown(Shutdown::Both); + } + }); + + format!("http://{address}/wham/usage") +} + +#[test] +fn operator_status_snapshot_includes_local_recovery_worktree_directories() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-199"); + + fs::create_dir_all(&worktree_path).expect("worktree directory should exist"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + + assert_eq!(snapshot.worktrees.len(), 1); + assert_eq!(snapshot.worktrees[0].issue_id, "PUB-199"); + assert!(!snapshot.worktrees[0].branch_name.is_empty()); + assert_eq!(snapshot.worktrees[0].worktree_path, ".worktrees/PUB-199"); + assert_eq!(snapshot.worktrees[0].ownership, "cleanup_only"); + assert!(snapshot.worktrees[0].ownership_reason.contains("local cleanup only")); + assert_eq!(snapshot.projects[0].retained_worktree_count, 1); +} + +#[test] +fn completed_retained_worktree_without_post_review_owner_is_cleanup_only() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-cleanup", + "PUB-199", + "Done", + &[], + Some(4), + "2026-03-13T07:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let worktree_path = config.worktree_root().join(&issue.identifier); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-199", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert!(snapshot.post_review_lanes.is_empty()); + assert_eq!(snapshot.projects[0].retained_worktree_count, 1); + assert_eq!(snapshot.worktrees[0].issue_identifier.as_deref(), Some("PUB-199")); + assert_eq!(snapshot.worktrees[0].issue_state.as_deref(), Some("Done")); + assert_eq!(snapshot.worktrees[0].ownership, "cleanup_only"); + assert!(snapshot.worktrees[0].ownership_reason.contains("Issue is Done")); + assert_eq!(snapshot_json["worktrees"][0]["ownership"], "cleanup_only"); + assert_eq!(snapshot_json["worktrees"][0]["issue_state"], "Done"); + assert!(rendered.contains("role: cleanup_only")); + assert!(rendered.contains("reason: Issue is Done")); + assert!(!rendered.contains("role: post_review_lane")); + assert!(!rendered.contains("classification: blocked")); + assert!(!rendered.contains("review_handoff_missing")); +} + +#[test] +fn operator_status_snapshot_reports_retry_backoff_from_worktree_marker() { + for (retry_kind, expected_wait_reason) in + [("failure", "failure_retry"), ("git_lock_contention", "git_lock_contention")] + { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "failed") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_retry_schedule( + &worktree_path, + "run-1", + 1, + retry_kind, + OffsetDateTime::now_utc().unix_timestamp() + 60, + ) + .expect("retry schedule marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.recent_runs.first().expect("recent run should exist"); + + assert_eq!(run.phase, "retry_backoff"); + assert_eq!(run.wait_reason.as_deref(), Some(expected_wait_reason)); + assert_eq!(run.retry_kind.as_deref(), Some(retry_kind)); + assert!(run.next_retry_at.is_some()); + assert_eq!(snapshot.projects[0].waiting_lane_count, 1); + assert_eq!(snapshot.projects[0].connector_state, "backoff"); + } +} + +#[test] +fn operator_status_snapshot_ignores_retry_schedule_on_running_attempt() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, process::id()) + .expect("run marker should write"); + state::write_run_retry_schedule( + &worktree_path, + "run-1", + 1, + "failure", + OffsetDateTime::now_utc().unix_timestamp() + 60, + ) + .expect("stale retry schedule marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should exist"); + + assert_eq!(run.phase, "executing"); + assert_eq!(run.wait_reason, None); + assert_eq!(run.retry_kind, None); + assert_eq!(run.next_retry_at, None); + assert_eq!(snapshot.projects[0].waiting_lane_count, 0); + assert_eq!(snapshot.projects[0].connector_state, "ok"); +} + +#[test] +fn operator_status_snapshot_reports_stalled_runs_explicitly() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "stalled") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.recent_runs.first().expect("recent run should exist"); + + assert_eq!(run.phase, "stalled"); + assert_eq!(run.wait_reason.as_deref(), Some("app_server_idle_timeout")); + assert_eq!(run.current_operation, state::RUN_OPERATION_IDLE); + assert!(!run.suspected_stall); +} + +#[test] +fn operator_status_snapshot_surfaces_reconciliation_operation_for_stalled_runs() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "stalled") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_operation_marker( + &worktree_path, + "run-1", + 1, + RUN_OPERATION_RECONCILIATION, + ) + .expect("reconciliation marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.recent_runs.first().expect("recent run should exist"); + + assert_eq!(run.phase, "stalled"); + assert_eq!(run.current_operation, state::RUN_OPERATION_RECONCILIATION); +} + +#[test] +fn operator_status_snapshot_preserves_stalled_run_activity_when_tagging_reconciliation() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "stalled") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, 42) + .expect("initial activity marker should write"); + + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + let marker_body = fs::read_to_string(&marker_path).expect("marker body should load"); + let stale_activity = OffsetDateTime::now_utc().unix_timestamp() - 600; + let rewritten = marker_body + .lines() + .map(|line| { + if line.starts_with("last_activity_unix_epoch=") { + format!("last_activity_unix_epoch={stale_activity}") + } else if line.starts_with("last_progress_unix_epoch=") { + format!("last_progress_unix_epoch={stale_activity}") + } else { + line.to_owned() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(&marker_path, rewritten).expect("marker body should rewrite"); + state::write_run_operation_marker_preserving_activity( + &worktree_path, + "run-1", + 1, + RUN_OPERATION_RECONCILIATION, + ) + .expect("reconciliation marker should preserve existing activity"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.recent_runs.first().expect("recent run should exist"); + + assert_eq!(marker.process_id(), Some(42)); + assert_eq!(marker.last_activity_unix_epoch(), Some(stale_activity)); + assert_eq!(run.current_operation, state::RUN_OPERATION_RECONCILIATION); + assert_eq!(run.process_id, Some(42)); +} + +#[test] +fn operator_status_snapshot_marks_soft_stalls_before_hard_timeout() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_operation_marker(&worktree_path, "run-1", 1, RUN_OPERATION_AGENT_RUN) + .expect("operation marker should write"); + + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + let marker_body = fs::read_to_string(&marker_path).expect("marker body should load"); + let suspected_age = (ACTIVE_RUN_IDLE_TIMEOUT.as_secs() / 2).saturating_add(1) as i64; + let stale_progress = OffsetDateTime::now_utc().unix_timestamp() - suspected_age; + let rewritten = marker_body + .lines() + .map(|line| { + if line.starts_with("last_progress_unix_epoch=") { + format!("last_progress_unix_epoch={stale_progress}") + } else { + line.to_owned() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(&marker_path, rewritten).expect("marker body should rewrite"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should exist"); + + assert_eq!(run.current_operation, state::RUN_OPERATION_AGENT_RUN); + assert!(run.last_progress_at.is_some()); + assert!(run.suspected_stall); +} + +#[test] +fn operator_status_snapshot_counts_stopped_active_process_as_attention_not_running() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, u32::MAX) + .expect("stopped process marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should remain visible"); + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(run.phase, "executing"); + assert_eq!(run.process_alive, Some(false)); + assert_eq!(project.active_run_count, 0); + assert_eq!(project.attention_count, 1); +} + +#[test] +fn operator_status_snapshot_keeps_unleased_live_process_in_running_lanes() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, process::id()) + .expect("live process marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should remain visible"); + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(run.run_id, "run-1"); + assert_eq!(run.status, "running"); + assert_eq!(run.attempt_status, "running"); + assert!(!run.active_lease); + assert_eq!(run.queue_lease_state, "not_held"); + assert_eq!(run.execution_liveness, "process_alive"); + assert_eq!(run.process_alive, Some(true)); + assert_eq!(project.active_run_count, 1); + assert_eq!(project.retained_worktree_count, 0); +} + +#[test] +fn operator_status_snapshot_promotes_starting_after_app_server_activity() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "starting") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_thread_status_marker( + &worktree_path, + "run-1", + 1, + Some("thread-1"), + Some("turn-1"), + "active", + &[], + ) + .expect("thread status should write"); + state::write_run_effective_runtime_marker( + &worktree_path, + "run-1", + 1, + &EffectiveRuntimeMarker { + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + effective_model: "gpt-5.4", + effective_model_provider: "openai", + effective_cwd: "/tmp/worktree", + effective_approval_policy: "never", + effective_approvals_reviewer: "human", + effective_sandbox_mode: "workspaceWrite", + }, + ) + .expect("effective runtime should write"); + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 2, + last_event_type: "model/response", + child_agent_activity: None, + protocol_activity: None, + }, + ) + .expect("protocol summary should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should remain visible"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(run.status, "running"); + assert_eq!(run.attempt_status, "starting"); + assert_eq!(run.phase, "executing"); + assert_eq!(run.queue_lease_state, "held"); + assert_eq!(run.execution_liveness, "process_alive"); + assert_eq!(run.thread_status.as_deref(), Some("active")); + assert_eq!(run.effective_model.as_deref(), Some("gpt-5.4")); + assert!(rendered.contains("status: running")); + assert!(rendered.contains("attempt_status: starting")); +} + +#[test] +fn operator_status_snapshot_counts_stale_starting_run_as_attention_not_running() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + let stale_activity = + OffsetDateTime::now_utc().unix_timestamp() - ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 - 30; + + state_store + .record_run_attempt("run-1", &issue.id, 1, "starting") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + fs::write( + worktree_path.join(RUN_ACTIVITY_MARKER_FILE), + format!( + "run_id=run-1\nattempt_number=1\nlast_activity_unix_epoch={stale_activity}\nlast_protocol_activity_unix_epoch={stale_activity}\nlast_progress_unix_epoch={stale_activity}\n" + ), + ) + .expect("stale processless marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should remain visible"); + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(run.status, "starting"); + assert_eq!(run.phase, "executing"); + assert_eq!(run.process_alive, None); + assert!(run.protocol_idle_for_seconds.is_some_and(|idle| { + u64::try_from(idle).is_ok_and(|idle| idle >= ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + })); + assert_eq!(project.active_run_count, 0); + assert_eq!(project.attention_count, 1); +} + +#[test] +fn operator_status_snapshot_excludes_completed_lingering_lease_from_active_runs() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let completed_issue = sample_issue_with_sort_fields( + "issue-1", + "XY-379", + "Done", + &[], + Some(3), + "2026-04-29T17:00:33.133Z", + ); + let active_issue = sample_issue_with_sort_fields( + "issue-2", + "XY-378", + "In Progress", + &[], + Some(3), + "2026-04-29T17:01:33.133Z", + ); + let completed_run_id = "xy-379-attempt-1-1777482033"; + let active_run_id = "xy-378-attempt-1-1777482000"; + + state_store + .record_run_attempt(completed_run_id, &completed_issue.id, 1, "running") + .expect("completed run should record"); + state_store + .upsert_lease("pubfi", &completed_issue.id, completed_run_id, "In Progress") + .expect("stale active lease should remain in runtime db"); + state_store + .append_event(completed_run_id, 1, "turn/completed", "{\"turn\":\"1\"}") + .expect("terminal protocol evidence should record"); + state_store + .update_run_status(completed_run_id, "succeeded") + .expect("terminal status should update"); + state_store + .record_run_attempt(active_run_id, &active_issue.id, 1, "running") + .expect("active run should record"); + state_store + .upsert_lease("pubfi", &active_issue.id, active_run_id, "In Progress") + .expect("active lease should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 25) + .expect("snapshot should build"); + let project = snapshot.projects.first().expect("project summary should exist"); + let completed_run = snapshot + .recent_runs + .iter() + .find(|run| run.run_id == completed_run_id) + .expect("completed stale-lease run should remain in history"); + + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.active_runs[0].run_id, active_run_id); + assert_eq!(snapshot.active_runs[0].phase, "executing"); + assert_eq!(project.active_run_count, 1); + assert_eq!(snapshot.recent_runs.len(), 2); + assert_eq!(completed_run.phase, "completed"); + assert!( + completed_run.active_lease, + "regression setup should keep the stale lease visible in history" + ); +} + +#[test] +fn operator_status_snapshot_rolls_current_child_bucket_elapsed_time_into_bucket() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + let started_at = OffsetDateTime::now_utc().unix_timestamp() - 90; + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 1, + last_event_type: "item/tool/call", + child_agent_activity: Some(&ChildAgentActivitySummary { + buckets: vec![state::ChildAgentActivityBucket { + name: String::from("Tracker"), + event_count: 1, + tool_call_count: 1, + ..state::ChildAgentActivityBucket::default() + }], + current_bucket: Some(String::from("Tracker")), + current_detail: Some(String::from("issue_progress_checkpoint")), + current_started_unix_epoch: Some(started_at), + current_elapsed_seconds: Some(0), + event_count: 1, + tool_call_count: 1, + ..ChildAgentActivitySummary::default() + }), + protocol_activity: None, + }, + ) + .expect("protocol activity marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should exist"); + let activity = run.child_agent_activity.as_ref().expect("activity should render"); + let protocol_activity = + run.protocol_activity.as_ref().expect("protocol fallback should render"); + let tracker_bucket = + activity.buckets.iter().find(|bucket| bucket.name == "Tracker").expect("tracker bucket"); + + assert_eq!(run.wait_reason.as_deref(), Some("tool_execution")); + assert_eq!(protocol_activity.waiting_reason.as_deref(), Some("tool_execution")); + assert!(activity.current_elapsed_seconds.is_some_and(|elapsed| elapsed >= 90)); + assert!( + tracker_bucket.wall_seconds >= 90, + "current tool-call elapsed time should contribute to tracker bucket wall time" + ); +} + +#[test] +fn operator_status_snapshot_uses_structured_protocol_activity_summary() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + let protocol_activity = ProtocolActivitySummary { + turn_status: Some(String::from("running")), + waiting_reason: Some(String::from("approval_or_user_input")), + rate_limit_status: Some(String::from("primary")), + recent_events: vec![ + state::ProtocolActivityEventSummary { + event_type: String::from("plan/update"), + category: String::from("plan"), + detail: Some(String::from("verify")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("item/tool/requestUserInput"), + category: String::from("item"), + detail: None, + }, + ], + }; + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 2, + last_event_type: "item/tool/requestUserInput", + child_agent_activity: None, + protocol_activity: Some(&protocol_activity), + }, + ) + .expect("protocol activity marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should exist"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(run.wait_reason.as_deref(), Some("approval_or_user_input")); + assert_eq!(run.protocol_activity.as_ref(), Some(&protocol_activity)); + assert!(rendered.contains("protocol_activity: turn=running; waiting=approval_or_user_input; rate_limit=primary; recent=item/tool/requestUserInput, plan/update:verify")); +} + +#[test] +fn operator_status_snapshot_ignores_marker_from_newer_attempt_for_stored_run() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "failed") + .expect("stored run should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-2", 2, process::id()) + .expect("newer attempt marker should write"); + state::write_run_retry_schedule( + &worktree_path, + "run-2", + 2, + "failure", + OffsetDateTime::now_utc().unix_timestamp() + 60, + ) + .expect("retry schedule marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.recent_runs.first().expect("recent run should exist"); + + assert_eq!(run.run_id, "run-1"); + assert_eq!(run.phase, "failed"); + assert_eq!(run.wait_reason, None); + assert_eq!(run.process_id, None); + assert_eq!(run.process_alive, None); + assert_eq!(run.retry_kind, None); + assert_eq!(run.next_retry_at, None); +} + +#[test] +fn operator_status_snapshot_keeps_all_active_runs_when_recent_runs_are_limited() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let first_issue = sample_issue_with_sort_fields( + "issue-1", + "PUB-101", + "Todo", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let second_issue = sample_issue_with_sort_fields( + "issue-2", + "PUB-102", + "Todo", + &[], + Some(3), + "2026-03-13T04:17:17.133Z", + ); + + for (run_id, issue, branch_suffix) in + [("run-1", &first_issue, "101"), ("run-2", &second_issue, "102")] + { + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + &format!("x/pubfi-pub-{branch_suffix}"), + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should record"); + } + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 1) + .expect("snapshot should build"); + + assert_eq!(snapshot.run_limit, 1); + assert_eq!(snapshot.recent_runs.len(), 2); + assert_eq!(snapshot.active_runs.len(), 2); + assert!(snapshot.active_runs.iter().all(|run| run.active_lease)); +} + +#[test] +fn operator_status_snapshot_keeps_terminal_run_after_lane_cleanup() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-1", + "PUB-101", + "Done", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + + state_store.record_run_attempt("run-done", &issue.id, 1, "running").expect("run should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-done", "In Progress") + .expect("active lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should record"); + state_store.update_run_status("run-done", "succeeded").expect("terminal status should update"); + state_store.clear_lease(&issue.id).expect("terminal cleanup should clear active lease"); + state_store.clear_worktree(&issue.id).expect("terminal cleanup should clear worktree"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 25) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(snapshot.active_runs.is_empty()); + assert_eq!(snapshot.recent_runs.len(), 1); + assert_eq!(snapshot.recent_runs[0].run_id, "run-done"); + assert_eq!(snapshot.recent_runs[0].phase, "completed"); + assert!(!snapshot.recent_runs[0].active_lease); + assert_eq!(snapshot.recent_runs[0].branch_name, None); + assert_eq!(snapshot.recent_runs[0].worktree_path, None); + assert_eq!(snapshot.history_lanes.len(), 1); + assert_eq!(snapshot.history_lanes[0].latest_run.run_id, "run-done"); + assert!(rendered.contains("Run ledger shown: 1 issue lanes from 1 history attempts")); + assert!(rendered.contains("run_id: run-done")); +} + +#[test] +fn status_hydration_does_not_fabricate_active_leases_for_recovered_candidates() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + orchestrator::hydrate_status_snapshot_state( + &config, + &state_store, + RecoveredRuntimeState { active_issues: vec![issue.clone()] }, + ) + .expect("status hydration should succeed"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + + assert!( + snapshot.active_runs.is_empty(), + "recovered retry candidates should not appear as active leased runs" + ); + assert!( + snapshot.recent_runs.is_empty(), + "status hydration should not persist synthetic recovered runs" + ); +} + +#[test] +fn live_operator_status_snapshot_hydrates_active_run_thread_and_event_metadata_from_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join(&issue.identifier); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker(&worktree_path, "run-1", 1) + .expect("activity marker should write"); + state::write_run_thread_marker(&worktree_path, "run-1", 1, "thread-1") + .expect("thread marker should write"); + state::write_run_turn_marker(&worktree_path, "run-1", 1, "turn-1") + .expect("turn marker should write"); + state::write_run_thread_status_marker( + &worktree_path, + "run-1", + 1, + Some("thread-1"), + Some("turn-1"), + "active", + &[String::from("waitingOnApproval")], + ) + .expect("thread status should write"); + state::write_run_effective_runtime_marker( + &worktree_path, + "run-1", + 1, + &EffectiveRuntimeMarker { + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + effective_model: "gpt-5.4", + effective_model_provider: "openai", + effective_cwd: "/tmp/worktree", + effective_approval_policy: "never", + effective_approvals_reviewer: "human", + effective_sandbox_mode: "workspaceWrite", + }, + ) + .expect("effective runtime should write"); + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 2, + last_event_type: "turn/completed", + child_agent_activity: None, + protocol_activity: None, + }, + ) + .expect("protocol summary should write"); + + let recovered_state = orchestrator::recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("runtime recovery should succeed"); + + orchestrator::hydrate_status_snapshot_state(&config, &state_store, recovered_state) + .expect("status hydration should succeed"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.active_runs[0].thread_id.as_deref(), Some("thread-1")); + assert_eq!(snapshot.active_runs[0].turn_id.as_deref(), Some("turn-1")); + assert_eq!(snapshot.active_runs[0].thread_status.as_deref(), Some("active")); + assert_eq!( + snapshot.active_runs[0].thread_active_flags, + vec![String::from("waitingOnApproval")] + ); + assert!(snapshot.active_runs[0].interactive_requested); + assert_eq!(snapshot.active_runs[0].event_count, 2); + assert_eq!(snapshot.active_runs[0].last_event_type.as_deref(), Some("turn/completed")); + assert_eq!(snapshot.active_runs[0].effective_model.as_deref(), Some("gpt-5.4")); + assert_eq!(snapshot.active_runs[0].effective_model_provider.as_deref(), Some("openai")); + assert_eq!(snapshot.active_runs[0].effective_approval_policy.as_deref(), Some("never")); + assert_eq!(snapshot.active_runs[0].effective_sandbox_mode.as_deref(), Some("workspaceWrite")); + assert!(snapshot.active_runs[0].last_event_at.is_some()); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/text.rs b/apps/decodex/src/orchestrator/tests/operator/status/text.rs new file mode 100644 index 00000000..1c259159 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status/text.rs @@ -0,0 +1,262 @@ +#[test] +fn operator_status_text_renders_human_readable_sections() { + let active_run = operator_status_text_active_run(); + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: vec![active_run.clone()], + queued_candidates: operator_status_text_queued_candidates(), + recent_runs: vec![active_run], + history_lanes: Vec::new(), + worktrees: operator_status_text_worktrees(), + post_review_lanes: operator_status_text_post_review_lanes(), + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("Project: pubfi")); + assert!(rendered.contains("Warnings: 0")); + assert!(rendered.contains("Running Lanes")); + assert!(rendered.contains( + "Run ledger shown: 0 issue lanes from 0 history attempts (running lanes inline)" + )); + assert!(rendered.contains("Run Ledger")); + assert!(rendered.contains("- none (running lanes are shown above)")); + assert!(rendered.contains("run_id: run-1")); + assert_eq!(rendered.matches("run_id: run-1").count(), 1); + assert!(rendered.contains("attempt_status: running")); + assert!(rendered.contains("phase: executing")); + assert!(rendered.contains("current_operation: agent_run")); + assert!(rendered.contains("queue_lease_state: held")); + assert!(rendered.contains("queue_lease: held")); + assert!(rendered.contains("execution_liveness: process_alive")); + assert!(rendered.contains( + "timing: run_idle=1 protocol_idle=1 last_progress=2026-03-14 10:00:01Z protocol_event=turn/completed @ 2026-03-14 10:00:01 events=4" + )); + assert!(rendered.contains( + "account: account=...acct01; plan=pro; status=selected; token=ok; primary=5h remaining=72%" + )); + assert!(rendered.contains( + "accounts: account=...acct01; plan=pro; status=selected" + )); + assert!(rendered.contains( + "account=...acct02; plan=plus; status=available; token=ok; primary=5h remaining=41%" + )); + assert!(rendered.contains( + "child_agent_activity: current=Model 10m52s; wall=12m14s; buckets=Model 11m33s, Browser/Image 41s; tool_calls=3" + )); + assert!(rendered.contains( + "protocol_activity: turn=completed; waiting=model_execution; rate_limit=none; recent=turn/completed:completed, item/tool/call:view_image" + )); + assert!(rendered.contains( + "context_pressure: input=current_window 105.0k, peak_window 105.0k (same as current), cumulative_input 4.27M; output_tokens=12.0k; largest_output=175.8KiB by view_image; warnings=view_image repeated 3 large outputs; largest 180000 bytes" + )); + assert!(rendered.contains("turn_id: turn-1")); + assert!(rendered.contains("thread_status: active")); + assert!(rendered.contains("thread_active_flags: waitingOnApproval")); + assert!(rendered.contains("interactive_requested: yes")); + assert!(rendered.contains("effective_model: gpt-5.4")); + assert!(rendered.contains("freshness_at: 2026-03-14 10:00:00Z")); + assert!(rendered.contains("freshness_source: last_run_activity_at")); + assert!(rendered.contains("updated_at: 2026-03-14 09:00:00")); + assert!(rendered.contains("last_run_activity_at: 2026-03-14 10:00:00Z")); + assert!(rendered.contains("last_progress_at: 2026-03-14 10:00:01Z")); + assert!(rendered.contains("protocol_event: turn/completed @ 2026-03-14 10:00:01")); + assert!(rendered.contains("Backlog: 1")); + assert!(rendered.contains("Active queue echoes: 1")); + assert!(rendered.contains("Stale closed queue labels: 1")); + assert!(rendered.contains("Backlog")); + assert!(rendered.contains("issue: PUB-102")); + assert!(rendered.contains("classification: ready")); + assert!(rendered.contains("Active Queue Echoes")); + assert!(rendered.contains("issue: PUB-101")); + assert!(rendered.contains("running_owner_run: run-1")); + assert!(rendered.contains("Stale Closed Queue Labels")); + assert!(rendered.contains("issue: PUB-105")); + assert!(rendered.contains("classification: closed")); + assert!(rendered.contains("Recovery worktrees: 2")); + assert!(rendered.contains("Recovery Worktrees")); + assert!(!rendered.contains("role: running_lane")); + assert!(rendered.contains("role: post_review_lane")); + assert!(rendered.contains("role: cleanup_only")); + assert!(rendered.contains("worktree_path: .worktrees/PUB-103")); + assert!(rendered.contains("worktree_path: .worktrees/PUB-104")); + + assert_recovery_worktree_roles_are_grouped(&rendered); +} + +#[test] +fn operator_status_text_explains_empty_backlog_checks() { + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: Vec::new(), + queued_candidates: Vec::new(), + recent_runs: Vec::new(), + history_lanes: Vec::new(), + worktrees: Vec::new(), + post_review_lanes: Vec::new(), + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("Backlog: 0")); + assert!(rendered.contains("Hint: check `Todo`")); + assert!(rendered.contains("`decodex:queued:`")); + assert!(rendered.contains("`decodex:queued:pubfi`")); + assert!(rendered.contains("opt-out/manual-only")); + assert!(rendered.contains("needs-attention")); + assert!(rendered.contains("non-terminal state")); + assert!(rendered.contains("dependency blockers")); + assert!(rendered.contains("available capacity")); +} + +#[test] +fn operator_status_text_surfaces_cleanup_blocker_pr_url() { + let pr_url = "https://github.com/hack-ink/decodex/pull/119"; + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: Vec::new(), + queued_candidates: Vec::new(), + recent_runs: Vec::new(), + history_lanes: Vec::new(), + worktrees: vec![orchestrator::OperatorWorktreeStatus { + issue_id: String::from("issue-3"), + issue_identifier: Some(String::from("PUB-103")), + issue_state: Some(String::from("Done")), + branch_name: String::from("x/pubfi-pub-103"), + worktree_path: String::from(".worktrees/PUB-103"), + ownership: String::from("post_review_lane"), + ownership_reason: String::from( + "Review & Landing owns this worktree as `cleanup_blocked`.", + ), + hygiene: None, + }], + post_review_lanes: vec![orchestrator::OperatorPostReviewLaneStatus { + issue_id: String::from("issue-3"), + issue_identifier: String::from("PUB-103"), + issue_state: String::from("Done"), + branch_name: String::from("x/pubfi-pub-103"), + worktree_path: String::from(".worktrees/PUB-103"), + classification: String::from("cleanup_blocked"), + reason: String::from("retry_budget_exhausted"), + pr_url: Some(String::from(pr_url)), + pr_state: Some(String::from("MERGED")), + review_decision: Some(String::from("APPROVED")), + mergeable: Some(String::from("MERGEABLE")), + check_state: Some(String::from("SUCCESS")), + unresolved_review_threads: Some(0), + }], + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("classification: cleanup_blocked")); + assert!(rendered.contains("reason: retry_budget_exhausted")); + assert!(rendered.contains(&format!("pr_url: {pr_url}"))); + assert!(!rendered.contains("pr_url: none")); +} + +#[test] +fn operator_status_text_terminal_run_freshness_uses_terminal_update() { + let mut terminal_run = operator_status_text_active_run(); + + terminal_run.status = String::from("succeeded"); + terminal_run.phase = String::from("completed"); + terminal_run.active_lease = true; + terminal_run.updated_at = String::from("2026-03-14 10:05:00"); + terminal_run.last_run_activity_at = Some(String::from("2026-03-14 10:10:00Z")); + + let history_lanes = orchestrator::operator_history_lanes(&[], &[terminal_run.clone()]); + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: Vec::new(), + queued_candidates: Vec::new(), + recent_runs: vec![terminal_run], + history_lanes, + worktrees: Vec::new(), + post_review_lanes: Vec::new(), + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("run_id: run-1")); + assert!(rendered.contains("phase: completed")); + assert!(rendered.contains("active_lease: yes")); + assert!(rendered.contains("freshness_at: 2026-03-14 10:05:00")); + assert!(rendered.contains("freshness_source: updated_at")); + assert!(rendered.contains("last_run_activity_at: 2026-03-14 10:10:00Z")); +} + +#[test] +fn operator_status_text_active_run_without_live_activity_does_not_promote_updated_at() { + let mut active_run = operator_status_text_active_run(); + + active_run.updated_at = String::from("2026-03-14 09:00:00"); + active_run.last_run_activity_at = None; + active_run.last_protocol_activity_at = None; + active_run.last_progress_at = None; + + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: vec![active_run.clone()], + queued_candidates: Vec::new(), + recent_runs: vec![active_run], + history_lanes: Vec::new(), + worktrees: Vec::new(), + post_review_lanes: Vec::new(), + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("freshness_at: none")); + assert!(rendered.contains("freshness_source: none")); + assert!(rendered.contains("updated_at: 2026-03-14 09:00:00")); +} + +#[test] +fn operator_status_text_explains_unleased_live_running_lane() { + let mut active_run = operator_status_text_active_run(); + + active_run.active_lease = false; + active_run.queue_lease_state = String::from("not_held"); + + let snapshot = OperatorStatusSnapshot { + project_id: String::from("pubfi"), + run_limit: 10, + warnings: Vec::new(), + connector_backoffs: Vec::new(), + projects: Vec::new(), + accounts: Vec::new(), + active_runs: vec![active_run.clone()], + queued_candidates: Vec::new(), + recent_runs: vec![active_run], + history_lanes: Vec::new(), + worktrees: Vec::new(), + post_review_lanes: Vec::new(), + }; + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!(rendered.contains("active_lease: no")); + assert!(rendered.contains("queue_lease_state: not_held")); + assert!(rendered.contains("queue_lease: not_held (process_alive keeps lane visible)")); + assert!(rendered.contains("execution_liveness: process_alive")); +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status_support.rs b/apps/decodex/src/orchestrator/tests/operator/status_support.rs new file mode 100644 index 00000000..02fe8f6c --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/operator/status_support.rs @@ -0,0 +1,413 @@ +use std::{panic, slice}; + +use orchestrator::{ + OperatorPostReviewLaneStatus, OperatorQueuedIssueStatus, OperatorSnapshotReadiness, + OperatorWorktreeStatus, +}; +use serde_json::Value; + +fn successful_linear_execution_history_comments(issue: &TrackerIssue) -> Vec { + vec![ + linear_execution_history_comment( + issue, + "intake", + "2026-04-29T10:00:00Z", + "intake", + |record| { + record.summary = Some(String::from("Queued issue for Decodex execution.")); + }, + ), + linear_execution_history_comment( + issue, + "review_handoff", + "2026-04-29T10:05:00Z", + "review", + |record| { + record.branch = Some(String::from("y/decodex-xy-355")); + record.worktree_path = Some(String::from(".worktrees/XY-355")); + record.pr_url = Some(String::from("https://github.com/hack-ink/decodex/pull/355")); + record.pr_head_sha = Some(String::from("1111111111111111111111111111111111111111")); + record.pr_base_ref = Some(String::from("main")); + record.commit_sha = Some(String::from("1111111111111111111111111111111111111111")); + record.validation_result = Some(String::from("passed")); + record.summary = Some(String::from("Opened a reviewable PR.")); + record.terminal_path = Some(String::from("review_handoff")); + }, + ), + linear_execution_history_comment( + issue, + "landed", + "2026-04-29T10:08:00Z", + "landed", + |record| { + record.branch = Some(String::from("y/decodex-xy-355")); + record.pr_url = Some(String::from("https://github.com/hack-ink/decodex/pull/355")); + record.pr_head_sha = Some(String::from("1111111111111111111111111111111111111111")); + record.pr_base_ref = Some(String::from("main")); + record.commit_sha = Some(String::from("2222222222222222222222222222222222222222")); + record.summary = Some(String::from("Merged the PR.")); + }, + ), + linear_execution_history_comment( + issue, + "needs_attention", + "2026-04-29T10:06:00Z", + "earlier-attention", + |record| { + record.summary = Some(String::from("Earlier attempt required operator attention.")); + record.error_class = Some(String::from("validation_failed")); + record.next_action = Some(String::from("Re-run the repaired lane.")); + record.blockers = Some(Vec::new()); + record.evidence = Some(vec![String::from("cargo make test failed")]); + record.terminal_path = Some(String::from("manual_attention")); + }, + ), + linear_execution_history_comment( + issue, + "closeout", + "2026-04-29T10:10:00Z", + "closeout", + |record| { + record.branch = Some(String::from("y/decodex-xy-355")); + record.pr_url = Some(String::from("https://github.com/hack-ink/decodex/pull/355")); + record.commit_sha = Some(String::from("2222222222222222222222222222222222222222")); + record.summary = Some(String::from("Completed retained closeout.")); + record.target_state = Some(String::from("Done")); + }, + ), + ] +} + +fn successful_linear_execution_history_comments_with_cleanup( + issue: &TrackerIssue, +) -> Vec { + let mut comments = successful_linear_execution_history_comments(issue); + + comments.push(linear_execution_history_comment( + issue, + "cleanup_complete", + "2026-04-29T10:11:00Z", + "cleanup", + |record| { + record.branch = Some(String::from("y/decodex-xy-355")); + record.worktree_path = Some(String::from(".worktrees/XY-355")); + record.pr_url = Some(String::from("https://github.com/hack-ink/decodex/pull/355")); + record.commit_sha = Some(String::from("2222222222222222222222222222222222222222")); + record.cleanup_status = Some(String::from("completed")); + record.summary = Some(String::from("Cleaned up the retained lane.")); + }, + )); + + comments +} + +fn seed_local_linear_execution_events(state_store: &StateStore, comments: &[TrackerComment]) { + for comment in comments { + let record = records::parse_linear_execution_event_record(&comment.body) + .expect("test comment should contain a valid Linear execution event"); + + state_store + .record_linear_execution_event(&record) + .expect("local execution event should persist"); + } +} + +fn linear_execution_history_comment( + issue: &TrackerIssue, + event_type: &str, + event_timestamp: &str, + stable_anchor: &str, + configure: F, +) -> TrackerComment +where + F: FnOnce(&mut records::LinearExecutionEventRecord), +{ + let mut record = records::LinearExecutionEventRecord::new( + LinearExecutionEventIdentity { + service_id: TEST_SERVICE_ID, + issue_id: &issue.id, + issue_identifier: &issue.identifier, + run_id: "xy-355-attempt-1-1777527013", + attempt_number: 1, + }, + event_type, + event_timestamp.to_owned(), + stable_anchor, + ); + + configure(&mut record); + + records::validate_linear_execution_event_record(&record) + .expect("test ledger record should be valid"); + + TrackerComment { + body: records::append_structured_comment_record( + &format!("Decodex execution event: {event_type}"), + &record, + ) + .expect("structured comment should serialize"), + created_at: event_timestamp.to_owned(), + } +} + +fn operator_status_text_codex_account() -> state::CodexAccountActivitySummary { + state::CodexAccountActivitySummary { + account_fingerprint: String::from("...acct01"), + email: Some(String::from("primary@example.com")), + plan_type: Some(String::from("pro")), + status: String::from("selected"), + refresh_status: String::from("not_needed"), + checked_at_unix_epoch: Some(1_742_000_000), + selected_at_unix_epoch: Some(1_742_000_001), + primary_window_seconds: Some(18_000), + primary_remaining_percent: Some(72), + primary_resets_at_unix_epoch: Some(1_742_018_000), + secondary_window_seconds: Some(604_800), + secondary_remaining_percent: Some(91), + secondary_resets_at_unix_epoch: Some(1_742_604_800), + credits_has_credits: Some(true), + credits_unlimited: Some(false), + credits_balance: Some(String::from("9.99")), + rate_limit_reached_type: None, + cooldown_until_unix_epoch: None, + note: Some(String::from("usage probe ok")), + } +} + +fn operator_status_text_backup_codex_account() -> state::CodexAccountActivitySummary { + state::CodexAccountActivitySummary { + account_fingerprint: String::from("...acct02"), + email: Some(String::from("backup@example.com")), + plan_type: Some(String::from("plus")), + status: String::from("available"), + refresh_status: String::from("not_needed"), + checked_at_unix_epoch: Some(1_742_000_002), + selected_at_unix_epoch: None, + primary_window_seconds: Some(18_000), + primary_remaining_percent: Some(41), + primary_resets_at_unix_epoch: Some(1_742_010_000), + secondary_window_seconds: Some(604_800), + secondary_remaining_percent: Some(88), + secondary_resets_at_unix_epoch: Some(1_742_590_000), + credits_has_credits: Some(true), + credits_unlimited: Some(false), + credits_balance: Some(String::from("4.20")), + rate_limit_reached_type: None, + cooldown_until_unix_epoch: None, + note: Some(String::from("usage probe ok")), + } +} + +fn operator_status_text_active_run() -> orchestrator::OperatorRunStatus { + let account = operator_status_text_codex_account(); + let backup_account = operator_status_text_backup_codex_account(); + + orchestrator::OperatorRunStatus { + project_id: String::from("pubfi"), + run_id: String::from("run-1"), + issue_id: String::from("issue-1"), + issue_identifier: Some(String::from("PUB-101")), + title: Some(String::from("Implement orchestration")), + attempt_number: 1, + status: String::from("running"), + attempt_status: String::from("running"), + phase: String::from("executing"), + wait_reason: None, + current_operation: String::from(RUN_OPERATION_AGENT_RUN), + thread_id: Some(String::from("thread-1")), + turn_id: Some(String::from("turn-1")), + thread_status: Some(String::from("active")), + thread_active_flags: vec![String::from("waitingOnApproval")], + interactive_requested: true, + continuation_pending: false, + active_lease: true, + queue_lease_state: String::from("held"), + execution_liveness: String::from("process_alive"), + updated_at: String::from("2026-03-14 09:00:00"), + last_run_activity_at: Some(String::from("2026-03-14 10:00:00Z")), + last_protocol_activity_at: Some(String::from("2026-03-14 10:00:01Z")), + last_progress_at: Some(String::from("2026-03-14 10:00:01Z")), + idle_for_seconds: Some(1), + protocol_idle_for_seconds: Some(1), + suspected_stall: false, + last_event_type: Some(String::from("turn/completed")), + last_event_at: Some(String::from("2026-03-14 10:00:01")), + event_count: 4, + process_id: Some(1_234), + process_alive: Some(true), + retry_kind: None, + next_retry_at: None, + effective_model: Some(String::from("gpt-5.4")), + effective_model_provider: Some(String::from("openai")), + effective_cwd: Some(String::from("/tmp/worktree")), + effective_approval_policy: Some(String::from("never")), + effective_approvals_reviewer: Some(String::from("human")), + effective_sandbox_mode: Some(String::from("workspaceWrite")), + child_agent_activity: Some(ChildAgentActivitySummary { + buckets: vec![ + state::ChildAgentActivityBucket { + name: String::from("Model"), + wall_seconds: 693, + event_count: 12, + tool_call_count: 0, + input_tokens: 4_270_000, + output_tokens: 12_000, + output_bytes: 0, + }, + state::ChildAgentActivityBucket { + name: String::from("Browser/Image"), + wall_seconds: 41, + event_count: 6, + tool_call_count: 3, + input_tokens: 0, + output_tokens: 0, + output_bytes: 180_000, + }, + ], + current_bucket: Some(String::from("Model")), + current_detail: Some(String::from("waiting after tool output")), + current_started_unix_epoch: None, + current_elapsed_seconds: Some(652), + wall_seconds: 734, + event_count: 18, + tool_call_count: 3, + input_tokens_current: Some(105_000), + input_tokens_max: Some(105_000), + input_tokens_cumulative: 4_270_000, + output_tokens_cumulative: 12_000, + largest_tool_output_bytes: Some(180_000), + largest_tool_output_tool: Some(String::from("view_image")), + large_output_warnings: vec![String::from( + "view_image repeated 3 large outputs; largest 180000 bytes", + )], + }), + protocol_activity: Some(ProtocolActivitySummary { + turn_status: Some(String::from("completed")), + waiting_reason: Some(String::from("model_execution")), + rate_limit_status: Some(String::from("none")), + recent_events: vec![ + state::ProtocolActivityEventSummary { + event_type: String::from("item/tool/call"), + category: String::from("item"), + detail: Some(String::from("view_image")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("turn/completed"), + category: String::from("turn"), + detail: Some(String::from("completed")), + }, + ], + }), + account: Some(account.clone()), + accounts: vec![account, backup_account], + branch_name: Some(String::from("x/pubfi-pub-101")), + worktree_path: Some(String::from(".worktrees/PUB-101")), + } +} + +fn operator_status_text_queued_candidates() -> Vec { + vec![ + orchestrator::OperatorQueuedIssueStatus { + issue_id: String::from("issue-1"), + issue_identifier: String::from("PUB-101"), + title: String::from("Running lane still has a backlog claim"), + state: String::from("In Progress"), + priority: Some(1), + created_at: String::from("2026-03-14T09:57:00Z"), + classification: String::from("claimed"), + reason: String::from("shared_claim_present"), + attention: None, + blocker_identifiers: vec![], + }, + orchestrator::OperatorQueuedIssueStatus { + issue_id: String::from("issue-2"), + issue_identifier: String::from("PUB-102"), + title: String::from("Implement backlog surface"), + state: String::from("Todo"), + priority: Some(2), + created_at: String::from("2026-03-14T09:58:00Z"), + classification: String::from("ready"), + reason: String::from("eligible_for_dispatch"), + attention: None, + blocker_identifiers: vec![], + }, + orchestrator::OperatorQueuedIssueStatus { + issue_id: String::from("issue-5"), + issue_identifier: String::from("PUB-105"), + title: String::from("Remove stale queue label"), + state: String::from("Done"), + priority: Some(3), + created_at: String::from("2026-03-14T09:59:00Z"), + classification: String::from("closed"), + reason: String::from("terminal_state"), + attention: None, + blocker_identifiers: vec![], + }, + ] +} + +fn operator_status_text_worktrees() -> Vec { + vec![ + orchestrator::OperatorWorktreeStatus { + issue_id: String::from("issue-4"), + issue_identifier: Some(String::from("PUB-104")), + issue_state: None, + branch_name: String::from("x/pubfi-pub-104"), + worktree_path: String::from(".worktrees/PUB-104"), + ownership: String::from("cleanup_only"), + ownership_reason: String::from( + "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", + ), + hygiene: None, + }, + orchestrator::OperatorWorktreeStatus { + issue_id: String::from("issue-1"), + issue_identifier: Some(String::from("PUB-101")), + issue_state: Some(String::from("In Progress")), + branch_name: String::from("x/pubfi-pub-101"), + worktree_path: String::from(".worktrees/PUB-101"), + ownership: String::from("active_lane"), + ownership_reason: String::from("Active lane `run-1` owns this worktree."), + hygiene: None, + }, + orchestrator::OperatorWorktreeStatus { + issue_id: String::from("issue-3"), + issue_identifier: Some(String::from("PUB-103")), + issue_state: Some(String::from("In Review")), + branch_name: String::from("x/pubfi-pub-103"), + worktree_path: String::from(".worktrees/PUB-103"), + ownership: String::from("post_review_lane"), + ownership_reason: String::from( + "Review & Landing owns this worktree as `ready_to_land`.", + ), + hygiene: None, + }, + ] +} + +fn operator_status_text_post_review_lanes() -> Vec { + vec![orchestrator::OperatorPostReviewLaneStatus { + issue_id: String::from("issue-3"), + issue_identifier: String::from("PUB-103"), + issue_state: String::from("In Review"), + branch_name: String::from("x/pubfi-pub-103"), + worktree_path: String::from(".worktrees/PUB-103"), + classification: String::from("ready_to_land"), + reason: String::from("checks_green"), + pr_url: Some(String::from("https://github.com/hack-ink/decodex/pull/103")), + pr_state: Some(String::from("OPEN")), + review_decision: Some(String::from("APPROVED")), + mergeable: Some(String::from("MERGEABLE")), + check_state: Some(String::from("SUCCESS")), + unresolved_review_threads: Some(0), + }] +} + +fn assert_recovery_worktree_roles_are_grouped(rendered: &str) { + let post_review_role_index = + rendered.find("role: post_review_lane").expect("post-review role should render"); + let recovery_role_index = + rendered.find("role: cleanup_only").expect("recovery role should render"); + + assert!(post_review_role_index < recovery_role_index); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/closeout/cleanup.rs b/apps/decodex/src/orchestrator/tests/recovery/closeout/cleanup.rs new file mode 100644 index 00000000..83764c17 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/closeout/cleanup.rs @@ -0,0 +1,518 @@ +#[test] +fn cleanup_completed_post_review_lane_preserves_worktree_when_remote_delete_fails() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let _tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + let head_oid = git_output(config.repo_root(), &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/67"; + let worktree = WorktreeSpec { + branch_name: String::from("main"), + issue_identifier: issue.identifier.clone(), + path: config.repo_root().to_path_buf(), + reused_existing: true, + }; + let _path_guard = install_fake_merged_pr_gh_response_with_delete_exit_code( + &temp_dir, + &worktree, + pr_url, + &head_oid, + 1, + ); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &ReviewHandoffMarker::new( + "run-closeout-cleanup", + 1, + "main", + pr_url, + "main", + "main", + head_oid, + ), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "main", + &config.repo_root().display().to_string(), + ) + .expect("worktree should record"); + + let issue_run = sample_closeout_issue_run(&issue, &worktree, "run-closeout-cleanup"); + let error = orchestrator::cleanup_completed_post_review_lane( + &config, + &workflow, + &state_store, + &issue_run, + ) + .expect_err("remote branch delete failures must stop cleanup"); + + assert!( + error.to_string().contains("Failed to delete retained remote branch"), + "cleanup should surface the remote delete failure" + ); + assert!( + config.repo_root().exists(), + "cleanup must preserve the retained worktree when remote delete fails" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "worktree mapping must remain so cleanup can retry later" + ); + assert!( + !git_output(config.repo_root(), &["branch", "--list", "main"]).is_empty(), + "cleanup must not mutate local branch state when remote delete fails before local cleanup" + ); +} + +#[test] +fn merged_closeout_retry_exhaustion_reports_cleanup_blocker_with_pr_url_after_default_branch_dirty() +{ + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/119"; + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["push", "origin", &format!("HEAD:{}", worktree.branch_name)]) + .status() + .expect("git push lane branch should run") + .success() + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + fs::write(config.repo_root().join("README.md"), "local repo override\n") + .expect("tracked repo-root file should become dirty"); + + let issue_run = sample_closeout_issue_run(&issue, &worktree, "run-closeout-dirty-root"); + let error = orchestrator::cleanup_completed_post_review_lane( + &config, + &workflow, + &state_store, + &issue_run, + ) + .expect_err("tracked repo-root dirtiness must block default-branch sync"); + + assert!( + error.to_string().contains("tracked local changes"), + "cleanup should surface the default-branch sync blocker: {error:?}" + ); + assert!(worktree.path.exists(), "cleanup must preserve the retained worktree"); + + for attempt in 1..=3 { + state_store + .record_run_attempt( + &format!("run-closeout-dirty-root-{attempt}"), + &issue.id, + attempt, + "failed", + ) + .expect("failed closeout attempt should record"); + } + + let mut merged_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + merged_review_state.state = String::from("MERGED"); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(merged_review_state)]), + ) + .expect("post-review status should classify the retained cleanup blocker"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "cleanup_blocked"); + assert_eq!(lanes[0].reason, "default_branch_worktree_dirty"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); + assert_eq!(lanes[0].pr_state.as_deref(), Some("MERGED")); +} + +#[test] +fn cleanup_completed_post_review_lane_fails_closed_when_pr_target_branch_drifted() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let _tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/180"; + let _path_guard = install_fake_merged_pr_gh_response_with_base_ref( + &temp_dir, + &worktree, + pr_url, + &head_oid, + "release/1.x", + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: worktree.branch_name.clone(), + issue_identifier: issue.identifier.clone(), + path: worktree.path.clone(), + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Closeout, + attempt_number: 1, + run_id: String::from("run-closeout-retargeted-pr"), + retry_budget_base: 0, + }; + let error = orchestrator::cleanup_completed_post_review_lane( + &config, + &workflow, + &state_store, + &issue_run, + ) + .expect_err("cleanup must fail closed when the merged PR target branch drifted"); + + assert!( + error.to_string().contains("expected PR") && error.to_string().contains("release/1.x"), + "cleanup should surface the authoritative PR target-branch mismatch" + ); + assert!( + worktree.path.exists(), + "cleanup must preserve the retained worktree when the merged PR target branch drifted" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "worktree mapping must remain so cleanup can retry after a corrected handoff" + ); +} + +#[test] +fn cleanup_completed_post_review_lane_deletes_local_lane_branch_after_worktree_cleanup() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let _tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/181"; + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["push", "origin", &format!("HEAD:{}", worktree.branch_name)]) + .status() + .expect("git push lane branch should run") + .success() + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let issue_run = + sample_closeout_issue_run(&issue, &worktree, "run-closeout-local-branch-cleanup"); + + orchestrator::cleanup_completed_post_review_lane(&config, &workflow, &state_store, &issue_run) + .expect("cleanup should succeed once merged closeout is authoritative"); + + assert!(!worktree.path.exists(), "cleanup should remove the retained worktree path"); + assert!( + git_output(config.repo_root(), &["branch", "--list", &worktree.branch_name]).is_empty(), + "cleanup should delete the retained local lane branch" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_none(), + "cleanup should clear retained worktree state after local branch deletion succeeds" + ); +} + +#[test] +fn cleanup_completed_post_review_lane_uses_persisted_handoff_marker() { + let cases = [ + ( + "matching current branch", + "https://github.com/hack-ink/decodex/pull/181", + "run-closeout-cleanup", + 1, + "run-closeout-cleanup", + ), + ( + "stale run identity with current marker", + "https://github.com/hack-ink/decodex/pull/182", + "run-closeout-stale", + 7, + "run-closeout-current", + ), + ]; + + for (case_name, pr_url, marker_run_id, marker_attempt, issue_run_id) in cases { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let _path_guard = + install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + git_status_success( + config.repo_root(), + &["push", "origin", &format!("HEAD:{}", worktree.branch_name)], + ); + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &ReviewHandoffMarker::new( + marker_run_id, + marker_attempt, + &worktree.branch_name, + pr_url, + "main", + &worktree.branch_name, + &head_oid, + ), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let issue_run = sample_closeout_issue_run(&issue, &worktree, issue_run_id); + + orchestrator::cleanup_completed_post_review_lane( + &config, + &workflow, + &state_store, + &issue_run, + ) + .expect("cleanup should use the persisted handoff marker"); + + assert!( + !worktree.path.exists(), + "cleanup should remove the retained worktree path for {case_name}" + ); + } +} + +#[test] +fn cleanup_completed_post_review_lane_preserves_worktree_when_local_branch_delete_fails() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Done", &[]); + let _tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/182"; + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + let blocking_worktree = config.worktree_root().join("blocking-local-branch-delete"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["push", "origin", &format!("HEAD:{}", worktree.branch_name)]) + .status() + .expect("git push lane branch should run") + .success() + ); + assert!( + Command::new("git") + .arg("-C") + .arg(&worktree.path) + .args(["checkout", "--quiet", "--detach"]) + .status() + .expect("git checkout --detach should run") + .success() + ); + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args([ + "worktree", + "add", + "--quiet", + blocking_worktree.to_string_lossy().as_ref(), + &worktree.branch_name, + ]) + .status() + .expect("git worktree add should run") + .success() + ); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let issue_run = + sample_closeout_issue_run(&issue, &worktree, "run-closeout-local-branch-delete-blocked"); + let error = orchestrator::cleanup_completed_post_review_lane( + &config, + &workflow, + &state_store, + &issue_run, + ) + .expect_err("cleanup should fail closed when another worktree still holds the lane branch"); + + assert!( + error.to_string().contains("Failed to delete retained local branch"), + "cleanup should surface the local branch deletion failure" + ); + assert!( + worktree.path.exists(), + "cleanup must preserve the retained worktree when local branch deletion fails" + ); + assert!( + state_store + .review_handoff_marker(config.service_id(), &issue.id, &worktree.branch_name) + .expect("review handoff marker read should succeed") + .is_some(), + "cleanup must preserve the runtime review handoff marker so closeout can retry later" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "worktree mapping must remain so cleanup can retry later" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs b/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs new file mode 100644 index 00000000..a0f8743e --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs @@ -0,0 +1,310 @@ +#[test] +fn closeout_dispatch_completes_merged_lane_without_agent_turn() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = issue_with_completed_state(sample_issue("In Review", &[active_label.as_str()])); + let mut completed_issue = issue.clone(); + + completed_issue.state = + TrackerState { id: String::from("state-done"), name: String::from("Done") }; + + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![ + vec![issue.clone()], + vec![issue.clone()], + vec![completed_issue.clone()], + vec![completed_issue.clone()], + ], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/701"; + let _path_guard = install_fake_closeout_gh_responses(&temp_dir, &worktree, pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + route_origin_github_url_to_local_bare_repo(config.repo_root(), &remote_root); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["push", "origin", &format!("HEAD:{}", worktree.branch_name)]) + .status() + .expect("git push lane branch should run") + .success() + ); + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let issue_run = sample_closeout_issue_run(&issue, &worktree, "pub-701-attempt-3-closeout"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "starting") + .expect("run attempt should record"); + + let summary = + orchestrator::execute_issue_run(&tracker, &config, &workflow, &state_store, issue_run) + .expect("deterministic closeout should complete"); + + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!( + tracker.state_updates.borrow().as_slice(), + [(issue.id.clone(), String::from("state-done"))] + ); + assert_eq!(tracker.comments.borrow().len(), 2); + assert!(tracker.comments.borrow()[0].contains("decodex closeout completed")); + + let event_types = tracker + .comments + .borrow() + .iter() + .filter_map(|comment| records::parse_linear_execution_event_record(comment)) + .map(|record| record.event_type) + .collect::>(); + + assert_eq!(event_types, vec![String::from("closeout"), String::from("cleanup_complete")]); + assert!(!worktree.path.exists(), "deterministic closeout should remove the retained worktree"); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_none(), + "deterministic closeout should clear retained worktree state" + ); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "deterministic closeout should not leave an active lease" + ); + assert_eq!( + tracker.label_removals.borrow().len(), + 2, + "deterministic closeout should clear active and queue lane labels" + ); +} + +#[test] +fn direct_closeout_dispatch_reuses_completed_handoff_run_identity_for_record_and_summary() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + + assert_closeout_lane_ready(&fixture); + + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &fixture.tracker, + project: &fixture.config, + workflow: &fixture.workflow, + state_store: &fixture.state_store, + issue_id: &fixture.issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: false, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Closeout, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("direct retained closeout should run") + .expect("closeout summary should be printed"); + let message = orchestrator::format_run_once_summary(&summary, false); + + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, fixture.completed_run_id); + assert_eq!(summary.attempt_number, 1); + assert!( + message.contains(&format!("run_id={}", fixture.completed_run_id)), + "terminal summary should print the completed handoff run id: {message}" + ); + assert!( + !message.contains("attempt-2"), + "direct closeout must not look like a hidden retry: {message}" + ); + + let issue_comments = fixture.tracker.issue_comments.borrow(); + let closeout_comments = + issue_comments.get(&fixture.issue.id).expect("closeout should write an issue comment"); + + assert!( + closeout_comments.iter().any(|comment| { + records::parse_linear_execution_event_record(&comment.body).is_some_and(|record| { + record.event_type == "closeout" + && record.run_id == fixture.completed_run_id + && record.attempt_number == 1 + && record.branch.as_deref() == Some(fixture.worktree.branch_name.as_str()) + && record.pr_url.as_deref() == Some(fixture.pr_url.as_str()) + }) + }), + "direct closeout event should reuse the completed handoff identity" + ); + assert!( + fixture + .state_store + .run_attempt_for_issue_attempt(&fixture.issue.id, 2) + .expect("second attempt lookup should succeed") + .is_none(), + "successful direct closeout should not create an invisible second attempt" + ); +} + +#[test] +fn same_run_closeout_reuses_matching_active_handoff_lease() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + + assert_closeout_lane_ready(&fixture); + + fixture + .state_store + .upsert_lease( + fixture.config.service_id(), + &fixture.issue.id, + &fixture.completed_run_id, + "In Review", + ) + .expect("handoff lease should recover before same-run closeout"); + + let source_summary = RunSummary { + project_id: fixture.config.service_id().to_owned(), + issue_id: fixture.issue.id.clone(), + issue_identifier: fixture.issue.identifier.clone(), + issue_state: String::from("In Review"), + initial_issue_state: String::from("Todo"), + retry_project_slug: fixture.config.service_id().to_owned(), + dispatch_mode: IssueDispatchMode::Normal, + branch_name: fixture.worktree.branch_name.clone(), + worktree_path: fixture.worktree.path.clone(), + attempt_number: 1, + run_id: fixture.completed_run_id.clone(), + continuation_pending: false, + }; + let summary = orchestrator::run_retained_closeout_for_handoff_summary( + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + &source_summary, + ) + .expect("same-run retained closeout should run") + .expect("same-run retained closeout should produce a summary"); + + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, fixture.completed_run_id); + assert!( + fixture + .state_store + .lease_for_issue(&fixture.issue.id) + .expect("lease lookup should succeed") + .is_none(), + "same-run closeout should clear the recovered handoff lease" + ); +} + +#[test] +fn closeout_completed_state_check_skips_redundant_transition() { + let (_temp_dir, _config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = issue_with_completed_state(sample_issue("Done", &[active_label.as_str()])); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let worktree = WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: PathBuf::from(".worktrees/PUB-101"), + reused_existing: true, + }; + let issue_run = sample_closeout_issue_run(&issue, &worktree, "pub-101-closeout-done"); + + orchestrator::ensure_closeout_issue_completed_state(&tracker, &workflow, &issue_run) + .expect("completed issue should not require another transition"); + + assert!( + tracker.state_updates.borrow().is_empty(), + "already completed issues should not be transitioned again" + ); +} + +#[test] +fn closeout_dispatch_validates_pr_before_marking_issue_done() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = issue_with_completed_state(sample_issue("In Review", &[active_label.as_str()])); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/702"; + let _path_guard = install_fake_closeout_gh_responses_with_state( + &temp_dir, &worktree, pr_url, &head_oid, "OPEN", + ); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + route_origin_github_url_to_local_bare_repo(config.repo_root(), &remote_root); + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let issue_run = sample_closeout_issue_run(&issue, &worktree, "pub-702-attempt-1-closeout"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "starting") + .expect("run attempt should record"); + + let error = + orchestrator::execute_issue_run(&tracker, &config, &workflow, &state_store, issue_run) + .expect_err("unmerged PR should stop deterministic closeout"); + + assert!( + error.to_string().contains("must be merged before closeout completes"), + "closeout should fail at PR validation: {error:?}" + ); + assert!( + tracker.state_updates.borrow().is_empty(), + "closeout must not mark the issue done before PR validation succeeds" + ); + assert!( + !tracker + .comments + .borrow() + .iter() + .any(|comment| comment.contains("decodex closeout completed")), + "closeout must not write a closeout completion record when PR validation fails" + ); + assert!( + !tracker + .comments + .borrow() + .iter() + .any(|comment| comment.contains("decodex run failed and needs attention")), + "closeout PR visibility races should remain retryable instead of terminal" + ); + assert!(worktree.path.exists(), "failed closeout should preserve the retained worktree"); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/closeout/identity.rs b/apps/decodex/src/orchestrator/tests/recovery/closeout/identity.rs new file mode 100644 index 00000000..78b1a1e8 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/closeout/identity.rs @@ -0,0 +1,277 @@ +#[test] +fn run_project_once_closeout_reuses_completed_handoff_run_identity_for_record_and_summary() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + + assert_closeout_lane_ready(&fixture); + + let planned = orchestrator::run_project_once( + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + true, + ) + .expect("retained closeout dry-run planning should succeed"); + let planned = + planned.expect("retained closeout should be selected before deterministic execution"); + + assert_eq!(planned.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(planned.run_id, fixture.completed_run_id); + assert_eq!(planned.attempt_number, 1); + + let summary = orchestrator::run_project_once( + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + false, + ) + .expect("retained closeout should run") + .expect("closeout summary should be printed"); + let message = orchestrator::format_run_once_summary(&summary, false); + + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, fixture.completed_run_id); + assert_eq!(summary.attempt_number, 1); + assert!( + message.contains(&format!("run_id={}", fixture.completed_run_id)), + "terminal summary should print the completed handoff run id: {message}" + ); + assert!( + !message.contains("attempt-2"), + "successful closeout must not look like a hidden retry: {message}" + ); + + let issue_comments = fixture.tracker.issue_comments.borrow(); + let closeout_comments = + issue_comments.get(&fixture.issue.id).expect("closeout should write an issue comment"); + + assert!( + closeout_comments.iter().any(|comment| { + records::parse_linear_execution_event_record(&comment.body).is_some_and(|record| { + record.event_type == "closeout" + && record.run_id == fixture.completed_run_id + && record.attempt_number == 1 + && record.branch.as_deref() == Some(fixture.worktree.branch_name.as_str()) + && record.pr_url.as_deref() == Some(fixture.pr_url.as_str()) + }) + }), + "closeout event should reuse the completed handoff identity" + ); + assert_eq!( + fixture + .state_store + .run_attempt(&fixture.completed_run_id) + .expect("run attempt lookup should succeed") + .expect("completed handoff attempt should remain recorded") + .status(), + "succeeded" + ); + assert!( + fixture + .state_store + .run_attempt_for_issue_attempt(&fixture.issue.id, 2) + .expect("second attempt lookup should succeed") + .is_none(), + "successful closeout should not create an invisible second attempt" + ); + assert_eq!( + fixture + .state_store + .next_attempt_number(&fixture.issue.id) + .expect("next attempt lookup should succeed"), + 2, + "the store should only know about the completed first attempt" + ); +} + +#[test] +fn daemon_planned_closeout_reuses_handoff_identity_after_parent_failed_status() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + let mut retry_queue = RetryQueue::default(); + + assert_closeout_lane_ready(&fixture); + + fixture + .state_store + .update_run_status(&fixture.completed_run_id, "failed") + .expect("daemon parent failed status should record"); + + seed_review_orchestration_marker_for_path( + &fixture.state_store, + fixture.config.service_id(), + &fixture.worktree.path, + &ReviewOrchestrationMarker::new( + &fixture.completed_run_id, + 1, + &fixture.worktree.branch_name, + &fixture.pr_url, + &fixture.head_oid, + "waiting_for_merge", + Some(TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID), + Some(TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT), + Some(0), + 0, + 1, + Some(TEST_EXTERNAL_REVIEW_AUTO_MERGE_ENABLED_AT), + ), + ); + + let (summary, from_retry_queue) = orchestrator::plan_next_daemon_run( + &mut retry_queue, + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + ) + .expect("daemon planning should succeed") + .expect("retained closeout should be selected"); + + assert!(!from_retry_queue, "status-visible closeout should not consume a retry claim"); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, fixture.completed_run_id); + assert_eq!(summary.attempt_number, 1); + assert!( + fixture + .state_store + .try_acquire_lease( + fixture.config.service_id(), + &summary.issue_id, + &summary.run_id, + &summary.issue_state, + ) + .expect("daemon parent should acquire the planned closeout lease") + ); + + let daemon_spawn_state = orchestrator::materialize_daemon_spawn_state( + &fixture.config, + &fixture.workflow, + &fixture.state_store, + &summary, + ) + .expect("daemon parent should materialize planned closeout state"); + + fixture + .state_store + .record_run_attempt(&summary.run_id, &summary.issue_id, summary.attempt_number, "starting") + .expect("daemon parent should record the planned closeout attempt"); + fixture + .state_store + .upsert_worktree( + fixture.config.service_id(), + &summary.issue_id, + &daemon_spawn_state.worktree.branch_name, + &daemon_spawn_state.worktree.path.display().to_string(), + ) + .expect("daemon parent should retain closeout worktree mapping"); + + let handoff_attempt = fixture + .state_store + .run_attempt(&fixture.completed_run_id) + .expect("handoff attempt lookup should succeed") + .expect("handoff attempt should still exist"); + + assert_eq!(handoff_attempt.attempt_number(), 1); + assert_eq!(handoff_attempt.status(), "starting"); + assert!( + fixture + .state_store + .run_attempt_for_issue_attempt(&fixture.issue.id, 2) + .expect("second attempt lookup should succeed") + .is_none(), + "daemon planning must not materialize a synthetic attempt 2" + ); +} + +#[test] +fn daemon_planned_closeout_allocates_retry_after_recorded_closeout_failure() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + let mut retry_queue = RetryQueue::default(); + + assert_closeout_lane_ready(&fixture); + + fixture + .state_store + .update_run_status(&fixture.completed_run_id, "failed") + .expect("failed closeout attempt should record"); + + state::write_run_retry_schedule( + &fixture.worktree.path, + &fixture.completed_run_id, + 1, + "failure", + 12_345, + ) + .expect("failed closeout retry schedule should write"); + + let (summary, from_retry_queue) = orchestrator::plan_next_daemon_run( + &mut retry_queue, + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + ) + .expect("daemon planning should succeed") + .expect("retained closeout should be selected"); + + assert!(!from_retry_queue, "normal planner fallback should not consume a retry claim"); + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.attempt_number, 2); + assert_ne!(summary.run_id, fixture.completed_run_id); +} + +#[test] +fn run_project_once_closeout_preserves_handoff_identity_after_fresh_activity_recovery() { + let fixture = closeout_identity_fixture(); + let _keep_fixture_alive = (&fixture._temp_dir, &fixture._path_guard); + + state::write_run_activity_marker(&fixture.worktree.path, &fixture.completed_run_id, 1) + .expect("fresh handoff activity should write"); + + let summary = orchestrator::run_project_once( + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + false, + ) + .expect("retained closeout should run after recovery") + .expect("closeout summary should be printed"); + let message = orchestrator::format_run_once_summary(&summary, false); + + assert_eq!(summary.dispatch_mode, IssueDispatchMode::Closeout); + assert_eq!(summary.run_id, fixture.completed_run_id); + assert_eq!(summary.attempt_number, 1); + assert!( + !message.contains("attempt-2"), + "recovered handoff closeout must not report a synthetic retry: {message}" + ); + assert_eq!( + fixture + .state_store + .run_attempt(&fixture.completed_run_id) + .expect("run attempt lookup should succeed") + .expect("completed handoff attempt should remain recorded") + .status(), + "succeeded" + ); + assert!( + fixture + .state_store + .run_attempt_for_issue_attempt(&fixture.issue.id, 2) + .expect("second attempt lookup should succeed") + .is_none(), + "recovered closeout should not create an invisible second attempt" + ); + assert!( + fixture + .state_store + .lease_for_issue(&fixture.issue.id) + .expect("lease lookup should succeed") + .is_none(), + "successful recovered closeout should not leave the rebuilt handoff lease" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs new file mode 100644 index 00000000..fe410df8 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -0,0 +1,987 @@ +fn reconciliation_sample_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +#[test] +fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let terminal_issue = sample_issue_with_sort_fields( + "issue-terminal", + "PUB-201", + "Done", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let nonactive_issue = sample_issue_with_sort_fields( + "issue-nonactive", + "PUB-202", + "Blocked", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let stalled_issue = sample_issue_with_sort_fields( + "issue-stalled", + "PUB-203", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![ + terminal_issue.clone(), + nonactive_issue.clone(), + stalled_issue.clone(), + ]); + + for issue in [&terminal_issue, &nonactive_issue, &stalled_issue] { + state_store + .record_run_attempt(&format!("run-{}", issue.identifier), &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, &format!("run-{}", issue.identifier), "In Progress") + .expect("lease should record"); + } + + state_store + .append_event( + &format!("run-{}", stalled_issue.identifier), + 1, + "thread/status/changed", + "{\"status\":\"active\"}", + ) + .expect("stalled issue protocol event should record"); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("active-run inspection should succeed"); + + assert!(actions.iter().any(|action| { + action.issue.id == terminal_issue.id + && matches!(action.disposition, orchestrator::ActiveRunDisposition::Terminal) + })); + assert!(actions.iter().any(|action| { + action.issue.id == nonactive_issue.id + && matches!(action.disposition, orchestrator::ActiveRunDisposition::NonActive) + })); + assert!(actions.iter().any(|action| { + action.issue.id == stalled_issue.id + && matches!( + action.disposition, + ActiveRunDisposition::Stalled{ idle_for } + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + ) + })); +} + +#[test] +fn active_run_reconciliation_detects_stalled_run_without_protocol_events() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let stalled_issue = sample_issue_with_sort_fields( + "issue-stalled-no-events", + "PUB-204", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![stalled_issue.clone()]); + + state_store + .record_run_attempt( + &format!("run-{}", stalled_issue.identifier), + &stalled_issue.id, + 1, + "running", + ) + .expect("run attempt should record"); + state_store + .upsert_lease( + "pubfi", + &stalled_issue.id, + &format!("run-{}", stalled_issue.identifier), + "In Progress", + ) + .expect("lease should record"); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("active-run inspection should succeed"); + + assert!(actions.iter().any(|action| { + action.issue.id == stalled_issue.id + && matches!( + action.disposition, + ActiveRunDisposition::Stalled{ idle_for } + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + ) + })); +} + +#[test] +fn active_run_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_ACTIVE_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = reconciliation_sample_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-closeout-active"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/180"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store.upsert_lease("pubfi", &issue.id, run_id, "Done").expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state::write_run_activity_marker(&worktree.path, run_id, 1) + .expect("fresh activity marker should write"); + + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + OffsetDateTime::now_utc().unix_timestamp(), + ) + .expect("active-run inspection should succeed"); + + assert!( + actions.is_empty(), + "completed retained closeout lanes with fresh activity must not be reconciled as terminal or non-active" + ); +} + +#[test] +fn active_daemon_child_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_ACTIVE_DAEMON_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = reconciliation_sample_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-daemon-closeout-active"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/181"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store.upsert_lease("pubfi", &issue.id, run_id, "Done").expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state::write_run_activity_marker(&worktree.path, run_id, 1) + .expect("fresh activity marker should write"); + + let actions = orchestrator::inspect_active_daemon_child_reconciliation( + &tracker, + &config, + &workflow, + &state_store, + ActiveChildRunContext { + child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + workflow: &workflow, + dispatch_mode: IssueDispatchMode::Closeout, + }, + ) + .expect("active daemon-child inspection should succeed"); + + assert!( + actions.is_empty(), + "completed retained closeout daemon children with fresh activity must not be reconciled as terminal or non-active" + ); +} + +#[test] +fn active_daemon_child_reconciliation_keeps_review_repair_lane_in_review() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = reconciliation_sample_service_owned_issue("In Review"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-review-repair-active"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained review-repair worktree should exist"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "In Review") + .expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let actions = orchestrator::inspect_active_daemon_child_reconciliation( + &tracker, + &config, + &workflow, + &state_store, + ActiveChildRunContext { + child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + workflow: &workflow, + dispatch_mode: IssueDispatchMode::ReviewRepair, + }, + ) + .expect("active review-repair daemon-child inspection should succeed"); + + assert!( + actions.is_empty(), + "active review-repair lanes in In Review must stay active instead of being interrupted as non-active" + ); +} + +#[test] +fn active_daemon_child_reconciliation_keeps_closeout_child_after_tracker_completion() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Done", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-closeout-completed"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "Done") + .expect("lease should record"); + + let actions = orchestrator::inspect_active_daemon_child_reconciliation( + &tracker, + &config, + &workflow, + &state_store, + ActiveChildRunContext { + child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + workflow: &workflow, + dispatch_mode: IssueDispatchMode::Closeout, + }, + ) + .expect("active closeout daemon-child inspection should succeed"); + + assert!( + actions.is_empty(), + "closeout children may legitimately observe a completed tracker issue while they finish local cleanup" + ); +} + +#[test] +fn active_run_reconciliation_treats_completed_retained_handoff_as_success() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue_with_sort_fields( + "issue-handoff-complete", + "PUB-205", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained review worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/205"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + state_store + .append_event(run_id, 1, "thread/status/changed", "{\"status\":\"active\"}") + .expect("stale run activity should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("active-run inspection should succeed"); + + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0].disposition, ActiveRunDisposition::RetainedReviewComplete)); + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + actions, + ) + .expect("completed retained handoff reconciliation should succeed"); + + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "retained post-review worktree must stay available for merge/closeout" + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "succeeded" + ); + assert!( + tracker.comments.borrow().iter().all(|comment| !comment.contains("stalled_run_detected")), + "completed retained handoff must not be routed through needs-attention" + ); +} + +#[test] +fn active_run_reconciliation_ignores_stale_retained_handoff_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue_with_sort_fields( + "issue-stale-handoff", + "PUB-205B", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-current"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained review worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + state_store + .append_event(run_id, 1, "thread/status/changed", "{\"status\":\"active\"}") + .expect("stale run activity should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &ReviewHandoffMarker::new( + "run-previous", + 1, + &worktree.branch_name, + "https://github.com/hack-ink/decodex/pull/205", + "main", + &worktree.branch_name, + &head_oid, + ), + ); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("active-run inspection should succeed"); + + assert_eq!(actions.len(), 1); + assert!(matches!( + actions[0].disposition, + ActiveRunDisposition::Stalled { idle_for } + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + )); +} + +#[test] +fn active_daemon_child_reconciliation_treats_completed_retained_handoff_as_success() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue_with_sort_fields( + "issue-daemon-handoff-complete", + "PUB-206", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained review worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/206"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + state_store + .append_event(run_id, 1, "thread/status/changed", "{\"status\":\"active\"}") + .expect("stale run activity should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_daemon_child_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + ActiveChildRunContext { + child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + workflow: &workflow, + dispatch_mode: IssueDispatchMode::Normal, + }, + now, + ) + .expect("active daemon-child inspection should succeed"); + + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0].disposition, ActiveRunDisposition::RetainedReviewComplete)); +} + +#[test] +fn stalled_idle_duration_ignores_future_last_activity() { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let run_id = "run-future-activity"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + + let last_activity = state_store + .last_run_activity_unix_epoch(run_id) + .expect("last activity lookup should succeed") + .expect("run activity should exist"); + + assert_eq!( + orchestrator::stalled_idle_duration( + &state_store, + &state_store + .run_attempt(run_id) + .expect("run lookup should succeed") + .expect("run attempt should exist"), + None, + last_activity - 1 + ) + .expect("idle duration should evaluate"), + None + ); +} + +#[test] +fn active_run_reconciliation_uses_worktree_activity_marker_from_child_process() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-shared-activity"; + let worktree_path = config.worktree_root().join("PUB-101"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let last_activity = state_store + .last_run_activity_unix_epoch(run_id) + .expect("last activity lookup should succeed") + .expect("run activity should exist"); + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + + fs::write( + &marker_path, + format!( + "run_id={run_id}\nattempt_number=1\nlast_activity_unix_epoch={}\n", + last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + ), + ) + .expect("activity marker should write"); + + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + ) + .expect("active run inspection should succeed"); + + assert!( + actions.is_empty(), + "fresh child activity marker should prevent daemon stall reconciliation" + ); +} + +#[test] +fn stalled_protocol_idle_duration_ignores_future_protocol_activity() { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-protocol-future-activity"; + + state_store + .record_run_attempt(run_id, "issue-1", 1, "running") + .expect("run attempt should record"); + state_store + .append_event(run_id, 1, "thread/status/changed", "{\"status\":\"active\"}") + .expect("protocol event should record"); + + let run_attempt = state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist"); + let last_activity = state_store + .last_protocol_activity_unix_epoch(run_id) + .expect("protocol activity lookup should succeed") + .expect("protocol activity should exist"); + + assert_eq!( + orchestrator::stalled_protocol_idle_duration( + &state_store, + &run_attempt, + None, + last_activity - 1, + ) + .expect("protocol idle duration should evaluate"), + None + ); +} + +#[test] +fn active_run_reconciliation_ignores_startable_preclaim_states() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-startable", + "PUB-204", + "Todo", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + + state_store + .record_run_attempt("run-startable", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-startable", "In Progress") + .expect("lease should record"); + + let now = OffsetDateTime::now_utc().unix_timestamp() + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("active-run inspection should succeed"); + + assert!(actions.is_empty(), "startable pre-claim states should not be interrupted"); +} + +#[test] +fn active_run_reconciliation_clears_terminal_lane_labels() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = reconciliation_sample_service_owned_issue("Done"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let run_id = "run-terminal"; + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let action = ActiveRunReconciliation { + issue: issue.clone(), + run_attempt: state_store + .run_attempt(run_id) + .expect("run attempt query should succeed") + .expect("run attempt should exist"), + worktree_mapping: state_store + .worktree_for_issue(&issue.id) + .expect("worktree query should succeed"), + disposition: ActiveRunDisposition::Terminal, + workflow: workflow.clone(), + }; + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + vec![action], + ) + .expect("terminal reconciliation should succeed"); + + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [ + (issue.id.clone(), vec![String::from("label-active")]), + (issue.id.clone(), vec![String::from("label-queued")]), + ] + ); +} + +#[test] +fn active_run_reconciliation_keeps_nonterminal_nonactive_worktrees() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let issue = sample_issue("Todo", &[]); + let run_id = "run-nonactive"; + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let action = ActiveRunReconciliation { + issue: issue.clone(), + run_attempt: state_store + .run_attempt(run_id) + .expect("run attempt query should succeed") + .expect("run attempt should exist"), + worktree_mapping: state_store + .worktree_for_issue(&issue.id) + .expect("worktree query should succeed"), + disposition: ActiveRunDisposition::NonActive, + workflow: workflow.clone(), + }; + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + vec![action], + ) + .expect("reconciliation should succeed"); + + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some() + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "interrupted" + ); +} + +#[test] +fn stalled_run_reconciliation_routes_to_needs_attention_without_cleanup() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let issue = sample_issue("In Progress", &[]); + let run_id = "run-stalled"; + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let action = ActiveRunReconciliation { + issue: issue.clone(), + run_attempt: state_store + .run_attempt(run_id) + .expect("run attempt query should succeed") + .expect("run attempt should exist"), + worktree_mapping: state_store + .worktree_for_issue(&issue.id) + .expect("worktree query should succeed"), + disposition: ActiveRunDisposition::Stalled { + idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + }, + workflow: workflow.clone(), + }; + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + vec![action], + ) + .expect("reconciliation should succeed"); + + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some() + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "stalled" + ); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains("stalled_run_detected") + && comment.contains("needs attention") + && comment.contains("clear label `decodex:needs-attention`") + })); +} + +#[test] +fn stalled_run_reconciliation_preserves_retry_budget_marker_from_retained_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let issue = sample_issue("In Progress", &[]); + let run_id = "run-stalled-budget"; + let worktree_path = config.worktree_root().join("PUB-101"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_retry_budget_attempt_count(&worktree_path, "older-run", 2, 2) + .expect("retry budget marker should write"); + + state_store + .record_run_attempt(run_id, &issue.id, 3, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let action = ActiveRunReconciliation { + issue: issue.clone(), + run_attempt: state_store + .run_attempt(run_id) + .expect("run attempt query should succeed") + .expect("run attempt should exist"), + worktree_mapping: state_store + .worktree_for_issue(&issue.id) + .expect("worktree query should succeed"), + disposition: ActiveRunDisposition::Stalled { + idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + }, + workflow: workflow.clone(), + }; + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + vec![action], + ) + .expect("reconciliation should succeed"); + + assert_eq!( + state::read_run_retry_budget_attempt_count(&worktree_path) + .expect("retry budget marker should read") + .expect("retry budget marker should remain present"), + 2, + "stalled reconciliation should preserve the retained retry-budget base" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs b/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs new file mode 100644 index 00000000..4fe2a3fa --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs @@ -0,0 +1,1002 @@ +#[test] +fn exited_child_reconciliation_detects_stalled_failed_runs_from_protocol_idle() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-stalled-after-exit", + "PUB-205", + "In Progress", + &[], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let run_id = "run-stalled-after-exit"; + let worktree_path = config.worktree_root().join(&issue.identifier); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-205", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run should exit as failed before daemon inspects it"); + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id, + attempt_number: 1, + thread_id: None, + turn_id: None, + event_count: 1, + last_event_type: "thread/status/changed", + child_agent_activity: None, + protocol_activity: None, + }, + ) + .expect("protocol marker should write"); + + let last_protocol_activity = + state::read_run_protocol_activity_marker(&worktree_path, run_id, 1) + .expect("protocol marker should read") + .expect("protocol activity should exist"); + let actions = orchestrator::inspect_exited_daemon_child_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + &issue.id, + run_id, + last_protocol_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + ) + .expect("exited child inspection should succeed"); + + assert!(actions.iter().any(|action| { + action.issue.id == issue.id + && matches!( + action.disposition, + ActiveRunDisposition::Stalled{ idle_for } + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + ) + })); +} + +#[test] +fn run_project_once_prefers_recovered_in_progress_worktree_after_empty_state_startup() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let expected_worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should be created") + .path; + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovered dry run should succeed") + .expect("active recovered issue should be selected"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Retry); + assert_eq!(summary.worktree_path, expected_worktree); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "worktree mapping should be reconstructed from the retained lane" + ); +} + +#[test] +fn recover_runtime_state_recovers_fresh_review_repair_activity_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Review"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("review-repair worktree should exist"); + + state::write_run_activity_marker(&worktree.path, "run-review-repair", 1) + .expect("fresh activity marker should write"); + + let recovered = orchestrator::recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("runtime recovery should succeed"); + + assert!( + recovered.active_issues.is_empty(), + "fresh review-repair activity should rebuild the lease instead of requeueing the lane" + ); + + let lease = state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("fresh review-repair lane should rebuild its lease"); + + assert_eq!(lease.run_id(), "run-review-repair"); + assert_eq!(lease.issue_state(), workflow.frontmatter().tracker().success_state()); +} + +#[test] +fn run_project_once_recovers_retained_worktree_from_issue_identifier() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue_with_project_slug_and_sort_fields( + "issue-1", + "PUB-101", + "tracker-project", + "In Progress", + &[active_label.as_str()], + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let expected_worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should be created") + .path; + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovered dry run should succeed") + .expect("active recovered issue should be selected"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Retry); + assert_eq!(summary.worktree_path, expected_worktree); +} + +#[test] +fn run_project_once_recovers_ready_post_review_lane_before_landing() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&base_config, "PATH"), + false, + ); + let issue = sample_active_issue("In Review"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained review worktree should be created"); + let pr_url = "https://github.com/hack-ink/decodex/pull/333"; + let head_subject = + r#"{"schema":"decodex/commit/1","summary":"Add retry hint","authority":"PUB-101"}"#; + let landed_subject = + r#"{"schema":"decodex/commit/1","summary":"Land Add retry hint","authority":"PUB-101"}"#; + let head_oid = + commit_worktree_change(&worktree.path, "retained-ready.txt", "ready\n", head_subject); + let (_path_guard, invocation_log_path) = + install_fake_ready_to_land_admin_merge_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("recovered retained post-review lane should reconcile"); + + assert!( + summary.is_none(), + "ready retained post-review landing should not dispatch a new active run" + ); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + ); + let gh_invocation = fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!(marker.phase(), "waiting_for_merge"); + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + head_oid, + String::from("--subject"), + String::from(landed_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + ] + ); +} + +#[test] +fn materialize_run_summary_worktree_creates_worktree_before_child_activity_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("dry-run planning should succeed") + .expect("brand-new lane should be selected"); + + assert!( + !summary.worktree_path.exists(), + "dry-run planning should not materialize the worktree yet" + ); + + let worktree = orchestrator::materialize_run_summary_worktree(&config, &workflow, &summary) + .expect("daemon parent should materialize the worktree before child startup"); + + assert_eq!(worktree.path, summary.worktree_path); + assert_eq!(worktree.branch_name, summary.branch_name); + assert!( + worktree.path.exists(), + "materialized worktree should exist before writing child activity markers" + ); + + state::write_run_activity_marker_for_process( + &worktree.path, + &summary.run_id, + summary.attempt_number, + process::id(), + ) + .expect("child activity marker should write after worktree materialization"); +} + +#[test] +fn cleanup_terminal_worktree_runs_before_remove_workspace_hook() { + let workflow_markdown = r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = ["printf '%s:%s\n' \"$DECODEX_ISSUE_ID\" \"$DECODEX_BRANCH\" > \"$DECODEX_REPO_ROOT/before-remove.log\""] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Follow the repository policy. + "#; + let (_temp_dir, config, workflow) = + temp_project_layout_with_workflow_markdown(workflow_markdown); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree("PUB-101", false) + .expect("worktree should exist before cleanup"); + + orchestrator::cleanup_terminal_worktree( + &state_store, + &worktree_manager, + &workflow, + "issue-1", + "PUB-101", + &worktree.branch_name, + &worktree.path, + ) + .expect("cleanup should succeed"); + + assert_eq!( + fs::read_to_string(config.repo_root().join("before-remove.log")) + .expect("before-remove hook log should exist"), + "PUB-101:x/pubfi-pub-101\n" + ); + assert!(!worktree.path.exists(), "cleanup should still remove the worktree"); + assert!( + !git_output(config.repo_root(), &["branch", "--list", &worktree.branch_name]).is_empty(), + "generic terminal cleanup should preserve the retained local branch ref" + ); +} + +#[test] +fn materialize_daemon_spawn_state_uses_retained_retry_budget_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let retained_worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained worktree should exist"); + + state::write_run_retry_budget_attempt_count(&retained_worktree.path, "older-run", 4, 2) + .expect("retry budget marker should write"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("dry-run planning should succeed") + .expect("retained lane should still be selected"); + let daemon_spawn_state = + orchestrator::materialize_daemon_spawn_state(&config, &workflow, &state_store, &summary) + .expect("daemon parent should materialize worktree and retry budget together"); + + assert_eq!(daemon_spawn_state.worktree.path, summary.worktree_path); + assert_eq!( + daemon_spawn_state.retry_budget_base, 2, + "daemon child handoff should preserve retry budget from the retained worktree marker" + ); +} + +#[test] +fn run_project_once_skips_recovered_worktree_with_fresh_activity_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should exist"); + + state::write_run_activity_marker(&worktree.path, "run-1", 1) + .expect("activity marker should write"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovery should succeed"); + + assert!( + summary.is_none(), + "fresh child activity should recover as an active lane instead of redispatching" + ); + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("lease should be reconstructed") + .run_id(), + "run-1" + ); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should succeed") + .expect("run attempt should be reconstructed") + .status(), + "running" + ); +} + +#[cfg(unix)] +#[test] +fn process_is_alive_handles_current_process_and_invalid_sentinel() { + assert!( + orchestrator::process_is_alive(process::id()), + "current process should always be reported as alive" + ); + assert!( + !orchestrator::process_is_alive(u32::MAX), + "sentinel pid values should never be treated as live processes" + ); +} + +#[test] +fn run_project_once_clears_recovered_lease_when_marker_turns_stale() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Progress"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should exist"); + + state::write_run_activity_marker(&worktree.path, "run-1", 1) + .expect("fresh activity marker should write"); + + let initial_summary = + orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("initial recovery should succeed"); + + assert!( + initial_summary.is_none(), + "fresh recovered activity should block redispatch and reconstruct the live lease" + ); + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("recovered lease should exist") + .run_id(), + "run-1" + ); + + state::write_run_activity_marker_for_process(&worktree.path, "run-1", 1, u32::MAX) + .expect("stale activity marker should write"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("stale recovery should succeed") + .expect("stale recovered lease should no longer block retry planning"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Retry); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "stale recovered markers should clear the reconstructed lease before retry planning" + ); +} + +#[test] +fn run_project_once_skips_recovered_terminal_guarded_worktree_after_empty_state_startup() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue_without_needs_attention_team_label("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should exist"); + + fs::write( + worktree.path.join(TERMINAL_GUARD_MARKER_FILE), + "run_id=pub-101-attempt-1-123\nattempt_number=1\n", + ) + .expect("terminal guard marker should write"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovery should succeed"); + + assert!( + summary.is_none(), + "restart recovery should not redispatch retained lanes guarded by a terminal marker" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "worktree mapping should still be reconstructed for guarded retained lanes" + ); +} + +#[test] +fn run_project_once_clears_terminal_queued_lane_labels_without_dispatch() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("Done", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("terminal queued cleanup should succeed"); + + assert!(summary.is_none(), "terminal queued issues should not dispatch"); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [ + (issue.id.clone(), vec![String::from("label-active")]), + (issue.id.clone(), vec![String::from("label-queued")]), + ] + ); +} + +#[test] +fn run_project_once_dry_run_keeps_terminal_queued_lane_labels() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("Done", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("terminal queued dry run should succeed"); + + assert!(summary.is_none(), "terminal queued dry run should not dispatch"); + assert!( + tracker.label_removals.borrow().is_empty(), + "dry run should not mutate terminal queued labels" + ); +} + +#[test] +fn run_project_once_preserves_terminal_recovered_worktree_without_prior_state_when_review_handoff_is_missing() + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("terminal retained worktree should be created"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("reconciliation should finish cleanly"); + + assert!( + summary.is_none(), + "blocked retained closeout with missing handoff should not redispatch during recovery" + ); + assert!( + worktree.path.exists(), + "terminal recovery should preserve the retained closeout worktree on disk for manual intervention" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "terminal recovery should preserve the retained closeout worktree mapping when review handoff is missing" + ); +} + +#[test] +fn run_project_once_clears_stale_completed_closeout_lease_but_keeps_worktree() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_STARTUP_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = sample_active_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let run_id = "run-closeout-startup"; + let pr_url = "https://github.com/hack-ink/decodex/pull/178"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, &issue.state.name) + .expect("stale lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("startup reconciliation should succeed"); + + assert!( + summary.is_none(), + "blocked retained closeout should not redispatch during startup recovery" + ); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "startup reconciliation should clear stale completed closeout leases" + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should still exist") + .status(), + "interrupted" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "startup reconciliation should preserve the retained closeout worktree mapping" + ); + assert!( + worktree.path.exists(), + "startup reconciliation should leave the retained closeout worktree on disk" + ); +} + +#[test] +fn run_project_once_preserves_fresh_completed_closeout_lease() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_STARTUP_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = sample_active_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let run_id = "run-closeout-fresh-startup"; + let pr_url = "https://github.com/hack-ink/decodex/pull/178"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state::write_run_activity_marker(&worktree.path, run_id, 1) + .expect("fresh activity marker should write"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, &issue.state.name) + .expect("fresh lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("startup reconciliation should succeed"); + + assert!( + summary.is_none(), + "fresh retained closeout activity should block redispatch during startup recovery" + ); + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("fresh retained closeout lease should survive") + .run_id(), + run_id + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should still exist") + .status(), + "running" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "startup reconciliation should preserve the retained closeout worktree mapping" + ); +} + +#[test] +fn run_project_once_preserves_completed_unmerged_closeout_worktree() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_active_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let run_id = "run-closeout-open-pr-startup"; + let pr_url = "https://github.com/hack-ink/decodex/pull/179"; + let _path_guard = install_fake_open_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &issue.id, run_id, &issue.state.name) + .expect("stale lease should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("startup reconciliation should succeed"); + + assert!( + summary.is_none(), + "completed retained closeout with an open PR should stay blocked during startup recovery" + ); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "startup reconciliation should clear stale completed closeout leases when the PR is still open" + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should still exist") + .status(), + "interrupted" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "startup reconciliation should preserve the retained closeout worktree mapping until the PR merges" + ); + assert!( + worktree.path.exists(), + "startup reconciliation should leave the retained closeout worktree on disk while waiting for merge" + ); +} + +#[test] +fn run_project_once_skips_recovered_worktree_without_service_active_label() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("foreign retained worktree should exist"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovery should succeed"); + + assert!( + summary.is_none(), + "recovery should skip retained worktrees that are not explicitly owned by this service" + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_none(), + "foreign retained worktrees should not be reconstructed into local service state" + ); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "foreign retained worktrees should not rebuild service leases" + ); +} + +#[test] +fn run_project_once_recovers_worktree_when_identifier_lookup_labels_are_truncated() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let listed_issue = sample_active_issue("In Progress"); + let mut identifier_lookup_issue = listed_issue.clone(); + + identifier_lookup_issue.labels_complete = false; + + identifier_lookup_issue.labels.retain(|label| label.name != active_label); + + let tracker = FakeTracker::with_refresh_snapshots( + vec![listed_issue.clone()], + vec![vec![listed_issue.clone()]], + ) + .with_identifier_lookup_issues(vec![identifier_lookup_issue]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let expected_worktree = worktree_manager + .ensure_worktree(&listed_issue.identifier, false) + .expect("recovered worktree should be created") + .path; + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovery should succeed") + .expect("ambiguous label pagination should still recover the owned retained lane"); + + assert_eq!(summary.issue_id, listed_issue.id); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Retry); + assert_eq!(summary.worktree_path, expected_worktree); +} + +#[test] +fn live_run_skips_issue_that_becomes_ineligible_after_worktree_prepare() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let listed_issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![listed_issue.clone()], + vec![vec![], vec![listed_issue.clone()], vec![sample_issue("In Progress", &[])]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("run once should succeed"); + + assert!(summary.is_none()); + assert!( + state_store.lease_for_issue(&listed_issue.id).expect("lease lookup should work").is_none() + ); + assert!( + state_store + .worktree_for_issue(&listed_issue.id) + .expect("worktree lookup should work") + .is_some() + ); + assert!(tracker.comments.borrow().is_empty()); +} + +#[test] +fn live_run_clears_claimed_lease_when_refresh_fails_after_worktree_prepare() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let listed_issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_error(vec![listed_issue.clone()], "transient refresh failure"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let error = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect_err("run once should propagate refresh failure"); + + assert!( + error.to_string().contains("transient refresh failure"), + "error should surface the refresh failure" + ); + assert!( + state_store.lease_for_issue(&listed_issue.id).expect("lease lookup should work").is_none() + ); +} + +#[test] +fn run_project_once_ignores_fresh_marker_for_exited_process() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("recovered worktree should exist"); + let exited_process_id = u32::MAX; + + state::write_run_activity_marker_for_process(&worktree.path, "run-1", 1, exited_process_id) + .expect("activity marker should write"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, true) + .expect("recovery should succeed") + .expect("dead process marker should not block retry planning"); + + assert_eq!(summary.issue_id, issue.id); + assert_eq!(summary.dispatch_mode, orchestrator::IssueDispatchMode::Retry); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "dead marker recovery should not reconstruct a live lease" + ); +} + +#[test] +fn idle_daemon_recovery_reconstructs_completed_closeout_worktree_mapping() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_DAEMON_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = sample_active_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/178"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + orchestrator::recover_and_reconcile_idle_daemon_state( + &tracker, + &config, + &workflow, + &state_store, + &worktree_manager, + ) + .expect("idle daemon recovery should succeed"); + + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some(), + "idle daemon recovery should reconstruct retained closeout worktree mappings from disk" + ); + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), + "blocked retained closeout recovery should not invent a live lease without fresh activity" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs new file mode 100644 index 00000000..bed138aa --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -0,0 +1,452 @@ +#[test] +fn terminal_failures_without_needs_attention_label_use_nonstartable_guard_state() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let mut issue = + sample_issue_without_needs_attention_team_label("Todo", &[active_label.as_str()]); + + for label in &mut issue.labels { + label.id = issue + .team + .labels + .iter() + .find(|team_label| team_label.name == label.name) + .map(|team_label| team_label.id.clone()) + .expect("issue label should resolve to a team label id"); + } + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(ManualAttentionRequested { + issue_identifier: issue.identifier.clone(), + label: String::from("decodex:needs-attention"), + run_id: issue_run.run_id.clone(), + }); + + fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("terminal failure handling should succeed"); + + assert_eq!( + tracker.state_updates.borrow().last(), + Some(&(issue.id.clone(), String::from("state-progress"))) + ); + assert_eq!( + tracker.label_removals.borrow().last(), + Some(&(issue.id.clone(), vec![String::from("label-active")])), + "terminal failure should clear the active automation label even when needs-attention is unavailable" + ); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains("does not exist on the team") + && comment.contains("remains in `In Progress`") + })); + assert_eq!( + state_store + .run_attempt(&issue_run.run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + orchestrator::TERMINAL_GUARDED_RUN_STATUS + ); + assert!( + issue_run.worktree.path.join(orchestrator::TERMINAL_GUARD_MARKER_FILE).exists(), + "fallback guard should leave a durable worktree marker for restart recovery" + ); +} + +#[test] +fn terminal_failures_apply_incremental_label_mutations_when_issue_labels_paginate() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let mut issue = sample_issue("Todo", &[active_label.as_str()]); + + issue.labels_complete = false; + + for label in &mut issue.labels { + label.id = issue + .team + .labels + .iter() + .find(|team_label| team_label.name == label.name) + .map(|team_label| team_label.id.clone()) + .expect("issue label should resolve to a team label id"); + } + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(ManualAttentionRequested { + issue_identifier: issue.identifier.clone(), + label: String::from("decodex:needs-attention"), + run_id: issue_run.run_id.clone(), + }); + + fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect( + "terminal failure should use incremental label mutations when issue labels paginate", + ); + + assert_eq!( + tracker.label_additions.borrow().as_slice(), + [(issue.id.clone(), vec![String::from("label-needs-attention")])] + ); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [(issue.id.clone(), vec![String::from("label-active")])] + ); + assert!( + tracker + .comments + .borrow() + .iter() + .any(|comment| comment.contains("decodex run failed and needs attention")), + "terminal failure should still leave a durable tracker comment" + ); + + let ledger_event = tracker + .comments + .borrow() + .iter() + .find_map(|comment| records::parse_linear_execution_event_record(comment)) + .expect("terminal failure should write a Linear execution event"); + + assert_eq!(ledger_event.event_type, "needs_attention"); + assert_eq!(ledger_event.error_class.as_deref(), Some("human_attention_required")); + assert_eq!(ledger_event.terminal_path.as_deref(), Some("manual_attention")); +} + +#[test] +fn review_policy_exhausted_failures_skip_retry_and_require_attention_pre_pr() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(ReviewPolicyStopRequested { + head_sha: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + issue_identifier: issue.identifier.clone(), + nonclean_rounds: Some(3), + reason: ReviewPolicyStopReason::Exhausted, + run_id: issue_run.run_id.clone(), + }); + + fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("review policy failure handling should succeed"); + + assert_eq!( + tracker.state_updates.borrow().last(), + Some(&(issue.id.clone(), String::from("state-todo"))) + ); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains("review_policy_exhausted") + && comment.contains("decide the next repair or redesign manually") + && comment.contains("bounded convergence research follow-up") + && comment.contains("machine-checkable") + && comment.contains("clear label `decodex:needs-attention`") + })); + assert!( + !tracker + .comments + .borrow() + .iter() + .any(|comment| { comment.contains("retryable_execution_failure") }) + ); +} + +#[test] +fn review_policy_blocked_failures_skip_retry_and_require_attention_in_review() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::ReviewRepair, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2-123"), + retry_budget_base: 0, + }; + let error = Report::new(ReviewPolicyStopRequested { + head_sha: String::from("18a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + issue_identifier: issue.identifier.clone(), + nonclean_rounds: Some(1), + reason: ReviewPolicyStopReason::Blocked, + run_id: issue_run.run_id.clone(), + }); + + fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("review policy failure handling should succeed"); + + assert_eq!( + tracker.state_updates.borrow().last(), + Some(&(issue.id.clone(), String::from("state-todo"))) + ); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains("review_policy_blocked") + && comment.contains("resolve the blocker manually") + && comment.contains("do not dispatch research") + && comment.contains("clear label `decodex:needs-attention`") + })); + assert!( + !tracker + .comments + .borrow() + .iter() + .any(|comment| { comment.contains("retryable_execution_failure") }) + ); +} + +#[test] +fn app_server_failures_skip_retry_and_require_attention() { + assert_app_server_failure_requires_attention( + Report::new(AppServerCapabilityPreflightFailure::blocked_for_test( + "skills", + "skills/list returned no enabled skills.", + )), + "app_server_runtime_preflight_failed", + "repair the local Codex config/model/provider/skills/plugin/MCP state", + ); + assert_app_server_failure_requires_attention( + Report::new(AppServerHomePreflightFailure::resolution_failed(String::from( + "app_server_preflight_failed: HOME is not set, so Decodex cannot resolve the shared Codex home for app-server dispatch.", + ))), + "app_server_codex_home_preflight_failed", + "keep CODEX_HOME/CODEX_SQLITE_HOME shared instead of per-account", + ); + assert_app_server_failure_requires_attention( + Report::new(AppServerHomePreflightFailure::initialize_mismatch( + String::from("/tmp/per-account-codex-home"), + String::from("/Users/test/.codex"), + )), + "app_server_codex_home_mismatch", + "restart `decodex serve`", + ); + assert_app_server_failure_requires_attention( + Report::new(AppServerTransportFailure::new(String::from( + "App-server stdout disconnected unexpectedly.", + ))), + "app_server_transport_disconnected", + "resolve the Codex app-server transport failure manually", + ); + assert_app_server_failure_requires_attention( + Report::new(AppServerTurnFailure::new( + "thread-1", + Some(String::from("turn-1")), + "failed", + "You've hit your usage limit.", + Some(String::from("usageLimitExceeded")), + )), + "app_server_usage_limit_exceeded", + "inspect Codex account usage", + ); +} + +#[test] +fn prepare_issue_run_clears_terminal_guard_marker_when_new_attempt_starts() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots(vec![], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("worktree should exist before retry guard clearing"); + let marker_path = worktree.path.join(TERMINAL_GUARD_MARKER_FILE); + + fs::write(&marker_path, "stale terminal guard\n").expect("terminal guard marker should write"); + + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + issue, + ) + .expect("issue preparation should succeed") + .expect("startable issue should produce a run plan"); + + assert_eq!(issue_run.worktree.path, worktree.path); + assert!( + !marker_path.exists(), + "starting a new attempt should clear stale terminal-guard markers" + ); +} + +#[test] +fn retryable_failures_ignore_prior_continuation_attempts_in_writeback() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Retry, + attempt_number: 4, + run_id: String::from("pub-101-attempt-4-123"), + retry_budget_base: 0, + }; + + state_store + .record_run_attempt("pub-101-attempt-1-123", &issue.id, 1, "succeeded") + .expect("first continuation attempt should record"); + state_store + .record_run_attempt("pub-101-attempt-2-123", &issue.id, 2, "succeeded") + .expect("second continuation attempt should record"); + state_store + .record_run_attempt("pub-101-attempt-3-123", &issue.id, 3, "succeeded") + .expect("third continuation attempt should record"); + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("current failed attempt should record"); + + orchestrator::handle_failure( + &tracker, + &config, + &workflow, + &state_store, + &issue_run, + &Report::msg("command failed"), + ) + .expect("retryable failure handling should succeed"); + + assert!(tracker.state_updates.borrow().is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains("retryable_execution_failure") + && comment.contains("- attempt: `4`") + && comment.contains("- retry_budget_attempt: `1` / `3`") + })); + assert!(!tracker.comments.borrow().iter().any(|comment| { + comment.contains("needs attention") || comment.contains("retry_budget_exhausted") + })); +} + +#[test] +fn manual_attention_failure_overrides_succeeded_run_status() { + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("run-1", "issue-1", 1, "succeeded") + .expect("run attempt should record"); + state_store.update_run_status("run-1", "failed").expect("failed outcome should persist"); + + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "failed" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_support.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_support.rs new file mode 100644 index 00000000..edefb0f6 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_support.rs @@ -0,0 +1,657 @@ +#[cfg(unix)] use std::os::unix::fs::PermissionsExt; + +struct CloseoutIdentityFixture { + _temp_dir: TempDir, + _path_guard: TestEnvVarGuard, + config: ServiceConfig, + workflow: WorkflowDocument, + tracker: FakeTracker, + state_store: StateStore, + issue: TrackerIssue, + worktree: WorktreeSpec, + pr_url: String, + head_oid: String, + completed_run_id: String, +} + +fn sample_active_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn sample_active_issue_without_needs_attention_team_label(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue_without_needs_attention_team_label(state_name, &[active_label.as_str()]) +} + +fn install_fake_open_pr_gh_response( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, +) -> TestEnvVarGuard { + let fake_gh_dir = temp_dir.path().join("fake-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let fake_gh_response = serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "url": pr_url, + "state": "OPEN", + "isDraft": false, + "reviewDecision": "APPROVED", + "baseRefName": "main", + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "headRefName": worktree.branch_name.clone(), + "headRefOid": head_oid, + "headRepository": { "name": "decodex" }, + "headRepositoryOwner": { "login": "hack-ink" }, + "reviewRequests": { "totalCount": 0 }, + "reviewThreads": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "commits": { + "nodes": [ + { "commit": { "statusCheckRollup": { "state": "SUCCESS" } } } + ] + } + } + } + } + }) + .to_string(); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write(&fake_gh_path, format!("#!/bin/sh\nprintf '%s' '{fake_gh_response}'\n")) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())) +} + +fn install_fake_ready_to_land_admin_merge_gh_response( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, +) -> (TestEnvVarGuard, PathBuf) { + let fake_gh_dir = temp_dir.path().join("fake-ready-to-land-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let invocation_log_path = temp_dir.path().join("ready-to-land-gh-invocation.log"); + let fake_graphql_response = serde_json::json!({ + "data": { + "repository": { + "mergeCommitAllowed": true, + "pullRequest": { + "url": pr_url, + "state": "OPEN", + "isDraft": false, + "reviewDecision": "APPROVED", + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "headRefName": worktree.branch_name.clone(), + "headRefOid": head_oid, + "headRepository": { "name": "decodex" }, + "headRepositoryOwner": { "login": "hack-ink" }, + "reactionGroups": [], + "comments": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "reviews": { "nodes": [] }, + "reviewRequests": { "totalCount": 0 }, + "reviewThreads": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "commits": { + "nodes": [ + { "commit": { "statusCheckRollup": { "state": "SUCCESS" } } } + ] + } + } + } + } + }) + .to_string(); + let fake_pr_view_response = serde_json::json!({ + "state": "MERGED", + "headRefOid": head_oid, + "mergeCommit": { "oid": "cafebabe" }, + }) + .to_string(); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"graphql\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"merge\" ]; then\n\ + printf '%s\\n' \"$@\" >> '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"view\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + fake_graphql_response, + invocation_log_path.display(), + fake_pr_view_response + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + ( + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())), + invocation_log_path, + ) +} + +fn install_fake_merged_pr_gh_response( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, +) -> TestEnvVarGuard { + install_fake_merged_pr_gh_response_with_base_ref_and_delete_exit_code( + temp_dir, worktree, pr_url, head_oid, "main", 0, + ) +} + +fn install_fake_merged_pr_gh_response_with_base_ref( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, + base_ref_name: &str, +) -> TestEnvVarGuard { + install_fake_merged_pr_gh_response_with_base_ref_and_delete_exit_code( + temp_dir, + worktree, + pr_url, + head_oid, + base_ref_name, + 0, + ) +} + +fn install_fake_merged_pr_gh_response_with_delete_exit_code( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, + delete_exit_code: i32, +) -> TestEnvVarGuard { + install_fake_merged_pr_gh_response_with_base_ref_and_delete_exit_code( + temp_dir, + worktree, + pr_url, + head_oid, + "main", + delete_exit_code, + ) +} + +fn install_fake_merged_pr_gh_response_with_base_ref_and_delete_exit_code( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, + base_ref_name: &str, + delete_exit_code: i32, +) -> TestEnvVarGuard { + let fake_gh_dir = temp_dir.path().join("fake-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let fake_gh_response = serde_json::json!({ + "data": { + "repository": { + "mergeCommitAllowed": true, + "pullRequest": { + "url": pr_url, + "state": "MERGED", + "isDraft": false, + "reviewDecision": "APPROVED", + "baseRefName": base_ref_name, + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "headRefName": worktree.branch_name.clone(), + "headRefOid": head_oid, + "headRepository": { "name": "decodex" }, + "headRepositoryOwner": { "login": "hack-ink" }, + "reactionGroups": [], + "comments": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "reviews": { "nodes": [] }, + "reviewRequests": { "totalCount": 0 }, + "reviewThreads": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "commits": { + "nodes": [ + { "commit": { "statusCheckRollup": { "state": "SUCCESS" } } } + ] + } + } + } + } + }) + .to_string(); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"graphql\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"--method\" ] && [ \"$3\" = \"DELETE\" ]; then\n\ + if [ {delete_exit_code} -eq 0 ]; then\n\ + exit 0\n\ + fi\n\ + echo 'delete denied by fake gh' >&2\n\ + exit {delete_exit_code}\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + fake_gh_response + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())) +} + +fn install_fake_closeout_gh_responses( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, +) -> TestEnvVarGuard { + install_fake_closeout_gh_responses_with_state(temp_dir, worktree, pr_url, head_oid, "MERGED") +} + +fn install_fake_closeout_gh_responses_with_state( + temp_dir: &TempDir, + worktree: &WorktreeSpec, + pr_url: &str, + head_oid: &str, + pr_state: &str, +) -> TestEnvVarGuard { + let fake_gh_dir = temp_dir.path().join("fake-closeout-bin"); + let fake_gh_path = fake_gh_dir.join("gh"); + let fake_pr_view_response = serde_json::json!({ + "url": pr_url, + "state": pr_state, + "isDraft": false, + "baseRefName": "main", + "headRefName": worktree.branch_name.clone(), + "headRefOid": head_oid, + "headRepository": { "name": "decodex" }, + "headRepositoryOwner": { "login": "hack-ink" } + }) + .to_string(); + let fake_graphql_response = serde_json::json!({ + "data": { + "repository": { + "mergeCommitAllowed": true, + "pullRequest": { + "url": pr_url, + "state": pr_state, + "isDraft": false, + "reviewDecision": "APPROVED", + "baseRefName": "main", + "mergeable": "MERGEABLE", + "mergeStateStatus": "CLEAN", + "headRefName": worktree.branch_name.clone(), + "headRefOid": head_oid, + "headRepository": { "name": "decodex" }, + "headRepositoryOwner": { "login": "hack-ink" }, + "reactionGroups": [], + "comments": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "reviews": { "nodes": [] }, + "reviewRequests": { "totalCount": 0 }, + "reviewThreads": { + "nodes": [], + "pageInfo": { "hasNextPage": false, "endCursor": null } + }, + "commits": { + "nodes": [ + { "commit": { "statusCheckRollup": { "state": "SUCCESS" } } } + ] + } + } + } + } + }) + .to_string(); + + fs::create_dir_all(&fake_gh_dir).expect("fake gh directory should exist"); + fs::write( + &fake_gh_path, + format!( + "#!/bin/sh\n\ +if [ \"$1\" = \"--version\" ]; then\n\ + printf '%s\\n' 'gh version 2.0.0'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"pr\" ] && [ \"$2\" = \"view\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"graphql\" ]; then\n\ + printf '%s' '{}'\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" = \"api\" ] && [ \"$2\" = \"--method\" ] && [ \"$3\" = \"DELETE\" ]; then\n\ + exit 0\n\ +fi\n\ +echo \"unexpected gh invocation: $*\" >&2\n\ +exit 1\n", + fake_pr_view_response, fake_graphql_response + ), + ) + .expect("fake gh script should write"); + + let mut permissions = + fs::metadata(&fake_gh_path).expect("fake gh metadata should read").permissions(); + + #[cfg(unix)] + PermissionsExt::set_mode(&mut permissions, 0o755); + fs::set_permissions(&fake_gh_path, permissions) + .expect("fake gh script should become executable"); + + let path_env = env::var("PATH").unwrap_or_default(); + + TestEnvVarGuard::set("PATH", &format!("{}:{path_env}", fake_gh_dir.display())) +} + +fn initialize_closeout_cleanup_origin(repo_root: &Path, remote_root: &Path) { + git_status_success( + remote_root.parent().expect("remote root should have parent"), + &[ + "init", + "--bare", + "--initial-branch", + "main", + remote_root.to_str().expect("remote path should be utf-8"), + ], + ); + git_status_success( + repo_root, + &["remote", "add", "origin", remote_root.to_string_lossy().as_ref()], + ); + git_status_success(repo_root, &["push", "-u", "origin", "main"]); +} + +fn route_origin_github_url_to_local_bare_repo(repo_root: &Path, remote_root: &Path) { + let github_remote = "https://github.com/hack-ink/decodex.git"; + let local_remote = format!("file://{}", remote_root.display()); + + git_status_success( + repo_root, + &["config", &format!("url.{local_remote}.insteadOf"), github_remote], + ); + git_status_success(repo_root, &["remote", "set-url", "origin", github_remote]); +} + +fn issue_with_completed_state(mut issue: TrackerIssue) -> TrackerIssue { + if !issue.team.states.iter().any(|state| state.name == "Done") { + issue + .team + .states + .push(TrackerState { id: String::from("state-done"), name: String::from("Done") }); + } + + issue +} + +fn sample_closeout_issue_run( + issue: &TrackerIssue, + worktree: &WorktreeSpec, + run_id: &str, +) -> orchestrator::IssueRunPlan { + orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: String::from("In Review"), + worktree: WorktreeSpec { + branch_name: worktree.branch_name.clone(), + issue_identifier: issue.identifier.clone(), + path: worktree.path.clone(), + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Closeout, + attempt_number: 1, + run_id: String::from(run_id), + retry_budget_base: 0, + } +} + +fn closeout_identity_fixture() -> CloseoutIdentityFixture { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = issue_with_completed_state(sample_issue("In Review", &[active_label.as_str()])); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]; 8]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained closeout worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = String::from("https://github.com/hack-ink/decodex/pull/703"); + let _path_guard = install_fake_closeout_gh_responses(&temp_dir, &worktree, &pr_url, &head_oid); + let remote_root = + config.repo_root().parent().expect("repo root should have a parent").join("origin.git"); + let completed_run_id = String::from("pub-703-attempt-1-111"); + + initialize_closeout_cleanup_origin(config.repo_root(), &remote_root); + route_origin_github_url_to_local_bare_repo(config.repo_root(), &remote_root); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["push", "origin", &format!("HEAD:{}", worktree.branch_name)]) + .status() + .expect("git push lane branch should run") + .success() + ); + + state_store + .upsert_review_handoff_marker( + config.service_id(), + &issue.id, + &ReviewHandoffMarker::new( + &completed_run_id, + 1, + &worktree.branch_name, + &pr_url, + "main", + &worktree.branch_name, + &head_oid, + ), + ) + .expect("review handoff marker should persist"); + state_store + .record_run_attempt(&completed_run_id, &issue.id, 1, "succeeded") + .expect("completed handoff attempt should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("retained closeout worktree should record"); + + CloseoutIdentityFixture { + _temp_dir: temp_dir, + _path_guard, + config, + workflow, + tracker, + state_store, + issue, + worktree, + pr_url, + head_oid, + completed_run_id, + } +} + +fn assert_closeout_lane_ready(fixture: &CloseoutIdentityFixture) { + let mut merged_review_state = sample_pull_request_review_state( + &fixture.pr_url, + &fixture.worktree.branch_name, + &fixture.head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + merged_review_state.state = String::from("MERGED"); + + let lanes = orchestrator::build_post_review_lane_statuses( + &fixture.tracker, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(merged_review_state)]), + ) + .expect("post-review lane status should build"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "continue"); + assert_eq!(lanes[0].reason, "pull_request_merged_closeout_pending"); + assert!( + orchestrator::issue_passes_closeout_dispatch_policy( + &fixture.tracker, + &fixture.issue, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + ) + .expect("closeout policy should evaluate"), + "closeout dispatch policy should accept the merged retained lane: {:?}", + orchestrator::closeout_dispatch_block_reason( + &fixture.tracker, + &fixture.issue, + &fixture.config, + &fixture.workflow, + &fixture.state_store, + ) + .expect("closeout block reason should evaluate") + ); +} + +fn assert_app_server_failure_requires_attention( + error: Report, + error_class: &str, + next_action_fragment: &str, +) { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + + fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("app-server failure handling should succeed"); + + assert_eq!( + tracker.state_updates.borrow().last(), + Some(&(issue.id.clone(), String::from("state-todo"))) + ); + assert!(tracker.comments.borrow().iter().any(|comment| { + comment.contains(error_class) + && comment.contains(next_action_fragment) + && comment.contains("clear label `decodex:needs-attention`") + })); + assert!( + !tracker + .comments + .borrow() + .iter() + .any(|comment| { comment.contains("retryable_execution_failure") }) + ); +} diff --git a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs new file mode 100644 index 00000000..5c61147d --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs @@ -0,0 +1,1501 @@ +const PUB_704_RETAINED_HEAD_SUBJECT: &str = + r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-704"}"#; +const PUB_704_RETAINED_LANDED_SUBJECT: &str = r#"{"schema":"decodex/commit/1","summary":"Land current retained handoff","authority":"PUB-704"}"#; + +fn sample_approved_clean_review_state( + pr_url: &str, + branch_name: &str, + head_oid: &str, +) -> PullRequestReviewState { + sample_pull_request_review_state( + pr_url, + branch_name, + head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ) +} + +fn sample_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn sample_service_owned_issue_without_needs_attention_team_label(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue_without_needs_attention_team_label(state_name, &[active_label.as_str()]) +} + +fn sample_service_owned_issue_with_project_slug_and_sort_fields( + id: &str, + identifier: &str, + project_slug: &str, + state_name: &str, + sort_value: Option, + updated_at: &str, +) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue_with_project_slug_and_sort_fields( + id, + identifier, + project_slug, + state_name, + &[active_label.as_str()], + sort_value, + updated_at, + ) +} + +#[test] +fn retry_delay_distinguishes_continuation_and_capped_failure_backoff() { + let (_, _, workflow) = temp_project_layout(); + + assert_eq!( + orchestrator::retry_delay(orchestrator::RetryKind::Continuation, 1, &workflow,), + Duration::from_millis(1_000) + ); + assert_eq!( + orchestrator::retry_delay(orchestrator::RetryKind::Failure, 1, &workflow), + Duration::from_millis(10_000) + ); + assert_eq!( + orchestrator::retry_delay(orchestrator::RetryKind::Failure, 10, &workflow), + Duration::from_millis(300_000) + ); +} + +#[test] +fn retry_run_dry_run_enforces_active_ownership() { + for (case_name, issue, expected_dispatch) in [ + ("active issue", sample_service_owned_issue("In Progress"), true), + ("unowned issue", sample_issue("In Progress", &[]), false), + ] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Retry, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("retry run should succeed"); + + assert_eq!(summary.is_some(), expected_dispatch, "{case_name}"); + } +} + +#[test] +fn targeted_run_dry_run_accepts_startable_issue_with_normal_dispatch() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Normal, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("targeted run should succeed"); + + assert!(summary.is_some(), "normal targeted dispatch should accept startable issues"); +} + +#[test] +fn retry_run_dry_run_rejects_terminal_guarded_issue_without_attention_label() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue_without_needs_attention_team_label("In Progress"); + let tracker = FakeTracker::with_refresh_snapshots( + vec![issue.clone()], + vec![vec![issue.clone()], vec![issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, TERMINAL_GUARDED_RUN_STATUS) + .expect("terminal guard attempt should record"); + + let summary = orchestrator::run_target_issue_once(TargetIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_id: &issue.id, + preferred_issue_state: None, + preferred_initial_issue_state: None, + dry_run: true, + lease_preacquired: false, + preferred_issue_claim_fd: None, + preferred_dispatch_slot_fd: None, + preferred_dispatch_slot_index: None, + dispatch_mode: IssueDispatchMode::Retry, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }) + .expect("retry run should succeed"); + + assert!( + summary.is_none(), + "retry should reject issues that remain in progress only as a terminal guard" + ); +} + +#[test] +fn schedule_retry_after_child_exit_records_failure_retries_for_active_dispatch_modes() { + for (issue_state, dispatch_mode, expected_dispatch_mode, run_id) in [ + ("In Progress", IssueDispatchMode::Retry, IssueDispatchMode::Retry, "run-1"), + ( + "In Review", + IssueDispatchMode::ReviewRepair, + IssueDispatchMode::ReviewRepair, + "run-review-repair", + ), + ] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue(issue_state); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + dispatch_mode, + exit_status, + ) + .expect("failure retry should schedule"); + + let entry = + retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.dispatch_mode, expected_dispatch_mode); + assert_eq!(entry.kind, orchestrator::RetryKind::Failure); + assert_eq!(entry.attempt, 1); + } +} + +#[test] +fn failure_retry_budget_ignores_prior_continuation_attempts() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-4"; + + state_store + .record_run_attempt("run-1", &issue.id, 1, "succeeded") + .expect("first continuation attempt should record"); + state_store + .record_run_attempt("run-2", &issue.id, 2, "succeeded") + .expect("second continuation attempt should record"); + state_store + .record_run_attempt("run-3", &issue.id, 3, "succeeded") + .expect("third continuation attempt should record"); + state_store + .record_run_attempt(run_id, &issue.id, 4, "failed") + .expect("first failure attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 4 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("first failure after continuations should still schedule"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.kind, orchestrator::RetryKind::Failure); + assert_eq!(entry.attempt, 1); + assert_eq!( + orchestrator::retry_delay(entry.kind, entry.attempt, &workflow), + Duration::from_millis(10_000) + ); +} + +#[test] +fn schedule_retry_after_child_exit_terminalizes_exhausted_review_repair_issue() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + for attempt in 1..=3 { + state_store + .record_run_attempt( + &format!("run-review-repair-{attempt}"), + &issue.id, + attempt, + "failed", + ) + .expect("failed repair attempt should record"); + } + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { + issue_id: &issue.id, + run_id: "run-review-repair-3", + attempt_number: 3, + }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::ReviewRepair, + exit_status, + ) + .expect("exhausted review-repair child exit should terminalize"); + + assert!(retry_queue.entries.is_empty(), "exhausted repair should not stay queued"); + assert_eq!( + tracker.state_updates.borrow().as_slice(), + &[(issue.id.clone(), String::from("state-todo"))] + ); + assert_eq!( + tracker.label_additions.borrow().as_slice(), + &[(issue.id.clone(), vec![String::from("label-needs-attention")])] + ); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + &[(issue.id.clone(), vec![String::from("label-active")])] + ); + assert!( + tracker + .comments + .borrow() + .iter() + .any(|comment| comment.contains("decodex run failed and needs attention")), + "terminal failure comment should explain the exhausted repair" + ); +} + +#[test] +fn schedule_retry_after_child_exit_counts_persisted_retry_budget_after_restart() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_retry_budget_attempt_count(&worktree.path, "previous-run", 2, 2) + .expect("persisted retry budget marker should write"); + + state_store + .record_run_attempt("run-review-repair-3", &issue.id, 3, "failed") + .expect("current failed repair attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { + issue_id: &issue.id, + run_id: "run-review-repair-3", + attempt_number: 3, + }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::ReviewRepair, + exit_status, + ) + .expect("persisted retry budget should contribute to child-exit terminalization"); + + assert!(retry_queue.entries.is_empty(), "exhausted repair should not stay queued"); + assert_eq!( + tracker.state_updates.borrow().as_slice(), + &[(issue.id.clone(), String::from("state-todo"))] + ); + assert_eq!( + tracker.label_additions.borrow().as_slice(), + &[(issue.id.clone(), vec![String::from("label-needs-attention")])] + ); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + &[(issue.id.clone(), vec![String::from("label-active")])] + ); +} + +#[test] +fn schedule_retry_after_child_exit_records_failure_retry_for_closeout_issue_after_tracker_completion() + { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_service_owned_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-closeout"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/175"; + let _path_guard = install_fake_merged_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let inspector = FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]); + + assert!( + orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &issue, + &config, + &workflow, + &state_store, + &inspector, + ) + .expect("completed retained lane should pass closeout retention"), + "completed closeout retries should only schedule when the retained PR lineage is already merged", + ); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + "In Review", + IssueDispatchMode::Closeout, + exit_status, + ) + .expect("closeout failure retry should schedule after tracker completion"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); + assert_eq!(entry.kind, orchestrator::RetryKind::Failure); + assert_eq!(entry.attempt, 1); +} + +#[test] +fn schedule_retry_after_child_exit_keeps_blocked_closeout_retry_for_completed_issue_with_open_pr() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&base_config, "HOME"); + let issue = sample_service_owned_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-closeout-open-pr"; + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/176"; + let _path_guard = install_fake_open_pr_gh_response(&temp_dir, &worktree, pr_url, &head_oid); + + seed_review_handoff_marker( + &state_store, + config.service_id(), + &issue.id, + &worktree.branch_name, + pr_url, + &head_oid, + ); + + let open_review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let inspector = FakePullRequestReviewStateInspector::new(vec![ + Ok(open_review_state.clone()), + Ok(open_review_state), + ]); + + assert!( + !orchestrator::issue_passes_closeout_dispatch_policy_with_inspector( + &tracker, + &issue, + &config, + &workflow, + &state_store, + &inspector, + ) + .expect("open retained lane should not pass closeout dispatch"), + "completed issues with an open PR must stay non-dispatchable", + ); + assert_eq!( + orchestrator::closeout_dispatch_block_reason_with_inspector( + &tracker, + &issue, + &config, + &workflow, + &state_store, + &inspector, + ) + .expect("block reason lookup should succeed"), + Some("pull_request_not_merged") + ); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + "In Review", + IssueDispatchMode::Closeout, + exit_status, + ) + .expect("blocked closeout retry should stay queued after child exit"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.dispatch_mode, orchestrator::IssueDispatchMode::Closeout); + assert_eq!(entry.kind, orchestrator::RetryKind::Failure); + assert_eq!(entry.attempt, 1); +} + +#[test] +fn queued_review_repair_retry_handles_backoff_budget_and_ownership() { + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::ReviewRepair, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("future review-repair retry should stay queued"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "review-repair retries should keep their queued backoff window until ready" + ); + } + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + for attempt in 1..=3 { + state_store + .record_run_attempt(&format!("run-{attempt}"), &issue.id, attempt, "failed") + .expect("failed repair attempt should record"); + } + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::ReviewRepair, + kind: RetryKind::Failure, + attempt: 3, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("exhausted review-repair retry should be dropped"); + + assert!(matches!(decision, RetryDispatchDecision::Continue)); + assert!( + retry_queue.entries.is_empty(), + "exhausted review-repair retry should not hold the queued claim" + ); + } + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::ReviewRepair, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("review-repair retry planning should succeed"); + + assert!(matches!(decision, RetryDispatchDecision::Continue)); + assert!( + !retry_queue.entries.contains_key(&issue.id), + "review-repair retries should be dropped when active ownership is gone" + ); + } +} + +#[test] +fn interrupted_exits_consume_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-3"; + + state_store + .record_run_attempt("run-1", &issue.id, 1, "interrupted") + .expect("first interrupted attempt should record"); + state_store + .record_run_attempt("run-2", &issue.id, 2, "interrupted") + .expect("second interrupted attempt should record"); + state_store + .record_run_attempt(run_id, &issue.id, 3, "interrupted") + .expect("third interrupted attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 3 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("retry scheduling should succeed"); + + assert!( + !retry_queue.entries.contains_key(&issue.id), + "interrupted exits should exhaust the retry budget" + ); +} + +#[test] +fn schedule_retry_after_child_exit_records_continuation_retry_for_clean_exit() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, CONTINUATION_PENDING_RUN_STATUS) + .expect("run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 0"]).status().expect("success exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("continuation retry should schedule"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.kind, orchestrator::RetryKind::Continuation); + assert_eq!(entry.attempt, 1); +} + +#[test] +fn schedule_retry_after_child_exit_preserves_specific_retry_schedule_kind_for_failure_retry() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_retry_schedule( + &worktree_path, + run_id, + 1, + "git_lock_contention", + OffsetDateTime::now_utc().unix_timestamp() + 30, + ) + .expect("specific retry schedule should write"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("failure retry should schedule"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("retry schedule should remain readable") + .expect("retry marker should exist"); + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.kind, orchestrator::RetryKind::Failure); + assert_eq!(marker.retry_kind(), Some("git_lock_contention")); +} + +#[test] +fn schedule_retry_after_child_exit_retains_continuation_retry_for_stale_startable_issue() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("Todo"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, CONTINUATION_PENDING_RUN_STATUS) + .expect("run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 0"]).status().expect("success exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("continuation retry should tolerate a stale startable tracker reread"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + + assert_eq!(entry.kind, orchestrator::RetryKind::Continuation); + assert_eq!(entry.attempt, 1); +} + +#[test] +fn schedule_retry_after_child_exit_skips_retry_for_completed_successful_run() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("Todo", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-1"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "succeeded") + .expect("completed run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 0"]).status().expect("success exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("completed successful runs should not schedule another retry"); + + assert!( + !retry_queue.entries.contains_key(&issue.id), + "successful review-handoff style exits must not reopen the same run as a continuation" + ); +} + +#[test] +fn schedule_retry_after_child_exit_requires_exact_run_id() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("other-run", &issue.id, 1, "running") + .expect("other run attempt should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id: "planned-run", attempt_number: 1 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("retry scheduling should succeed"); + + assert!( + !retry_queue.entries.contains_key(&issue.id), + "retry scheduling should ignore a different run that only matches the issue and attempt" + ); +} + +#[test] +fn exited_retry_child_keeps_queued_claim_when_no_run_attempt_was_persisted() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let mut child = + Command::new("sh").args(["-c", "exit 1"]).spawn().expect("child process should spawn"); + + for _ in 0..20 { + if child.try_wait().expect("child status should query").is_some() { + break; + } + + thread::sleep(Duration::from_millis(10)); + } + + let mut active_children = vec![orchestrator::DaemonRunChild { + child, + issue_id: issue.id.clone(), + run_id: String::from("planned-run"), + attempt_number: 1, + initial_issue_state: issue.state.name.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: orchestrator::IssueDispatchMode::Retry, + from_retry_queue: true, + workflow: workflow.clone(), + }]; + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now(), + }); + + orchestrator::inspect_or_clear_active_children( + &mut active_children, + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + &worktree_manager, + ) + .expect("exited child cleanup should succeed"); + + assert!(active_children.is_empty(), "exited child should be cleared"); + assert!( + retry_queue.entries.contains_key(&issue.id), + "retry claim should remain queued when the child exits before persisting a run attempt" + ); +} + +#[test] +fn exited_successful_child_marks_recent_run_succeeded_before_cleanup() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let run_id = "planned-run"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + + let mut child = + Command::new("sh").args(["-c", "exit 0"]).spawn().expect("child process should spawn"); + + for _ in 0..20 { + if child.try_wait().expect("child status should query").is_some() { + break; + } + + thread::sleep(Duration::from_millis(10)); + } + + let mut active_children = vec![orchestrator::DaemonRunChild { + child, + issue_id: issue.id.clone(), + run_id: String::from(run_id), + attempt_number: 1, + initial_issue_state: issue.state.name.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: orchestrator::IssueDispatchMode::Retry, + from_retry_queue: false, + workflow: workflow.clone(), + }]; + let mut retry_queue = RetryQueue::default(); + + orchestrator::inspect_or_clear_active_children( + &mut active_children, + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + &worktree_manager, + ) + .expect("exited child cleanup should succeed"); + + assert!(active_children.is_empty(), "exited child should be cleared"); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should remain recorded") + .status(), + "succeeded" + ); +} + +#[test] +fn exited_unsuccessful_child_does_not_downgrade_persisted_success() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let run_id = "planned-run"; + + state_store + .record_run_attempt(run_id, &issue.id, 1, "succeeded") + .expect("run attempt should record completed child outcome"); + + let mut child = + Command::new("sh").args(["-c", "exit 1"]).spawn().expect("child process should spawn"); + + for _ in 0..20 { + if child.try_wait().expect("child status should query").is_some() { + break; + } + + thread::sleep(Duration::from_millis(10)); + } + + let mut active_children = vec![orchestrator::DaemonRunChild { + child, + issue_id: issue.id.clone(), + run_id: String::from(run_id), + attempt_number: 1, + initial_issue_state: issue.state.name.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: orchestrator::IssueDispatchMode::Retry, + from_retry_queue: false, + workflow: workflow.clone(), + }]; + let mut retry_queue = RetryQueue::default(); + + orchestrator::inspect_or_clear_active_children( + &mut active_children, + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + &worktree_manager, + ) + .expect("exited child cleanup should succeed"); + + assert!(active_children.is_empty(), "exited child should be cleared"); + assert!( + retry_queue.entries.is_empty(), + "persisted success should not schedule a retry after a late wrapper failure" + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should remain recorded") + .status(), + "succeeded" + ); +} + +fn assert_fake_admin_merge_invocation( + invocation_log_path: &Path, + head_oid: &str, + merge_subject: &str, + pr_url: &str, +) { + let gh_invocation = fs::read_to_string(invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + String::from(head_oid), + String::from("--subject"), + String::from(merge_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + ] + ); +} + +fn assert_fake_admin_merge_invocation_present( + invocation_log_path: &Path, + head_oid: &str, + merge_subject: &str, + pr_url: &str, +) { + let gh_invocation_log = + fs::read_to_string(invocation_log_path).expect("fake gh invocation log should read"); + let expected_invocation = [ + "pr", + "merge", + "--admin", + "--merge", + "--match-head-commit", + head_oid, + "--subject", + merge_subject, + "--body", + "", + pr_url, + ] + .join("\n"); + + assert!( + gh_invocation_log.contains(&expected_invocation), + "fake gh invocation log should contain the admin merge command" + ); +} + +fn stop_daemon_children(active_children: &mut [DaemonRunChild]) { + for daemon_child in active_children { + let _ = daemon_child.child.kill(); + let _ = daemon_child.child.wait(); + } +} + +fn spawn_sleeping_daemon_child( + active_issue: &TrackerIssue, + workflow: &WorkflowDocument, +) -> DaemonRunChild { + let child = + Command::new("sh").args(["-c", "sleep 30"]).spawn().expect("child process should spawn"); + + DaemonRunChild { + child, + issue_id: active_issue.id.clone(), + run_id: String::from("active-run"), + attempt_number: 1, + initial_issue_state: active_issue.state.name.clone(), + retry_project_slug: active_issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + from_retry_queue: false, + workflow: workflow.clone(), + } +} + +#[test] +fn daemon_tick_reconciles_ready_retained_review_lane_before_dry_run_planning() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&base_config, "PATH"), + false, + ); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let active_issue = sample_service_owned_issue_with_project_slug_and_sort_fields( + "issue-active", + "PUB-200", + TEST_SERVICE_ID, + "In Progress", + Some(1), + "2026-03-13T04:16:17.133Z", + ); + let retained_issue = sample_service_owned_issue_with_project_slug_and_sort_fields( + "issue-retained", + "PUB-704", + TEST_SERVICE_ID, + "In Review", + Some(2), + "2026-03-13T04:18:17.133Z", + ); + let tracker = FakeTracker::with_refresh_snapshots( + vec![active_issue.clone(), retained_issue.clone()], + vec![vec![active_issue.clone()], vec![retained_issue.clone()]], + ); + let retained_worktree = worktree_manager + .ensure_worktree(&retained_issue.identifier, false) + .expect("retained worktree should exist"); + let pr_url = "https://github.com/hack-ink/decodex/pull/704"; + let head_oid = commit_worktree_change( + &retained_worktree.path, + "retained.txt", + "ready\n", + PUB_704_RETAINED_HEAD_SUBJECT, + ); + + state_store + .upsert_worktree( + config.service_id(), + &retained_issue.id, + &retained_worktree.branch_name, + &retained_worktree.path.display().to_string(), + ) + .expect("retained worktree should record"); + state_store + .record_run_attempt("active-run", &active_issue.id, 1, "running") + .expect("active run should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &retained_issue.id, + &sample_review_handoff_marker(&retained_worktree.branch_name, pr_url, &head_oid), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + &retained_worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let mut active_children = vec![spawn_sleeping_daemon_child(&active_issue, &workflow)]; + let mut retry_queue = RetryQueue::default(); + let result = orchestrator::run_daemon_tick_with_review_state_inspector( + &service_config_path(config.repo_root()), + &state_store, + &mut active_children, + &mut retry_queue, + DaemonTickRuntimeContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + worktree_manager: &worktree_manager, + review_state_inspector: &FakePullRequestReviewStateInspector::new(vec![Ok( + review_state, + )]), + }, + ); + + stop_daemon_children(&mut active_children); + + result.expect("daemon tick should reconcile retained review lanes"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &retained_worktree.path, + ); + + assert_eq!(marker.phase(), "waiting_for_merge"); + + assert_fake_admin_merge_invocation( + &invocation_log_path, + &head_oid, + PUB_704_RETAINED_LANDED_SUBJECT, + pr_url, + ); +} + +#[test] +fn daemon_tick_clears_terminal_mapping_without_worktree_before_retained_land() { + let (temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&base_config, "PATH"), + false, + ); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let mut terminal_issue = sample_service_owned_issue_with_project_slug_and_sort_fields( + "issue-terminal", + "PUB-703", + TEST_SERVICE_ID, + "Done", + Some(1), + "2026-03-13T04:16:17.133Z", + ); + + terminal_issue.labels.clear(); + terminal_issue.team.labels.clear(); + + let retained_issue = sample_service_owned_issue_with_project_slug_and_sort_fields( + "issue-retained", + "PUB-704", + TEST_SERVICE_ID, + "In Review", + Some(2), + "2026-03-13T04:18:17.133Z", + ); + let tracker = FakeTracker::with_refresh_snapshots( + vec![terminal_issue.clone(), retained_issue.clone()], + vec![ + vec![terminal_issue.clone(), retained_issue.clone()], + vec![terminal_issue.clone(), retained_issue.clone()], + vec![retained_issue.clone()], + ], + ); + let retained_worktree = worktree_manager + .ensure_worktree(&retained_issue.identifier, false) + .expect("retained worktree should exist"); + let pr_url = "https://github.com/hack-ink/decodex/pull/704"; + let head_oid = commit_worktree_change( + &retained_worktree.path, + "retained.txt", + "ready\n", + PUB_704_RETAINED_HEAD_SUBJECT, + ); + + state_store + .upsert_worktree( + config.service_id(), + &terminal_issue.id, + "x/pubfi-pub-703", + &config.worktree_root().join("PUB-703").display().to_string(), + ) + .expect("terminal stale worktree should record"); + state_store + .upsert_worktree( + config.service_id(), + &retained_issue.id, + &retained_worktree.branch_name, + &retained_worktree.path.display().to_string(), + ) + .expect("retained worktree should record"); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &retained_issue.id, + &sample_review_handoff_marker(&retained_worktree.branch_name, pr_url, &head_oid), + ); + + let mut active_children = Vec::new(); + let mut retry_queue = RetryQueue::default(); + + orchestrator::run_daemon_tick_with_review_state_inspector( + &service_config_path(config.repo_root()), + &state_store, + &mut active_children, + &mut retry_queue, + DaemonTickRuntimeContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + worktree_manager: &worktree_manager, + review_state_inspector: &FakePullRequestReviewStateInspector::new(vec![Ok( + sample_approved_clean_review_state( + pr_url, + &retained_worktree.branch_name, + &head_oid, + ), + )]), + }, + ) + .expect("daemon tick should not fail on stale terminal worktree state"); + + assert!( + state_store + .worktree_for_issue(&terminal_issue.id) + .expect("terminal worktree lookup should succeed") + .is_none(), + "terminal mapping without a local worktree should be cleared" + ); + + assert_fake_admin_merge_invocation_present( + &invocation_log_path, + &head_oid, + PUB_704_RETAINED_LANDED_SUBJECT, + pr_url, + ); +} diff --git a/apps/decodex/src/orchestrator/tests/retry/selection.rs b/apps/decodex/src/orchestrator/tests/retry/selection.rs new file mode 100644 index 00000000..62ca14d6 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/retry/selection.rs @@ -0,0 +1,709 @@ +fn selection_sample_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn selection_sample_service_owned_issue_with_sort_fields( + id: &str, + identifier: &str, + state_name: &str, + sort_value: Option, + updated_at: &str, +) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue_with_sort_fields( + id, + identifier, + state_name, + &[active_label.as_str()], + sort_value, + updated_at, + ) +} + +fn selection_sample_service_owned_issue_with_project_slug_and_sort_fields( + id: &str, + identifier: &str, + project_slug: &str, + state_name: &str, + sort_value: Option, + updated_at: &str, +) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue_with_project_slug_and_sort_fields( + id, + identifier, + project_slug, + state_name, + &[active_label.as_str()], + sort_value, + updated_at, + ) +} + +#[test] +fn queued_retry_blocks_normal_candidate_selection_until_due() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 2, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!(!retry_queue.is_empty(), "future retry should keep the queued claim"); +} + +#[test] +fn queued_retry_stays_blocked_when_project_lookup_blips_before_due_time() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]) + .with_project_lookup_error("transient project lookup failure"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 2, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("future retry should not fail on project lookup blips"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!(!retry_queue.is_empty(), "future retry should keep the queued claim"); +} + +#[test] +fn blocked_future_retry_excludes_all_queued_retries_before_normal_fallback() { + let workflow = WorkflowDocument::parse_markdown( + &sample_workflow_markdown("pubfi", &[], "Multi-slot daemon policy.\n", 1) + .replace("max_concurrent_agents = 1", "max_concurrent_agents = 2"), + ) + .expect("workflow should parse"); + let first_future_retry = selection_sample_service_owned_issue_with_sort_fields( + "issue-1", + "PUB-101", + "In Progress", + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let second_future_retry = selection_sample_service_owned_issue_with_sort_fields( + "issue-2", + "PUB-102", + "In Progress", + Some(2), + "2026-03-13T04:17:17.133Z", + ); + let todo_issue = sample_issue_with_sort_fields( + "issue-3", + "PUB-103", + "Todo", + &[], + Some(1), + "2026-03-13T04:18:17.133Z", + ); + let listed_issues = + vec![first_future_retry.clone(), second_future_retry.clone(), todo_issue.clone()]; + let tracker = FakeTracker::with_refresh_snapshots( + listed_issues.clone(), + vec![listed_issues.clone(), listed_issues.clone(), listed_issues], + ); + let (_temp_dir, config, _default_workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let retained_worktree = worktree_manager.plan_for_issue(&second_future_retry.identifier); + + fs::create_dir_all(&retained_worktree.path) + .expect("retained future retry worktree should exist for recovery"); + + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: first_future_retry.id.clone(), + retry_project_slug: first_future_retry + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 2, + ready_at: Instant::now() + Duration::from_secs(60), + }); + retry_queue.upsert(RetryEntry { + issue_id: second_future_retry.id.clone(), + retry_project_slug: second_future_retry + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 2, + ready_at: Instant::now() + Duration::from_secs(120), + }); + + let next_run = orchestrator::plan_next_daemon_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("daemon planning should succeed") + .expect("normal work should still fill open capacity"); + + assert!(!next_run.1, "alternate work should not dispatch from the retry queue"); + assert_eq!(next_run.0.issue_id, todo_issue.id); + assert_eq!(next_run.0.issue_identifier, todo_issue.identifier); + assert_eq!(next_run.0.issue_state, "In Progress"); + assert_eq!(next_run.0.dispatch_mode, orchestrator::IssueDispatchMode::Normal); + assert!( + retry_queue.entries.contains_key(&first_future_retry.id) + && retry_queue.entries.contains_key(&second_future_retry.id), + "all queued retries should stay queued instead of bypassing their ready_at through retained recovery" + ); +} + +#[test] +fn future_retry_claim_stays_blocked_when_issue_moves_to_another_project_before_due_time() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue_with_project_slug_and_sort_fields( + "issue-1", + "PUB-101", + "other-project", + "In Progress", + Some(3), + "2026-03-13T04:16:17.133Z", + ); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: String::from("pubfi"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 2, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should keep queued claims before their due time"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "future retries should keep their queued claim until due when the issue is still active" + ); +} + +#[test] +fn retry_claim_releases_when_issue_becomes_non_active() { + for (ready_delay, description) in [ + (Duration::from_secs(60), "future"), + (Duration::from_secs(0), "due"), + ] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now() + ready_delay, + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); + assert!( + retry_queue.is_empty(), + "{description} non-active issue should release the queued claim" + ); + } +} + +#[test] +fn future_retry_claim_release_clears_persisted_retry_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-101"); + let mut retry_queue = RetryQueue::default(); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_retry_schedule( + &worktree_path, + "run-1", + 1, + "failure", + OffsetDateTime::now_utc().unix_timestamp() + 60, + ) + .expect("retry schedule should write"); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); + assert!(retry_queue.is_empty(), "non-active issue should release the queued claim early"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("marker should load") + .expect("marker should still exist"); + + assert_eq!(marker.retry_kind(), None); + assert_eq!(marker.retry_ready_at_unix_epoch(), None); +} + +#[test] +fn due_continuation_retry_dispatches_when_issue_still_reflects_startable_state() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("Todo"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: Some(issue.state.name.clone()), + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Continuation, + attempt: 1, + ready_at: Instant::now(), + }); + + let next_run = orchestrator::plan_next_daemon_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("daemon planning should succeed") + .expect("the continuation retry should still dispatch"); + + assert!(next_run.1, "continuation work should still come from the retry queue"); + assert_eq!(next_run.0.issue_id, issue.id); + assert_eq!(next_run.0.issue_identifier, issue.identifier); + assert_eq!(next_run.0.issue_state, "In Progress"); + assert_eq!(next_run.0.dispatch_mode, orchestrator::IssueDispatchMode::Retry); +} + +#[test] +fn due_continuation_retry_releases_when_issue_moves_to_different_startable_state() { + let workflow = WorkflowDocument::parse_markdown( + &sample_workflow_markdown("pubfi", &[], "Continuation retry policy.\n", 1) + .replace("startable_states = [\"Todo\"]", "startable_states = [\"Todo\", \"Backlog\"]"), + ) + .expect("workflow should parse"); + let (_temp_dir, config, _default_workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("Backlog"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: Some(String::from("Todo")), + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Continuation, + attempt: 1, + ready_at: Instant::now(), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); + assert!( + retry_queue.is_empty(), + "continuation retention should reject a different startable state instead of reopening the old thread" + ); +} + +#[test] +fn due_continuation_retry_stays_queued_when_global_concurrency_is_exhausted() { + let workflow = WorkflowDocument::parse_markdown(&sample_workflow_markdown( + "pubfi", + &[], + "Continuation retry policy.\n", + 1, + )) + .expect("workflow should parse"); + let (_temp_dir, config, _default_workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("Todo"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + state_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("dispatch slot root should configure"); + state_store + .upsert_lease(config.service_id(), "issue-other", "run-other", "In Progress") + .expect("competing in-progress lease should record"); + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: Some(issue.state.name.clone()), + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Continuation, + attempt: 1, + ready_at: Instant::now(), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "stale-startable continuation retries should remain queued when the global concurrency cap blocks dispatch" + ); +} + +#[test] +fn future_retry_claim_releases_when_issue_returns_to_todo_before_due_time() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("Todo"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now() + Duration::from_secs(60), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); + assert!(retry_queue.is_empty(), "todo issues should not retain queued retry claims"); +} + +#[test] +fn due_retry_claim_stays_queued_when_dispatch_slot_is_temporarily_unavailable() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut retry_queue = RetryQueue::default(); + + state_store + .upsert_lease("pubfi", "issue-other", "run-other", "In Progress") + .expect("temporary competing lease should record"); + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now(), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "active retry entry should remain queued while another lease temporarily holds the slot" + ); +} + +#[test] +fn due_retry_claim_stays_queued_when_issue_is_claimed_by_another_process() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = selection_sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let local_store = StateStore::open_in_memory().expect("local state store should open"); + let remote_store = StateStore::open_in_memory().expect("remote state store should open"); + let mut retry_queue = RetryQueue::default(); + + local_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("local dispatch slot root should configure"); + remote_store + .configure_dispatch_slot_root( + config.service_id(), + config.worktree_root(), + workflow.frontmatter().execution().max_concurrent_agents(), + ) + .expect("remote dispatch slot root should configure"); + + assert!( + remote_store + .try_acquire_lease(config.service_id(), &issue.id, "run-foreign", "In Progress") + .expect("foreign process should acquire the shared issue claim") + ); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: None, + dispatch_mode: IssueDispatchMode::Retry, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now(), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &local_store, + ) + .expect("retry planning should succeed"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "retry queue should keep the claim until the foreign issue claim clears" + ); +} + +#[test] +fn due_closeout_retry_stays_queued_when_pr_state_read_fails() { + let (_temp_dir, base_config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var( + &base_config, + "DECODEX_TEST_MISSING_DELIVERY_CLOSEOUT_GITHUB_TOKEN", + ); + let issue = selection_sample_service_owned_issue("Done"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/178"; + let mut retry_queue = RetryQueue::default(); + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + retry_queue.upsert(RetryEntry { + issue_id: issue.id.clone(), + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + continuation_initial_issue_state: Some(String::from("In Review")), + dispatch_mode: IssueDispatchMode::Closeout, + kind: RetryKind::Failure, + attempt: 1, + ready_at: Instant::now(), + }); + + let decision = orchestrator::plan_due_retry_run( + &mut retry_queue, + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("retry planning should degrade GH read failures to a blocked queued retry"); + + assert!(matches!( + decision, + RetryDispatchDecision::Blocked{ excluded_issue_ids } + if excluded_issue_ids == vec![issue.id.clone()] + )); + assert!( + retry_queue.entries.contains_key(&issue.id), + "completed closeout retries must remain queued when GH state inspection is temporarily unavailable" + ); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/classification_checks.rs b/apps/decodex/src/orchestrator/tests/review_landing/classification_checks.rs new file mode 100644 index 00000000..eb5ac9b3 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/classification_checks.rs @@ -0,0 +1,687 @@ +#[test] +fn classify_post_review_lane_blocks_completed_issue_until_pull_request_is_merged() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Done", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "issue_completed_before_pull_request_merged"); +} + +#[test] +fn classify_post_review_lane_waits_for_pending_required_checks_before_ready_to_land() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "pass_waiting_for_gates", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "UNSTABLE", + Some("PENDING"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::WaitForReview); + assert_eq!(classification.reason, "external_review_passed_waiting_gates"); +} + +#[test] +fn classify_post_review_lane_routes_non_clean_landing_to_agent_fallback() { + for (merge_state, status_check_state) in [("HAS_HOOKS", Some("SUCCESS")), ("UNSTABLE", Some("FAILURE"))] { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "waiting_for_result", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + merge_state, + status_check_state, + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::NeedsReviewRepair); + assert_eq!(classification.reason, "retained_landing_agent_fallback_required"); + } +} + +#[test] +fn classify_post_review_lane_waits_for_review_before_optional_failed_checks() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "waiting_for_result", + 0, + ), + ); + + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("REVIEW_REQUIRED"), + "MERGEABLE", + "UNSTABLE", + Some("FAILURE"), + 0, + ); + + add_external_review_ack(&mut review_state); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::WaitForReview); + assert_eq!(classification.reason, "external_review_result_pending"); +} + +#[test] +fn classify_post_review_lane_requires_review_repair_before_review_when_required_checks_fail() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("REVIEW_REQUIRED"), + "MERGEABLE", + "BLOCKED", + Some("FAILURE"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::NeedsReviewRepair); + assert_eq!(classification.reason, "required_checks_failed"); +} + +#[test] +fn classify_post_review_lane_blocks_checkout_branch_mismatch() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-999")), + local_head_oid: Some(head_oid), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("classification should degrade to blocked"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "worktree_checkout_branch_mismatch"); +} + +#[test] +fn classify_post_review_lane_blocks_pull_request_state_read_failures() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Err(color_eyre::eyre::eyre!( + "gh api failed" + ))]), + ) + .expect("classification should degrade to blocked"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "pull_request_state_read_failed"); +} + +#[test] +fn classify_post_review_lane_blocks_missing_or_blank_github_token_env_var() { + let missing = classify_post_review_lane_with_github_token_env_var(None); + + assert_eq!(missing.decision, PostReviewLaneDecision::Block); + assert_eq!(missing.reason, "pull_request_state_read_failed"); + + let env_var = format!("DECODEX_TEST_BLANK_STATUS_GITHUB_TOKEN_ENV_{}", process::id()); + let _env_guard = TestEnvVarGuard::set(&env_var, ""); + let blank = classify_post_review_lane_with_github_token_env_var(Some(env_var)); + + assert_eq!(blank.decision, PostReviewLaneDecision::Block); + assert_eq!(blank.reason, "pull_request_state_read_failed"); +} + +fn classify_post_review_lane_with_github_token_env_var( + github_token_env_var: Option, +) -> PostReviewLaneClassification { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid), + }; + + orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &GhPullRequestReviewStateInspector { github_token_env_var }, + ) + .expect("classification should degrade to blocked") +} + +#[test] +fn merge_pull_request_review_state_page_counts_unresolved_threads_across_pages() { + let first_page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 100, + true, + Some("cursor-1"), + ); + let repository = sample_pull_request_review_state_repository(first_page); + let mut review_state = orchestrator::pull_request_review_state_from_page( + &repository, + repository.pull_request.as_ref().expect("pull request should exist"), + ); + let next_page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 1, + false, + None, + ); + let next_repository = sample_pull_request_review_state_repository(next_page); + let next_cursor = orchestrator::merge_pull_request_review_state_page( + &mut review_state, + &next_repository, + next_repository.pull_request.as_ref().expect("pull request should exist"), + ) + .expect("page merge should succeed"); + + assert_eq!(review_state.unresolved_review_threads, 101); + assert_eq!(next_cursor, None); +} + +#[test] +fn merge_pull_request_issue_comment_page_appends_comments_across_pages() { + let first_page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 0, + false, + None, + ); + let repository = sample_pull_request_review_state_repository(first_page); + let mut review_state = orchestrator::pull_request_review_state_from_page( + &repository, + repository.pull_request.as_ref().expect("pull request should exist"), + ); + let next_page = PullRequestIssueCommentsNode { + url: String::from("https://github.com/hack-ink/decodex/pull/174"), + comments: PullRequestIssueCommentConnection { + nodes: vec![PullRequestIssueCommentNode { + database_id: 501, + body: String::from("Looks good"), + created_at: String::from("2025-11-03T00:00:00Z"), + author: Some(PullRequestActor { + login: String::from(crate::orchestrator::EXTERNAL_REVIEW_ACTOR_LOGIN), + }), + reaction_groups: Vec::new(), + }], + page_info: PullRequestPageInfo { has_next_page: false, end_cursor: None }, + }, + }; + let next_cursor = + orchestrator::merge_pull_request_issue_comment_page(&mut review_state, &next_page) + .expect("comment page merge should succeed"); + + assert_eq!(review_state.issue_comments.len(), 1); + assert_eq!(review_state.issue_comments[0].database_id, 501); + assert_eq!(next_cursor, None); +} + +#[test] +fn merge_pull_request_review_state_page_rejects_changed_metadata_across_pages() { + type ReviewPageMutation = fn(&mut PullRequestReviewStateNode); + + let cases: [(&str, ReviewPageMutation); 4] = [ + ("review metadata", |page| { + page.review_decision = Some(String::from("CHANGES_REQUESTED")); + }), + ("pending review request count", |page| { + page.review_requests.total_count = 1; + }), + ("head repository owner", |page| { + page.head_repository_owner = + Some(PullRequestRepositoryOwner { login: String::from("someone-else") }); + }), + ("head repository name", |page| { + page.head_repository = + Some(PullRequestRepository { name: String::from("decodex-fork") }); + }), + ]; + + for (case_name, mutate) in cases { + assert_review_state_page_rejects_changed_metadata(case_name, mutate); + } +} + +fn assert_review_state_page_rejects_changed_metadata( + case_name: &str, + mutate: fn(&mut PullRequestReviewStateNode), +) { + let first_page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 100, + true, + Some("cursor-1"), + ); + let repository = sample_pull_request_review_state_repository(first_page); + let mut review_state = orchestrator::pull_request_review_state_from_page( + &repository, + repository.pull_request.as_ref().expect("pull request should exist"), + ); + let mut next_page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 1, + false, + None, + ); + + mutate(&mut next_page); + + let next_repository = sample_pull_request_review_state_repository(next_page); + let error = orchestrator::merge_pull_request_review_state_page( + &mut review_state, + &next_repository, + next_repository.pull_request.as_ref().expect("pull request should exist"), + ) + .expect_err("changed metadata should fail"); + + assert!(error.to_string().contains("changed while paginating"), "{case_name}"); +} + +#[test] +fn pull_request_review_state_query_requests_required_fields() { + for expected_fragment in [ + "mergeCommitAllowed", + "headRepository {\n name\n }", + "comments(first: 100) {\n nodes {\n databaseId", + "pageInfo {\n hasNextPage\n endCursor\n }", + ] { + assert!( + orchestrator::PULL_REQUEST_REVIEW_STATE_QUERY.contains(expected_fragment), + "query should include {expected_fragment}" + ); + } +} + +#[test] +fn next_pull_request_review_threads_cursor_requires_end_cursor_when_pagination_continues() { + let page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + 100, + true, + None, + ); + let error = orchestrator::next_pull_request_review_threads_cursor(&page) + .expect_err("missing end cursor should fail"); + + assert!(error.to_string().contains("without an end cursor")); +} + +#[test] +fn next_pull_request_issue_comments_cursor_requires_end_cursor_when_pagination_continues() { + let comments = PullRequestIssueCommentConnection { + nodes: Vec::new(), + page_info: PullRequestPageInfo { has_next_page: true, end_cursor: None }, + }; + let error = orchestrator::next_pull_request_issue_comments_cursor( + &comments, + "https://github.com/hack-ink/decodex/pull/174", + ) + .expect_err("missing end cursor should fail"); + + assert!(error.to_string().contains("without an end cursor")); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/classification_review.rs b/apps/decodex/src/orchestrator/tests/review_landing/classification_review.rs new file mode 100644 index 00000000..7c87db9a --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/classification_review.rs @@ -0,0 +1,898 @@ +#[test] +fn classify_post_review_lane_requires_review_repair_for_unresolved_threads() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 2, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::NeedsReviewRepair); + assert_eq!(classification.reason, "unresolved_review_threads"); +} + +#[test] +fn classify_post_review_lane_ignores_stale_review_orchestration_record_from_prior_handoff() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + + state_store + .upsert_review_orchestration_marker( + "pubfi", + &issue.id, + &ReviewOrchestrationMarker::new( + "run-0", + 7, + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/99", + "deadbeef", + "waiting_for_result", + Some(TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID), + Some(TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT), + Some(1), + 2, + 3, + None, + ), + ) + .expect("stale review orchestration marker should persist"); + + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::WaitForReview); + assert_eq!(classification.reason, "external_review_request_pending"); +} + +#[test] +fn classify_post_review_lane_request_pending_waits_for_green_checks_before_external_review_request() +{ + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("PENDING"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::WaitForReview); + assert_eq!(classification.reason, "external_review_request_waiting_for_green_checks",); +} + +#[test] +fn classify_post_review_lane_request_pending_routes_fixable_ci_red_to_repair() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "BLOCKED", + Some("FAILURE"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::NeedsReviewRepair); + assert_eq!(classification.reason, "required_checks_failed"); +} + +#[test] +fn classify_post_review_lane_request_pending_blocks_unhandled_ci_red() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "UNSTABLE", + Some("FAILURE"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "external_review_request_ci_red_manual_attention",); +} + +#[test] +fn classify_post_review_lane_requires_review_repair_for_non_thread_review_summary_findings() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "waiting_for_result", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("COMMENTED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_findings(&mut review_state, "Please cover the failing edge case."); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::NeedsReviewRepair); + assert_eq!(classification.reason, "external_review_feedback_pending_repair"); +} + +#[test] +fn classify_post_review_lane_ready_to_land_allows_zero_required_review_repos() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "waiting_for_result", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + None, + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::ReadyToLand); + assert_eq!(classification.reason, "external_review_passed_strict"); +} + +#[test] +fn classify_post_review_lane_blocks_stale_review_handoff_head_without_lineage_proof() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let marker_head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let current_head_oid = String::from("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &marker_head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(current_head_oid.clone()), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + ¤t_head_oid, + None, + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "review_handoff_lineage_check_failed"); +} + +#[test] +fn classify_post_review_lane_blocks_when_pull_request_head_differs_from_worktree_head() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let branch_name = worktree.branch_name.clone(); + let worktree_path = worktree.path.clone(); + let current_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_head_oid = String::from("feedfacefeedfacefeedfacefeedfacefeedface"); + let marker_head_oid = current_head_oid.clone(); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &branch_name, + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees(config.service_id()) + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + &branch_name, + "https://github.com/hack-ink/decodex/pull/174", + &marker_head_oid, + )), + local_branch_name: Some(branch_name.clone()), + local_head_oid: Some(current_head_oid), + }; + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + &branch_name, + &pr_head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Block); + assert_eq!(classification.reason, "pull_request_head_mismatch"); +} + +#[test] +fn classify_post_review_lane_waits_when_external_review_request_is_still_pending() { + for review_decision in [None, Some("APPROVED")] { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + + seed_review_orchestration_marker( + &state_store, + TEST_SERVICE_ID, + &snapshot.issue.id, + &sample_review_orchestration_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + "waiting_for_ack", + 0, + ), + ); + + let mut review_state = sample_pull_request_review_state_with_pending_requests( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + review_decision, + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + 1, + ); + + add_external_review_ack(&mut review_state); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::WaitForReview); + assert_eq!(classification.reason, "external_review_result_pending"); + } +} + +#[test] +fn classify_post_review_lane_continues_when_pull_request_is_already_merged() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let mut review_state = sample_pull_request_review_state( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let classification = orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed"); + + assert_eq!(classification.decision, PostReviewLaneDecision::Continue); + assert_eq!(classification.reason, "pull_request_merged_closeout_pending"); +} + +#[test] +fn classify_post_review_lane_repairs_unmergeable_pr_before_review_waits() { + let cases = [ + ( + "conflict before review required wait", + Some("REVIEW_REQUIRED"), + "CONFLICTING", + "DIRTY", + 0, + "pull_request_merge_conflict", + ), + ( + "conflict before pending review wait", + None, + "CONFLICTING", + "DIRTY", + 1, + "pull_request_merge_conflict", + ), + ( + "behind before review required wait", + Some("REVIEW_REQUIRED"), + "MERGEABLE", + "BEHIND", + 0, + "pull_request_branch_behind_base", + ), + ( + "behind before pending review wait", + None, + "MERGEABLE", + "BEHIND", + 1, + "pull_request_branch_behind_base", + ), + ]; + + for (case_name, review_decision, mergeable, merge_state, pending_requests, reason) in cases { + let classification = classify_post_review_lane_with_pr_state( + review_decision, + mergeable, + merge_state, + Some("SUCCESS"), + pending_requests, + ); + + assert_eq!( + classification.decision, + PostReviewLaneDecision::NeedsReviewRepair, + "{case_name}" + ); + assert_eq!(classification.reason, reason, "{case_name}"); + } +} + +fn classify_post_review_lane_with_pr_state( + review_decision: Option<&str>, + mergeable: &str, + merge_state: &str, + status_check_state: Option<&str>, + pending_review_requests: usize, +) -> PostReviewLaneClassification { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let head_oid = String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"); + let worktree_path = temp_dir.path().join("lane"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let worktree = state_store + .list_worktrees("pubfi") + .expect("worktree list should succeed") + .into_iter() + .next() + .expect("worktree should exist"); + let snapshot = PostReviewLaneSnapshot { + issue, + worktree, + review_handoff: Some(sample_review_handoff_marker( + "x/pubfi-pub-101", + "https://github.com/hack-ink/decodex/pull/174", + &head_oid, + )), + local_branch_name: Some(String::from("x/pubfi-pub-101")), + local_head_oid: Some(head_oid.clone()), + }; + let review_state = sample_pull_request_review_state_with_pending_requests( + "https://github.com/hack-ink/decodex/pull/174", + "x/pubfi-pub-101", + &head_oid, + review_decision, + mergeable, + merge_state, + status_check_state, + 0, + pending_review_requests, + ); + + orchestrator::classify_post_review_lane( + &snapshot, + &state_store, + &sample_workflow(), + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("classification should succeed") +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs b/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs new file mode 100644 index 00000000..aa7cde26 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs @@ -0,0 +1,1249 @@ +#[test] +fn reconcile_post_review_orchestration_requests_external_review_without_thumbs_up_baseline() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&config, "PATH"); + let _path_guard = install_fake_post_issue_comment_gh_response( + &temp_dir, + TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID, + "2025-11-03T00:00:00Z", + ); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let initial_review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(initial_review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "waiting_for_ack"); + assert_eq!(marker.request_description_thumbs_up_count(), None); +} + +#[test] +fn reconcile_post_review_orchestration_uses_matching_handoff_record_for_current_branch() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&config, "PATH"); + let _path_guard = install_fake_post_issue_comment_gh_response( + &temp_dir, + TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID, + "2025-11-03T00:00:00Z", + ); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let current_branch = "main"; + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + current_branch, + &repo_root.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker(current_branch, pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + current_branch, + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + current_branch, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "waiting_for_ack"); + assert_eq!(marker.pr_url(), pr_url); +} + +#[test] +fn reconcile_post_review_orchestration_skips_merged_landed_lineage_without_manual_attention() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let pr_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let merge_commit_oid = + commit_worktree_change(&worktree.path, "landed.txt", "landed\n", "land retained lane"); + let current_head_oid = + commit_worktree_change(&worktree.path, "later.txt", "later\n", "advance main later"); + let pr_url = "https://github.com/hack-ink/decodex/pull/203"; + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &pr_head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + &worktree.branch_name, + pr_url, + ¤t_head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &pr_head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + review_state.merge_commit_oid = Some(merge_commit_oid); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("merged post-review orchestration should not fail"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + ); + + assert_eq!(marker.phase(), "request_pending"); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.state_updates.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_runs_admin_merge_after_external_pass() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&config, "PATH"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let landed_merge_subject = r#"{"schema":"decodex/commit/1","summary":"Land current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + let gh_invocation = fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!(marker.phase(), "waiting_for_merge"); + assert!(marker.auto_merge_enabled_at_unix_epoch().is_some()); + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + head_oid.clone(), + String::from("--subject"), + String::from(landed_merge_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + ] + ); +} + +#[test] +fn reconcile_post_review_orchestration_routes_non_clean_landing_to_agent_fallback() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&config, "PATH"); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "HAS_HOOKS", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "repair_required"); + assert!( + !invocation_log_path.exists(), + "non-clean retained landing must not invoke runtime admin merge" + ); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_runs_admin_merge_without_external_review_when_disabled() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&config, "PATH"), + false, + ); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let landed_merge_subject = r#"{"schema":"decodex/commit/1","summary":"Land current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + let gh_invocation = fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!(marker.phase(), "waiting_for_merge"); + assert!(marker.auto_merge_enabled_at_unix_epoch().is_some()); + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + head_oid, + String::from("--subject"), + String::from(landed_merge_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + ] + ); + assert!( + tracker.comments.borrow().is_empty(), + "runtime orchestration state should stay in StateStore rather than Linear comments", + ); +} + +#[test] +fn reconcile_post_review_orchestration_routes_internal_review_only_non_clean_landing_to_agent_fallback( +) { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled( + &service_config_with_github_token_env_var(&config, "PATH"), + false, + ); + let (_path_guard, invocation_log_path) = install_fake_admin_merge_gh_response(&temp_dir); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "HAS_HOOKS", + Some("SUCCESS"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "repair_required"); + assert!( + !invocation_log_path.exists(), + "non-clean internal-review-only retained landing must not invoke runtime admin merge" + ); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_tolerates_already_merged_merge_race() { + let (temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_github_token_env_var(&config, "PATH"); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let merge_subject = r#"{"schema":"decodex/commit/1","summary":"current retained handoff","authority":"PUB-101"}"#; + let landed_merge_subject = r#"{"schema":"decodex/commit/1","summary":"Land current retained handoff","authority":"PUB-101"}"#; + let head_oid = commit_worktree_change(&repo_root, "retained.txt", "ready\n", merge_subject); + let (_path_guard, invocation_log_path) = + install_fake_admin_merge_gh_response_with_merge_exit_code(&temp_dir, &head_oid, 1); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should accept an already-merged PR race"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + let gh_invocation = fs::read_to_string(&invocation_log_path) + .expect("fake gh invocation log should read") + .lines() + .map(str::to_owned) + .collect::>(); + + assert_eq!(marker.phase(), "waiting_for_merge"); + assert!(marker.auto_merge_enabled_at_unix_epoch().is_some()); + assert_eq!( + gh_invocation, + vec![ + String::from("pr"), + String::from("merge"), + String::from("--admin"), + String::from("--merge"), + String::from("--match-head-commit"), + head_oid, + String::from("--subject"), + String::from(landed_merge_subject), + String::from("--body"), + String::new(), + String::from(pr_url), + String::from("pr"), + String::from("view"), + String::from(pr_url), + String::from("--json"), + String::from("state,headRefOid,mergeCommit"), + ] + ); + assert!( + tracker.comments.borrow().is_empty(), + "already-merged race handling should persist orchestration in StateStore, not Linear comments", + ); + assert!(tracker.label_additions.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_waits_for_green_checks_before_requesting_external_review() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("PENDING"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "request_pending"); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_routes_fixable_ci_red_to_repair_before_requesting_external_review() + { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "BLOCKED", + Some("FAILURE"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "repair_required"); + + let comments = tracker.comments.borrow(); + + assert!(comments.is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_routes_thread_only_external_review_to_repair() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 1, + ); + + add_external_review_ack(&mut review_state); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "repair_required"); + + let comments = tracker.comments.borrow(); + + assert!(comments.is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_fails_closed_when_pull_request_is_closed() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("CLOSED"); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + assert_eq!(tracker.state_updates.borrow().len(), 1); + assert_eq!(tracker.comments.borrow().len(), 1); + assert_eq!(tracker.label_additions.borrow().len(), 1); + assert_eq!(tracker.label_removals.borrow().len(), 1); +} + +#[test] +fn reconcile_post_review_orchestration_skips_issue_with_active_lease() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + assert!( + state_store + .try_acquire_lease( + config.service_id(), + &issue.id, + "active-repair-run", + workflow.frontmatter().tracker().in_progress_state(), + ) + .expect("active lease should acquire") + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review orchestration should skip active repair lanes"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "waiting_for_result"); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.state_updates.borrow().is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_fails_closed_when_review_handoff_is_missing() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review orchestration should succeed"); + + assert_eq!(tracker.state_updates.borrow().len(), 1); + assert_eq!(tracker.comments.borrow().len(), 1); + assert_eq!(tracker.label_additions.borrow().len(), 1); + assert_eq!(tracker.label_removals.borrow().len(), 1); +} + +#[test] +fn reconcile_post_review_orchestration_skips_issue_without_service_active_label() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review orchestration should skip unowned retained lanes"); + + let marker = persisted_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + ); + + assert_eq!(marker.phase(), "request_pending"); + assert!(tracker.comments.borrow().is_empty()); + assert!(tracker.state_updates.borrow().is_empty()); + assert!(tracker.label_updates.borrow().is_empty()); + assert!(tracker.label_additions.borrow().is_empty()); + assert!(tracker.label_removals.borrow().is_empty()); +} + +#[test] +fn reconcile_post_review_orchestration_blocks_unhandled_ci_red_before_requesting_external_review() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = post_review_sample_service_owned_issue("In Review"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "request_pending", + None, + None, + None, + 0, + 0, + None, + ), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "UNSTABLE", + Some("FAILURE"), + 0, + ); + + orchestrator::reconcile_post_review_orchestration_with_inspector( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review orchestration should succeed"); + + let comments = tracker.comments.borrow(); + let comment = comments.first().expect("manual attention comment should be written"); + let ledger_event = records::parse_linear_execution_event_record(comment) + .expect("manual attention comment should include an execution ledger event"); + + assert_eq!(comments.len(), 1); + assert_eq!( + ledger_event.error_class.as_deref(), + Some("external_review_request_ci_red_manual_attention") + ); + assert_eq!(tracker.label_additions.borrow().len(), 1); + assert_eq!(tracker.label_removals.borrow().len(), 1); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/review_state.rs b/apps/decodex/src/orchestrator/tests/review_landing/review_state.rs new file mode 100644 index 00000000..e0e1b6c5 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/review_state.rs @@ -0,0 +1,91 @@ +#[test] +fn pull_request_review_state_from_page_scopes_signals_to_external_review_actor() { + let mut page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/173", + "main", + "deadbeef", + 0, + false, + None, + ); + + page.reaction_groups.push(PullRequestReactionGroup { + content: String::from("THUMBS_UP"), + users: PullRequestReactionUsersConnection { + nodes: vec![ + PullRequestActor { + login: String::from(crate::orchestrator::EXTERNAL_REVIEW_ACTOR_LOGIN), + }, + PullRequestActor { login: String::from(TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN) }, + ], + }, + }); + page.comments.nodes.push(PullRequestIssueCommentNode { + database_id: TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID, + body: String::from(EXTERNAL_REVIEW_REQUEST_BODY), + created_at: String::from("2025-11-03T00:00:00Z"), + author: Some(PullRequestActor { login: String::from("lane-owner") }), + reaction_groups: vec![PullRequestReactionGroup { + content: String::from("EYES"), + users: PullRequestReactionUsersConnection { + nodes: vec![ + PullRequestActor { login: String::from(TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN) }, + PullRequestActor { + login: String::from(crate::orchestrator::EXTERNAL_REVIEW_ACTOR_LOGIN), + }, + ], + }, + }], + }); + page.reviews.nodes.push(PullRequestReviewNode { + body: String::from(EXTERNAL_REVIEW_PASS_PHRASE), + state: String::from("APPROVED"), + submitted_at: Some(String::from("2025-11-03T00:00:01Z")), + author: Some(PullRequestActor { + login: String::from(TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN), + }), + }); + + let repository = sample_pull_request_review_state_repository(page); + let review_state = orchestrator::pull_request_review_state_from_page( + &repository, + repository.pull_request.as_ref().expect("pull request should exist"), + ); + + assert_eq!(review_state.issue_description_external_review_thumbs_up_count, 1); + assert_eq!(review_state.issue_comments.len(), 1); + assert_eq!(review_state.issue_comments[0].external_review_eyes_reaction_count, 1); + assert_eq!( + review_state.reviews[0].author_login.as_deref(), + Some(TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN) + ); +} + +#[test] +fn pull_request_review_state_from_page_skips_pending_reviews_without_submitted_timestamp() { + let mut page = sample_pull_request_review_state_page( + "https://github.com/hack-ink/decodex/pull/173", + "main", + "deadbeef", + 0, + false, + None, + ); + + page.reviews.nodes.push(PullRequestReviewNode { + body: String::from("pending"), + state: String::from("PENDING"), + submitted_at: None, + author: Some(PullRequestActor { + login: String::from(EXTERNAL_REVIEW_ACTOR_LOGIN), + }), + }); + + let repository = sample_pull_request_review_state_repository(page); + let review_state = orchestrator::pull_request_review_state_from_page( + &repository, + repository.pull_request.as_ref().expect("pull request should exist"), + ); + + assert!(review_state.reviews.is_empty()); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/status_markers.rs b/apps/decodex/src/orchestrator/tests/review_landing/status_markers.rs new file mode 100644 index 00000000..4798726e --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/status_markers.rs @@ -0,0 +1,57 @@ +#[test] +fn ensure_review_orchestration_marker_ignores_stale_tracker_record_from_prior_handoff() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let current_pr_url = "https://github.com/hack-ink/decodex/pull/173"; + let head_oid = "abc123"; + + state_store + .upsert_review_orchestration_marker( + config.service_id(), + &issue.id, + &ReviewOrchestrationMarker::new( + String::from("run-0"), + 7, + String::from("x/pubfi-pub-101"), + String::from("https://github.com/hack-ink/decodex/pull/99"), + String::from("deadbeef"), + String::from("waiting_for_merge"), + Some(TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID), + Some(TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT), + Some(1), + 2, + 3, + Some(TEST_EXTERNAL_REVIEW_AUTO_MERGE_ENABLED_AT), + ), + ) + .expect("stale orchestration marker should persist"); + + let marker = super::ensure_review_orchestration_marker( + config.service_id(), + &state_store, + &issue, + &sample_review_handoff_marker("x/pubfi-pub-101", current_pr_url, head_oid), + head_oid, + ) + .expect("fresh review orchestration marker should initialize"); + + assert_eq!(marker.run_id(), "run-1"); + assert_eq!(marker.attempt_number(), 1); + assert_eq!(marker.pr_url(), current_pr_url); + assert_eq!(marker.phase(), "request_pending"); + + let persisted_marker = state_store + .review_orchestration_marker( + config.service_id(), + &issue.id, + &sample_review_handoff_marker("x/pubfi-pub-101", current_pr_url, head_oid), + ) + .expect("runtime orchestration lookup should succeed") + .expect("runtime orchestration marker should persist"); + + assert_eq!(persisted_marker.run_id(), "run-1"); + assert_eq!(persisted_marker.attempt_number(), 1); + assert_eq!(persisted_marker.pr_url(), current_pr_url); + assert_eq!(persisted_marker.phase(), "request_pending"); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs b/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs new file mode 100644 index 00000000..34814b9c --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs @@ -0,0 +1,1134 @@ +#[test] +fn build_post_review_lane_statuses_reports_ready_to_land() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "ready_to_land"); + assert_eq!(lanes[0].reason, "external_review_passed_strict"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_skips_external_review_when_disabled() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let config = service_config_with_external_review_enabled(&config, false); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "ready_to_land"); + assert_eq!(lanes[0].reason, "internal_review_only_ready_to_land"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_routes_mixed_external_pass_and_feedback_to_repair() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, "waiting_for_result", 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + + review_state.issue_description_external_review_thumbs_up_count = 1; + + add_external_review_summary( + &mut review_state, + "Didn't find any major issues. Please fix X.", + "COMMENTED", + TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT + 1, + ); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "needs_review_repair"); + assert_eq!(lanes[0].reason, "external_review_feedback_pending_repair"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_ignores_non_external_review_signals() { + for (phase, signal, expected_reason) in [ + ("waiting_for_ack", "ack", "external_review_ack_pending"), + ("waiting_for_result", "pass", "external_review_result_pending"), + ] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_orchestration_marker("main", pr_url, &head_oid, phase, 1), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + match signal { + "ack" => add_review_request_ack_from_actor( + &mut review_state, + TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN, + ), + "pass" => { + add_external_review_ack(&mut review_state); + add_external_review_pass_from_actor( + &mut review_state, + TEST_NON_EXTERNAL_REVIEW_ACTOR_LOGIN, + ); + } + + _ => unreachable!("test case should use a known non-external signal"), + } + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "wait_for_review"); + assert_eq!(lanes[0].reason, expected_reason); + } +} + +#[test] +fn build_post_review_lane_statuses_accepts_existing_description_thumbs_up_for_later_pass_rounds() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &ReviewOrchestrationMarker::new( + "run-1", + 1, + "main", + pr_url, + &head_oid, + "waiting_for_result", + Some(TEST_EXTERNAL_REVIEW_REQUEST_COMMENT_ID), + Some(TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT), + Some(1), + 0, + 1, + None, + ), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + + review_state.issue_description_external_review_thumbs_up_count = 1; + + add_external_review_summary( + &mut review_state, + EXTERNAL_REVIEW_PASS_PHRASE, + "APPROVED", + TEST_EXTERNAL_REVIEW_REQUEST_CREATED_AT + 1, + ); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "ready_to_land"); + assert_eq!(lanes[0].reason, "external_review_passed_strict"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_keeps_completed_issue_visible_for_closeout_tail_work() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("Done", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].issue_state, "Done"); + assert_eq!(lanes[0].classification, "continue"); + assert_eq!(lanes[0].reason, "pull_request_merged_closeout_pending"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_keeps_merged_closeout_visible_after_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root().to_path_buf(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &repo_root, + &sample_review_handoff_marker("main", pr_url, &head_oid), + ); + + for attempt in 1..=3 { + state_store + .record_run_attempt(&format!("run-{attempt}"), &issue.id, attempt, "failed") + .expect("failed attempt should record"); + } + + let mut review_state = sample_pull_request_review_state( + pr_url, + "main", + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].issue_state, "In Review"); + assert_eq!(lanes[0].classification, "continue"); + assert_eq!(lanes[0].reason, "pull_request_merged_closeout_pending"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); + assert_eq!(lanes[0].pr_state.as_deref(), Some("MERGED")); +} + +#[test] +fn build_post_review_lane_statuses_keeps_merged_closeout_visible_after_landed_main_advances() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let pr_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let merge_commit_oid = + commit_worktree_change(&worktree.path, "landed.txt", "landed\n", "land retained lane"); + let current_head_oid = + commit_worktree_change(&worktree.path, "later.txt", "later\n", "advance main later"); + let pr_url = "https://github.com/hack-ink/decodex/pull/203"; + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &pr_head_oid), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &pr_head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + review_state.state = String::from("MERGED"); + review_state.merge_commit_oid = Some(merge_commit_oid); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(git_output(&worktree.path, &["rev-parse", "HEAD"]), current_head_oid); + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "continue"); + assert_eq!(lanes[0].reason, "pull_request_merged_closeout_pending"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); + assert_eq!(lanes[0].pr_state.as_deref(), Some("MERGED")); +} + +#[test] +fn build_post_review_lane_statuses_leaves_managed_worktree_git_metadata_untouched() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["config", "--local", "codex.github-identity", "y"]) + .status() + .expect("git config should run") + .success() + ); + assert!( + Command::new("git") + .arg("-C") + .arg(config.repo_root()) + .args(["config", "--local", "codex.linear-workspace", "hackink"]) + .status() + .expect("git config should run") + .success() + ); + + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&worktree.path) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + remove_local_git_metadata_for_post_review_status(&worktree.path); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_orchestration_marker( + &worktree.branch_name, + pr_url, + &head_oid, + "waiting_for_result", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "ready_to_land"); + assert_eq!(try_git_local_config_value(&worktree.path, "codex.github-identity"), None); + assert_eq!(try_git_local_config_value(&worktree.path, "codex.linear-workspace"), None); + assert_eq!(git_remote_url(&worktree.path, "origin"), None); +} + +#[test] +fn build_post_review_lane_statuses_blocks_missing_review_handoff_record() { + for managed_worktree in [false, true] { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + if managed_worktree { + let worktree_manager = WorktreeManager::new( + config.service_id(), + config.repo_root(), + config.worktree_root(), + ); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("worktree should exist"); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + } else { + let repo_root = config.repo_root().to_path_buf(); + + state_store + .upsert_worktree("pubfi", &issue.id, "main", &repo_root.display().to_string()) + .expect("worktree should record"); + } + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "missing_review_handoff_record"); + } +} + +#[test] +fn build_post_review_lane_statuses_allows_descendant_review_handoff_head_after_repair_push() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let marker_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let current_head_oid = + commit_worktree_change(&worktree.path, "repair.txt", "repair push\n", "repair push"); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &marker_head_oid), + ); + seed_review_orchestration_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_orchestration_marker( + &worktree.branch_name, + pr_url, + ¤t_head_oid, + "waiting_for_result", + 1, + ), + ); + + let mut review_state = sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + ¤t_head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ); + + add_external_review_ack(&mut review_state); + add_external_review_pass(&mut review_state); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(review_state)]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "ready_to_land"); + assert_eq!(lanes[0].reason, "external_review_passed_strict"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); +} + +#[test] +fn build_post_review_lane_statuses_blocks_review_handoff_lineage_rewrite() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let marker_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + git_status_success(&worktree.path, &["checkout", "--orphan", "rewrite-history"]); + + fs::write(worktree.path.join("rewrite.txt"), "rewritten history\n") + .expect("rewrite file should write"); + + git_status_success(&worktree.path, &["add", "rewrite.txt"]); + git_status_success(&worktree.path, &["commit", "-m", "rewrite history"]); + git_status_success(&worktree.path, &["branch", "-M", &worktree.branch_name]); + + let rewritten_head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &marker_head_oid), + ); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &rewritten_head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "review_handoff_lineage_mismatch"); +} + +#[test] +fn build_post_review_lane_statuses_blocks_nonactive_labeled_post_review_issues() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + for (labels, expected_reason) in [ + (&["decodex:manual-only"][..], "issue_opted_out"), + (&["decodex:needs-attention"][..], "issue_needs_attention"), + ] { + let issue = sample_issue("In Review", labels); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "x/pubfi-pub-101", + &config.repo_root().display().to_string(), + ) + .expect("worktree should record"); + + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, expected_reason); + + state_store.clear_worktree(&issue.id).expect("worktree should clear between label cases"); + } +} + +#[test] +fn build_post_review_lane_statuses_blocks_exhausted_retry_budget() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "x/pubfi-pub-101", + &config.repo_root().display().to_string(), + ) + .expect("worktree should record"); + + for attempt in 1..=3 { + state_store + .record_run_attempt(&format!("run-{attempt}"), &issue.id, attempt, "failed") + .expect("failed attempt should record"); + } + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "retry_budget_exhausted"); +} + +#[test] +fn build_post_review_lane_statuses_keeps_unmerged_retry_budget_blocked() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = worktree_manager + .ensure_worktree(&issue.identifier, false) + .expect("retained worktree should exist"); + let head_oid = git_output(&worktree.path, &["rev-parse", "HEAD"]); + let pr_url = "https://github.com/hack-ink/decodex/pull/120"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &ReviewHandoffMarker::new( + "run-review-handoff", + 1, + &worktree.branch_name, + pr_url, + "main", + &worktree.branch_name, + &head_oid, + ), + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + for attempt in 1..=3 { + state_store + .record_run_attempt(&format!("run-{attempt}"), &issue.id, attempt, "failed") + .expect("failed attempt should record"); + } + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(vec![Ok(sample_pull_request_review_state( + pr_url, + &worktree.branch_name, + &head_oid, + Some("APPROVED"), + "MERGEABLE", + "CLEAN", + Some("SUCCESS"), + 0, + ))]), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "retry_budget_exhausted"); + assert_eq!(lanes[0].pr_url.as_deref(), Some(pr_url)); + assert_eq!(lanes[0].pr_state.as_deref(), Some("OPEN")); +} + +#[test] +fn build_post_review_lane_statuses_blocks_worktree_head_read_failures() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let branch_ref_path = + config.repo_root().join(".git").join("refs").join("heads").join(&worktree.branch_name); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&worktree.path) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + seed_review_handoff_marker_value( + &state_store, + config.service_id(), + &issue.id, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + fs::remove_file(&branch_ref_path).expect("branch ref should remove"); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "worktree_head_read_failed"); +} + +#[test] +fn build_post_review_lane_statuses_blocks_missing_worktree_checkout_branch() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Review", &[]); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let worktree = + worktree_manager.ensure_worktree(&issue.identifier, false).expect("worktree should exist"); + let head_oid = String::from_utf8( + Command::new("git") + .arg("-C") + .arg(&worktree.path) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse should run") + .stdout, + ) + .expect("git output should be utf-8") + .trim() + .to_owned(); + let pr_url = "https://github.com/hack-ink/decodex/pull/173"; + + assert!( + Command::new("git") + .arg("-C") + .arg(&worktree.path) + .args(["checkout", "--detach", &head_oid]) + .status() + .expect("git checkout --detach should run") + .success() + ); + + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + ) + .expect("worktree should record"); + + seed_review_handoff_marker_for_path( + &state_store, + config.service_id(), + &worktree.path, + &sample_review_handoff_marker(&worktree.branch_name, pr_url, &head_oid), + ); + + let lanes = orchestrator::build_post_review_lane_statuses( + &tracker, + &config, + &workflow, + &state_store, + &FakePullRequestReviewStateInspector::new(Vec::new()), + ) + .expect("post-review lane status build should succeed"); + + assert_eq!(lanes.len(), 1); + assert_eq!(lanes[0].classification, "blocked"); + assert_eq!(lanes[0].reason, "worktree_checkout_branch_missing"); +} diff --git a/apps/decodex/src/orchestrator/tests/review_landing/status_support.rs b/apps/decodex/src/orchestrator/tests/review_landing/status_support.rs new file mode 100644 index 00000000..2c49728d --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/review_landing/status_support.rs @@ -0,0 +1,22 @@ +fn post_review_sample_service_owned_issue(state_name: &str) -> TrackerIssue { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + + sample_issue(state_name, &[active_label.as_str()]) +} + +fn remove_local_git_metadata_for_post_review_status(worktree_path: &Path) { + let commands: &[&[&str]] = &[ + &["config", "--local", "--unset-all", "codex.github-identity"], + &["config", "--local", "--unset-all", "codex.linear-workspace"], + &["remote", "remove", "origin"], + ]; + + for args in commands { + Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(*args) + .status() + .expect("git metadata cleanup should run"); + } +} diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs new file mode 100644 index 00000000..60cd93a6 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -0,0 +1,1211 @@ +use orchestrator::{ + AgentGitCredentialEnvironment, AgentGitCredentialsUnavailable, RepoGateFailureKind, +}; + +fn git_config_value( + repo_root: &Path, + key: &str, + credentials: Option<&AgentGitCredentialEnvironment>, +) -> Option { + let mut probe = Command::new("git"); + + probe.arg("-C").arg(repo_root).args(["config", "--get", key]); + + if let Some(credentials) = credentials { + credentials.process_env().apply_to(&mut probe).expect("agent env should apply"); + } + + let output = probe.output().expect("git config probe should run"); + + output.status.success().then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn injected_git_config_keys(credentials: &AgentGitCredentialEnvironment) -> Vec { + let mut probe = Command::new("git"); + + credentials.process_env().apply_to(&mut probe).expect("agent env should apply"); + + probe + .get_envs() + .filter_map(|(key, value)| { + Some((key.to_string_lossy().into_owned(), value?.to_string_lossy().into_owned())) + }) + .filter(|(key, _)| key.starts_with("GIT_CONFIG_KEY_")) + .map(|(_, value)| value) + .collect() +} + +#[test] +fn terminal_failure_comments_surface_actionable_error_classes() { + for (error_class, next_action, expected_snippets) in [ + ( + "human_attention_required", + "inspect the issue comment and worktree, resolve the blocker manually, clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + &["inspect the issue comment and worktree", "resolve the blocker manually"][..], + ), + ( + "review_handoff_writeback_failed", + "inspect the tracker state, PR, and worktree, repair the incomplete review handoff manually, clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + &["repair the incomplete review handoff manually"][..], + ), + ( + "stalled_run_detected", + "inspect the worktree and app-server activity for the stalled lane, resolve the blocker manually, `decodex:needs-attention` could not be applied because it does not exist on the team; the issue remains in `In Progress` to block automatic retries, so move it back to a startable state manually if another automated run is desired", + &["does not exist on the team", "remains in `In Progress`"][..], + ), + ] { + let comment = orchestrator::format_terminal_failure_comment( + "pub-101-attempt-1-123", + 1, + String::from(".worktrees/PUB-101"), + "x/pubfi-pub-101", + error_class, + next_action, + ); + + assert!(comment.contains(&format!("- error_class: `{error_class}`"))); + assert!(comment.contains("Sensitive runtime details were withheld")); + + for expected_snippet in expected_snippets { + assert!(comment.contains(expected_snippet), "{error_class} missing {expected_snippet}"); + } + } +} + +#[test] +fn review_policy_terminal_failure_details_include_research_boundaries() { + for (reason, error_class, expected_snippet) in [ + ( + ReviewPolicyStopReason::Exhausted, + "review_policy_exhausted", + "bounded convergence research follow-up", + ), + ( + ReviewPolicyStopReason::ArchitectureReviewRequired, + "architecture_review_required", + "bounded architecture research follow-up", + ), + ( + ReviewPolicyStopReason::Blocked, + "review_policy_blocked", + "do not dispatch research", + ), + ] { + let error = Report::new(ReviewPolicyStopRequested { + head_sha: String::from("08a20f7dfb9526e7421a5f095b1c6adec84e52d6"), + issue_identifier: String::from("PUB-101"), + nonclean_rounds: Some(3), + reason, + run_id: String::from("pub-101-attempt-1-123"), + }); + let (actual_error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ); + + assert_eq!(actual_error_class, error_class); + assert!(next_action.contains(expected_snippet), "{error_class} missing research boundary"); + assert!(next_action.contains("clear label `decodex:needs-attention`")); + } +} + +#[test] +fn preserve_manual_attention_request_wraps_finalize_miss() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = orchestrator::preserve_manual_attention_request( + Ok(RunCompletionDisposition::ManualAttention), + &issue_run, + &workflow, + Report::msg("run completed without issue_terminal_finalize"), + ); + + assert!(error.downcast_ref::().is_some()); + assert!(error.to_string().contains("run completed without issue_terminal_finalize")); +} + +#[test] +fn retained_partial_progress_uses_actionable_terminal_failure_comment() { + let error = Report::new(RetainedPartialProgress { + issue_identifier: String::from("PUB-101"), + run_id: String::from("pub-101-attempt-3-123"), + worktree_path: String::from(".worktrees/PUB-101"), + }); + let (error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ); + + assert_eq!(error_class, "partial_progress_retained"); + assert!(next_action.contains("inspect retained worktree `.worktrees/PUB-101`")); + assert!(next_action.contains("finish validation and PR handoff or reset the patch manually")); + assert!(next_action.contains("clear label `decodex:needs-attention`")); +} + +#[test] +fn ensure_automation_activity_label_noops_when_active_ownership_is_confirmed() { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let mut issue = sample_issue("In Progress", &[]); + + issue.labels_complete = false; + + issue.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::new(vec![issue.clone()]) + .with_label_lookup_issues(&active_label, vec![issue.clone()]); + + orchestrator::ensure_automation_activity_label(&tracker, &issue, TEST_SERVICE_ID, true).expect( + "server-confirmed active ownership should not fail when the first label page is truncated", + ); + + assert!( + tracker.label_updates.borrow().is_empty() + && tracker.label_additions.borrow().is_empty() + && tracker.label_removals.borrow().is_empty(), + "server-confirmed active ownership should not trigger a label mutation" + ); + + let mut issue = sample_issue("In Progress", &[active_label.as_str()]); + + issue.team.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::new(vec![issue.clone()]); + + orchestrator::ensure_automation_activity_label(&tracker, &issue, TEST_SERVICE_ID, true) + .expect("existing active ownership should not require a paginated team-label lookup"); + + assert!( + tracker.label_updates.borrow().is_empty() + && tracker.label_additions.borrow().is_empty() + && tracker.label_removals.borrow().is_empty(), + "no-op active-label checks should not trigger a label mutation" + ); +} + +#[test] +fn ensure_automation_activity_label_uses_incremental_team_label_lookup_for_mutation() { + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let mut issue = sample_issue("In Progress", &[]); + + issue.labels_complete = false; + + issue.team.labels.retain(|label| label.name != active_label.as_str()); + + let tracker = FakeTracker::new(vec![issue.clone()]).with_team_label_lookup_id( + &issue.team.id, + &active_label, + "label-active", + ); + + orchestrator::ensure_automation_activity_label(&tracker, &issue, TEST_SERVICE_ID, true) + .expect("active-label mutation should resolve the team label id server-side"); + + assert_eq!( + tracker.label_additions.borrow().as_slice(), + [(issue.id.clone(), vec![String::from("label-active")])], + ); + assert!(tracker.label_updates.borrow().is_empty()); +} + +#[test] +fn review_policy_terminal_failure_comments_use_runtime_owned_error_classes() { + for (error_class, next_action) in [ + ( + "review_policy_exhausted", + "inspect the repeated review findings and current worktree, decide the next repair or redesign manually, prepare a bounded convergence research follow-up only after the current head, review phase, non-clean round count, and validated findings are structured and machine-checkable, clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ), + ( + "architecture_review_required", + "inspect the current findings and worktree, perform the required architecture review manually, prepare a bounded architecture research follow-up only after the current head, review phase, stop class, and architecture concern are structured and machine-checkable, clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ), + ( + "review_policy_blocked", + "inspect the blocking condition and worktree, resolve the blocker manually, do not dispatch research unless the blocker is reclassified as a structured architecture or convergence stop, clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ), + ] { + let comment = orchestrator::format_terminal_failure_comment( + "pub-101-attempt-1-123", + 1, + String::from(".worktrees/PUB-101"), + "x/pubfi-pub-101", + error_class, + next_action, + ); + + assert!(comment.contains(&format!("- error_class: `{error_class}`"))); + assert!(comment.contains("Sensitive runtime details were withheld")); + } +} + +#[test] +fn retry_failure_comments_withhold_raw_error_text() { + let comment = orchestrator::format_retry_comment(RetryComment { + run_id: "pub-101-attempt-1-123", + attempt_number: 1, + retry_budget_attempt_number: 1, + max_attempts: 3, + worktree_path: String::from(".worktrees/PUB-101"), + branch_name: "x/pubfi-pub-101", + error_class: "retryable_execution_failure", + next_action: "decodex will retry automatically", + }); + + assert!(comment.contains("- error_class: `retryable_execution_failure`")); + assert!(comment.contains("Sensitive runtime details were withheld")); + assert!(!comment.contains("error:")); +} + +#[test] +fn repo_gate_retry_comments_preserve_continued_repair_error_class() { + let comment = orchestrator::format_retry_comment(RetryComment { + run_id: "pub-101-attempt-1-123", + attempt_number: 1, + retry_budget_attempt_number: 1, + max_attempts: 3, + worktree_path: String::from(".worktrees/PUB-101"), + branch_name: "x/pubfi-pub-101", + error_class: "repo_gate_verify_failed", + next_action: "additional agent repair is required before repo verification can pass; decodex will retry automatically", + }); + + assert!(comment.contains("- error_class: `repo_gate_verify_failed`")); + assert!( + comment.contains("additional agent repair is required before repo verification can pass") + ); +} + +#[test] +fn repo_gate_lock_contention_retry_comments_preserve_specific_error_class() { + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::GitLockContention, + String::from( + "Failed to inspect tracked-file cleanliness after repo gate verification in `/tmp/repo`: fatal: Unable to create '.git/index.lock': File exists.", + ), + )); + let (error_class, next_action) = orchestrator::retry_comment_details(&error); + + assert_eq!(error_class, "repo_gate_git_lock_contention"); + assert!(next_action.contains("`.git/index.lock`")); + assert!(next_action.contains("retry automatically")); +} + +#[test] +fn repo_gate_lock_contention_runtime_retry_writes_specific_retry_schedule_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path.clone(), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::GitLockContention, + String::from( + "Failed to inspect tracked-file cleanliness after repo gate verification in `/tmp/repo`: fatal: Unable to create '.git/index.lock': File exists.", + ), + )); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + orchestrator::write_retry_schedule_marker_for_runtime_retry(&error, &workflow, &issue_run, 1) + .expect("lock contention should write a specific retry marker"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("retry schedule should remain readable") + .expect("retry marker should exist"); + + assert_eq!(marker.retry_kind(), Some("git_lock_contention")); + assert!( + marker.retry_ready_at_unix_epoch().is_some_and( + |retry_ready_at| retry_ready_at > OffsetDateTime::now_utc().unix_timestamp() + ) + ); +} + +#[test] +fn retry_budget_current_failure_does_not_double_count_handed_off_base() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.worktree_root().join("PUB-101"), + reused_existing: false, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2-456"), + retry_budget_base: 1, + }; + + state_store + .record_run_attempt("pub-101-attempt-1-123", &issue.id, 1, "failed") + .expect("previous failed attempt should record"); + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("current failed attempt should record"); + + assert_eq!( + orchestrator::retry_budget_attempts_for_current_failure(&state_store, &issue_run) + .expect("retry budget should compute"), + 2, + "the daemon handoff base already includes the previous persisted failed attempt" + ); +} + +#[test] +fn repo_gate_terminal_failures_preserve_specific_error_class_after_retry_exhaustion() { + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + String::from("Repo verify command `cargo make test` failed in `/tmp/repo`: test failed"), + )); + let (error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ); + + assert_eq!(error_class, "repo_gate_verify_failed"); + assert!(next_action.contains("repair the repo verification failure manually")); +} + +#[test] +fn repo_gate_lock_contention_terminal_failures_preserve_specific_error_class_after_retry_exhaustion() + { + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::GitLockContention, + String::from( + "Failed to inspect tracked-file cleanliness after repo gate verification in `/tmp/repo`: fatal: Unable to create '.git/index.lock': File exists.", + ), + )); + let (error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ); + + assert_eq!(error_class, "repo_gate_git_lock_contention"); + assert!(next_action.contains("active or stale `.git/index.lock` holder")); +} + +#[test] +fn app_server_terminal_failures_preserve_specific_error_classes() { + let cases = [ + ( + Report::new(AppServerCapabilityPreflightFailure::blocked_for_test( + "model", + "configured model was not present in model/list.", + )), + "app_server_runtime_preflight_failed", + "repair the local Codex config/model/provider/skills/plugin/MCP state", + ), + ( + Report::new(AppServerHomePreflightFailure::resolution_failed(String::from( + "app_server_preflight_failed: HOME is not set, so Decodex cannot resolve the shared Codex home for app-server dispatch.", + ))), + "app_server_codex_home_preflight_failed", + "keep CODEX_HOME/CODEX_SQLITE_HOME shared instead of per-account", + ), + ( + Report::new(AppServerHomePreflightFailure::initialize_mismatch( + String::from("/tmp/per-account-codex-home"), + String::from("/Users/test/.codex"), + )), + "app_server_codex_home_mismatch", + "restart `decodex serve`", + ), + ( + Report::new(AppServerTransportFailure::new(String::from( + "App-server stdout disconnected unexpectedly.", + ))), + "app_server_transport_disconnected", + "inspect the local app-server stderr tail", + ), + ( + Report::new(AppServerTurnFailure::new( + "thread-1", + Some(String::from("turn-1")), + "failed", + "You've hit your usage limit.", + Some(String::from("usageLimitExceeded")), + )), + "app_server_usage_limit_exceeded", + "inspect Codex account usage", + ), + ]; + + for (error, expected_class, expected_action) in cases { + let (error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired", + ); + + assert_eq!(error_class, expected_class); + assert!(next_action.contains(expected_action)); + assert!(next_action.contains("clear label `decodex:needs-attention`")); + } +} + +#[test] +fn repo_gate_runtime_failures_require_manual_attention_without_retry_budget_wait() { + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::CommandSpawnFailed, + String::from( + "Failed to spawn repo gate command `cargo make fmt` in `/tmp/repo` via `/bin/sh` `-c`: missing tool", + ), + )); + let repo_gate_failure = error + .downcast_ref::() + .expect("repo gate failure should downcast"); + + assert_eq!( + repo_gate_failure.disposition(), + orchestrator::RepoGateFailureDisposition::NeedsHumanAttention + ); + assert_eq!(repo_gate_failure.error_class(), "repo_gate_command_spawn_failed"); +} + +#[test] +fn operation_marker_write_failures_do_not_abort_completion_flow() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let occupied_path = temp_dir.path().join("occupied"); + + fs::write(&occupied_path, "not a directory").expect("blocking file should write"); + orchestrator::write_run_operation_marker_best_effort( + &occupied_path, + "run-1", + 1, + RUN_OPERATION_REPO_GATE, + ); + orchestrator::write_run_operation_marker_best_effort( + &occupied_path, + "run-1", + 1, + RUN_OPERATION_RECONCILIATION, + ); + + assert!(occupied_path.is_file()); + assert!(!occupied_path.join(RUN_ACTIVITY_MARKER_FILE).exists()); +} + +#[test] +fn validate_review_handoff_runtime_requires_gh_and_github_token_authority() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + + { + let _env_lock = TestEnvVarGuard::lock(); + let missing_env_var = format!("DECODEX_TEST_MISSING_GITHUB_TOKEN_ENV_{}", process::id()); + let config_missing_github = + service_config_with_github_token_env_var(&config, &missing_env_var); + + assert!(orchestrator::validate_review_handoff_runtime(&config, true).is_ok()); + assert!(orchestrator::validate_review_handoff_runtime(&config, false).is_ok()); + assert!(orchestrator::validate_daemon_runtime().is_ok()); + assert!(orchestrator::validate_command_available("git", "test preflight").is_ok()); + + let error = orchestrator::validate_review_handoff_runtime(&config_missing_github, false) + .expect_err("missing github token env-var should fail live preflight"); + + assert!(error.to_string().contains("github.token_env_var")); + } + + let env_var = format!("DECODEX_TEST_BLANK_GITHUB_TOKEN_ENV_{}", process::id()); + let _env_guard = TestEnvVarGuard::set(&env_var, ""); + let config_blank_github = service_config_with_github_token_env_var(&config, &env_var); + let error = orchestrator::validate_review_handoff_runtime(&config_blank_github, false) + .expect_err("blank github token authority should fail live preflight"); + + assert!(error.to_string().contains("must not be blank")); + + let error = orchestrator::validate_command_available( + "__decodex_missing_command__", + "PR-backed review handoff", + ) + .expect_err("missing command should fail preflight"); + + assert!( + error.to_string().contains("Required command `__decodex_missing_command__` is unavailable") + ); +} + +#[test] +fn agent_git_credentials_use_runtime_env_without_persisting_the_token() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let env_var = format!("DECODEX_TEST_AGENT_GITHUB_TOKEN_ENV_{}", process::id()); + let _env_guard = TestEnvVarGuard::set(&env_var, "secret-token-value"); + let config = service_config_with_github_token_env_var(&config, &env_var); + let askpass_path = + orchestrator::agent_git_askpass_path(config.worktree_root(), "run/with spaces"); + let credentials = + orchestrator::prepare_agent_git_credentials(&config, "run/with spaces", config.repo_root()) + .expect("agent Git credentials should prepare"); + let script = fs::read_to_string(&askpass_path).expect("askpass script should exist"); + + assert!(askpass_path.exists()); + assert!(script.contains("GH_TOKEN")); + assert!(!script.contains("secret-token-value")); + + let inherited_signing_key = git_config_value(config.repo_root(), "user.signingkey", None); + let agent_signing_key = + git_config_value(config.repo_root(), "user.signingkey", Some(&credentials)); + + assert_eq!( + agent_signing_key, inherited_signing_key, + "agent git environment should preserve inherited signing keys when the repo has no local key" + ); + assert_eq!( + git_config_value(config.repo_root(), "commit.gpgsign", Some(&credentials)).as_deref(), + Some("false") + ); + + let inherited_git_config_keys = injected_git_config_keys(&credentials); + + assert!( + !inherited_git_config_keys.iter().any(|key| key == "commit.gpgsign"), + "agent git environment should not disable inherited commit signing" + ); + assert!( + !inherited_git_config_keys.iter().any(|key| key == "tag.gpgsign"), + "agent git environment should not disable inherited tag signing" + ); + assert!( + !inherited_git_config_keys.iter().any(|key| key == "user.signingkey"), + "agent git environment should not mask inherited signing keys" + ); + + #[cfg(unix)] + { + assert_eq!( + std::os::unix::fs::PermissionsExt::mode( + &fs::metadata(&askpass_path).expect("askpass metadata should load").permissions(), + ) & 0o777, + 0o700 + ); + + let github_username = Command::new(&askpass_path) + .arg("Username for 'https://github.com/hack-ink/decodex.git'") + .env("GH_TOKEN", "secret-token-value") + .output() + .expect("askpass helper should execute"); + + assert!(github_username.status.success()); + assert_eq!(String::from_utf8_lossy(&github_username.stdout).trim(), "x-access-token"); + + let github_password = Command::new(&askpass_path) + .arg("Password for 'https://x-access-token@github.com/hack-ink/decodex.git'") + .env("GH_TOKEN", "secret-token-value") + .output() + .expect("askpass helper should execute"); + + assert!(github_password.status.success()); + assert_eq!(String::from_utf8_lossy(&github_password.stdout).trim(), "secret-token-value"); + + let foreign_password = Command::new(&askpass_path) + .arg("Password for 'https://example.com/repo.git'") + .env("GH_TOKEN", "secret-token-value") + .output() + .expect("askpass helper should execute"); + + assert!( + !foreign_password.status.success(), + "askpass helper should reject non-GitHub prompts" + ); + assert!( + !String::from_utf8_lossy(&foreign_password.stdout).contains("secret-token-value"), + "askpass helper should not leak the GitHub token to non-GitHub prompts" + ); + + let lookalike_password = Command::new(&askpass_path) + .arg("Password for 'https://x-access-token@github.com.evil/repo.git'") + .env("GH_TOKEN", "secret-token-value") + .output() + .expect("askpass helper should execute"); + + assert!( + !lookalike_password.status.success(), + "askpass helper should reject GitHub lookalike hosts" + ); + assert!( + !String::from_utf8_lossy(&lookalike_password.stdout).contains("secret-token-value"), + "askpass helper should not leak the GitHub token to lookalike hosts" + ); + } + + drop(credentials); + + assert!( + !askpass_path.exists(), + "runtime askpass helper should be removed after the run environment drops" + ); +} + +#[test] +fn agent_git_credentials_pin_repo_local_signing_key_when_configured() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let env_var = format!("DECODEX_TEST_AGENT_SIGNING_GITHUB_TOKEN_ENV_{}", process::id()); + let _env_guard = TestEnvVarGuard::set(&env_var, "secret-token-value"); + let config = service_config_with_github_token_env_var(&config, &env_var); + + git_status_success(config.repo_root(), &["config", "user.signingkey", "route-y-signing-key"]); + + let credentials = orchestrator::prepare_agent_git_credentials( + &config, + "run-with-signing", + config.repo_root(), + ) + .expect("agent Git credentials should prepare"); + let mut signing_key_probe = Command::new("git"); + + signing_key_probe.arg("-C").arg(config.repo_root()).args([ + "config", + "--get", + "user.signingkey", + ]); + credentials.process_env().apply_to(&mut signing_key_probe).expect("agent env should apply"); + + let output = signing_key_probe.output().expect("git signing key probe should run"); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "route-y-signing-key"); +} + +#[test] +fn missing_agent_git_credentials_stop_without_retry() { + let _env_lock = TestEnvVarGuard::lock(); + let (_temp_dir, config, _workflow) = temp_project_layout(); + let missing_env_var = format!("DECODEX_TEST_MISSING_AGENT_GITHUB_TOKEN_ENV_{}", process::id()); + let config = service_config_with_github_token_env_var(&config, &missing_env_var); + let error = match orchestrator::prepare_agent_git_credentials( + &config, + "run-missing-token", + config.repo_root(), + ) { + Ok(_) => panic!("missing github token should fail before app-server launch"), + Err(error) => error, + }; + let credentials_error = error + .downcast_ref::() + .expect("credential preflight failure should be typed"); + let (error_class, next_action) = orchestrator::terminal_failure_comment_details( + false, + &error, + "clear label `decodex:needs-attention`", + ); + + assert_eq!(credentials_error.token_env_var, missing_env_var); + assert_eq!(error_class, "github_credentials_unavailable"); + assert!(next_action.contains("configure")); + assert!(next_action.contains(&missing_env_var)); +} + +#[test] +fn live_run_without_candidate_does_not_require_github_token_authority() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::with_refresh_snapshots_and_project(vec![], vec![vec![]], true); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("empty backlog should not require github token authority"); + + assert!(summary.is_none()); +} + +#[test] +fn prepare_issue_run_with_candidate_does_not_require_github_token_authority_before_agent_execution() +{ + let (_temp_dir, config, workflow) = temp_project_layout(); + let listed_issue = sample_issue("Todo", &[]); + let tracker = FakeTracker::with_refresh_snapshots( + vec![listed_issue.clone()], + vec![vec![listed_issue.clone()]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); + let issue_run = orchestrator::prepare_issue_run( + PrepareIssueRunContext { + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + worktree_manager: &worktree_manager, + dry_run: false, + lease_preacquired: false, + dispatch_mode: IssueDispatchMode::Normal, + preferred_issue_state: None, + preferred_initial_issue_state: None, + preferred_run_identity: None, + preferred_retry_budget_base: None, + }, + listed_issue.clone(), + ) + .expect("candidate dispatch should prepare without github token authority") + .expect("candidate issue should plan a run"); + + assert_eq!(issue_run.issue.id, listed_issue.id); + assert_eq!(issue_run.issue_state, "In Progress"); + assert!( + state_store.lease_for_issue(&listed_issue.id).expect("lease lookup should work").is_some() + ); + assert!( + state_store + .worktree_for_issue(&listed_issue.id) + .expect("worktree lookup should work") + .is_some() + ); + assert_eq!( + state_store + .latest_run_attempt_for_issue(&listed_issue.id) + .expect("run attempt lookup should work") + .expect("starting attempt should record") + .status(), + "starting" + ); +} + +#[test] +fn execute_issue_run_clears_lease_when_active_label_setup_fails() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let mut listed_issue = sample_issue("Todo", &[]); + let mut refreshed_issue = listed_issue.clone(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let worktree_path = config.worktree_root().join(&listed_issue.identifier); + + listed_issue.team.labels.retain(|label| label.name != active_label); + refreshed_issue.team.labels.retain(|label| label.name != active_label); + + let tracker = FakeTracker::with_refresh_snapshots( + vec![listed_issue.clone()], + vec![vec![refreshed_issue]], + ); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue_run = IssueRunPlan { + issue: listed_issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: listed_issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: listed_issue.identifier.clone(), + path: worktree_path.clone(), + reused_existing: false, + }, + retry_project_slug: listed_issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + + state_store + .record_run_attempt( + &issue_run.run_id, + &listed_issue.id, + issue_run.attempt_number, + "starting", + ) + .expect("run attempt should record"); + state_store + .upsert_lease(config.service_id(), &listed_issue.id, &issue_run.run_id, "In Progress") + .expect("lease should record"); + + let error = orchestrator::execute_issue_run( + &tracker, + &config, + &workflow, + &state_store, + issue_run.clone(), + ) + .expect_err("active-label setup failure should abort execution"); + + assert!(error.to_string().contains("required label")); + assert!( + state_store.lease_for_issue(&listed_issue.id).expect("lease lookup should work").is_none(), + "active-label setup failures should still release the lease" + ); + assert_eq!( + state_store + .run_attempt(&issue_run.run_id) + .expect("run attempt lookup should work") + .expect("run attempt should exist") + .status(), + "failed", + "active-label setup failures should mark the run failed before returning" + ); +} + +#[test] +fn reconciliation_clears_stale_leases_and_terminal_worktrees() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("Done", &[active_label.as_str()]); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let tracker = + FakeTracker::new(vec![issue.clone()]).with_label_lookup_issues(&queue_label, vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("reconciliation should succeed"); + + assert!(summary.is_none()); + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should work").is_none()); + assert!( + state_store.worktree_for_issue(&issue.id).expect("worktree lookup should work").is_none() + ); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should work") + .expect("run attempt should exist") + .status(), + "terminated" + ); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [ + (issue.id.clone(), vec![String::from("label-active")]), + (issue.id.clone(), vec![String::from("label-queued")]), + ] + ); +} + +#[test] +fn reconciliation_runs_without_project_validation() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("Done", &[active_label.as_str()]); + let queue_label = tracker::automation_queue_label(TEST_SERVICE_ID); + let tracker = FakeTracker::with_refresh_snapshots_and_project( + vec![issue.clone()], + vec![vec![issue.clone()]], + false, + ) + .with_label_lookup_issues(&queue_label, vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + + let summary = orchestrator::run_project_once(&tracker, &config, &workflow, &state_store, false) + .expect("reconciliation should still succeed without any project validation"); + + assert!(summary.is_none(), "reconciliation-only startup should not dispatch a new lane here"); + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should work").is_none()); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should work") + .expect("run attempt should exist") + .status(), + "terminated" + ); + assert_eq!( + tracker.label_removals.borrow().as_slice(), + [ + (issue.id.clone(), vec![String::from("label-active")]), + (issue.id.clone(), vec![String::from("label-queued")]), + ] + ); +} + +#[test] +fn exited_child_cleanup_updates_status_and_retry_budget_by_interrupt_flag() { + for (case_name, mark_interrupted, expected_status, expected_retry_budget) in [ + ("clean exit", false, "running", 0), + ("interrupted exit", true, "interrupted", 1), + ] { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + + orchestrator::clear_orphaned_daemon_child_state( + &state_store, + ChildRunRef { + issue_id: &issue.id, + run_id: "run-1", + attempt_number: 1, + }, + mark_interrupted, + ) + .expect(case_name); + + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none() + ); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + expected_status, + "{case_name}" + ); + assert_eq!( + state_store + .retry_budget_attempt_count(&issue.id) + .expect("retry budget count should succeed"), + expected_retry_budget, + "{case_name}" + ); + } +} + +#[test] +fn exited_child_cleanup_handles_worktree_mapping_ownership() { + { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Done", &[]); + let removed_worktree_path = temp_dir.path().join("removed-lane"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store.update_run_status("run-1", "succeeded").expect("run status should update"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &removed_worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + orchestrator::clear_orphaned_daemon_child_state( + &state_store, + ChildRunRef { + issue_id: &issue.id, + run_id: "run-1", + attempt_number: 1, + }, + false, + ) + .expect("removed worktree cleanup should succeed"); + + assert!( + state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none() + ); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_none() + ); + assert_eq!( + state_store + .run_attempt("run-1") + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "succeeded" + ); + } + { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Review", &[]); + let existing_worktree_path = temp_dir.path().join("retained-lane"); + + fs::create_dir_all(&existing_worktree_path).expect("worktree path should exist"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &existing_worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + orchestrator::clear_orphaned_daemon_child_state( + &state_store, + ChildRunRef { + issue_id: &issue.id, + run_id: "run-1", + attempt_number: 1, + }, + false, + ) + .expect("existing worktree cleanup should succeed"); + + assert_eq!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .expect("worktree mapping should remain") + .worktree_path(), + existing_worktree_path.as_path() + ); + } +} + +#[test] +fn exited_child_cleanup_requires_exact_run_id() { + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + + state_store + .record_run_attempt("other-run", &issue.id, 1, "running") + .expect("other run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "other-run", "In Progress") + .expect("lease should record"); + + orchestrator::clear_orphaned_daemon_child_state( + &state_store, + ChildRunRef { issue_id: &issue.id, run_id: "planned-run", attempt_number: 1 }, + true, + ) + .expect("orphaned child cleanup should succeed"); + + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("lease should remain attached to the other run") + .run_id(), + "other-run" + ); + assert_eq!( + state_store + .run_attempt("other-run") + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "running" + ); +} + +#[test] +fn exited_child_cleanup_keeps_other_run_lease_and_worktree_mapping() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let removed_worktree_path = temp_dir.path().join("removed-lane"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .record_run_attempt("other-run", &issue.id, 2, "running") + .expect("other run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "other-run", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &removed_worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + orchestrator::clear_orphaned_daemon_child_state( + &state_store, + ChildRunRef { issue_id: &issue.id, run_id: "run-1", attempt_number: 1 }, + false, + ) + .expect("orphaned child cleanup should succeed"); + + assert_eq!( + state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("lease should remain attached to the other run") + .run_id(), + "other-run" + ); + assert_eq!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .expect("worktree mapping should remain") + .worktree_path(), + removed_worktree_path.as_path() + ); +} diff --git a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs new file mode 100644 index 00000000..b1499faa --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs @@ -0,0 +1,147 @@ +#[test] +fn repo_gate_rejects_dirty_tracked_files_left_by_canonicalize_commands() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + + commit_worktree_change(repo_root, "tracked.txt", "before\n", "add tracked file"); + + let error = orchestrator::run_repo_gate_commands( + &[String::from("printf 'after\\n' > tracked.txt")], + &[String::from("grep -qx 'after' tracked.txt")], + repo_root, + ) + .expect_err("tracked autofix rewrites should fail the repo gate"); + let tracked_contents = fs::read_to_string(repo_root.join("tracked.txt")) + .expect("tracked file should remain readable"); + let tracked_status = git_output(repo_root, &["status", "--porcelain", "--untracked-files=no"]); + let repo_gate_failure = error + .downcast_ref::() + .expect("repo gate failures should preserve structured classification"); + + assert!(error.to_string().contains("verification")); + assert_eq!(repo_gate_failure.error_class(), "repo_gate_tracked_rewrites_left"); + assert_eq!( + repo_gate_failure.disposition(), + orchestrator::RepoGateFailureDisposition::ContinueRepair + ); + assert_eq!(tracked_contents, "after\n"); + assert!(tracked_status.contains("tracked.txt")); +} + +#[test] +fn repo_gate_cleanliness_check_spawn_failures_require_human_attention() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + let error = orchestrator::run_repo_gate_cleanliness_check_with_git( + OsStr::new("/definitely-missing-git-for-tests"), + repo_root, + ) + .expect_err("missing git binary should preserve repo gate classification"); + let repo_gate_failure = error + .downcast_ref::() + .expect("repo gate failures should preserve structured classification"); + + assert!(error.to_string().contains("tracked-file cleanliness check")); + assert_eq!(repo_gate_failure.error_class(), "repo_gate_command_spawn_failed"); + assert_eq!( + repo_gate_failure.disposition(), + orchestrator::RepoGateFailureDisposition::NeedsHumanAttention + ); +} + +#[test] +fn repo_gate_classifies_git_index_lock_contention_as_retryable_runtime_failure() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + let error = orchestrator::run_repo_gate_commands( + &[String::from( + "printf \"%s\\n\" \"fatal: Unable to create '.git/index.lock': File exists.\" >&2; exit 1", + )], + &[], + repo_root, + ) + .expect_err("git index.lock contention should fail the repo gate"); + let repo_gate_failure = error + .downcast_ref::() + .expect("repo gate failures should preserve structured classification"); + + assert_eq!(repo_gate_failure.error_class(), "repo_gate_git_lock_contention"); + assert_eq!( + repo_gate_failure.disposition(), + orchestrator::RepoGateFailureDisposition::RetryAfterBackoff + ); +} + +#[test] +fn repo_gate_selects_matching_profile_for_scoped_lane_changes() { + let (temp_dir, config, workflow) = + temp_project_layout_with_workflow_markdown(&profile_scoped_workflow_markdown("pubfi")); + let repo_root = config.repo_root(); + let remote_root = temp_dir.path().join("origin.git"); + + add_origin_remote(repo_root, &remote_root); + checkout_new_branch(repo_root, "config-subset"); + commit_worktree_change( + repo_root, + "config/new-surface.toml", + "name = \"new-surface\"\n", + "config subset change", + ); + + let selection = + orchestrator::select_repo_gate_for_worktree(workflow.frontmatter().execution(), repo_root); + + assert_eq!(selection.profile_name(), Some("config_subset")); + assert!(selection.canonicalize_commands().is_empty()); + assert_eq!(selection.verify_commands(), ["python3 -c 'print(\"ok\")'"]); +} + +#[test] +fn repo_gate_falls_back_to_full_gate_when_changed_file_classification_is_unavailable() { + let (_temp_dir, config, workflow) = + temp_project_layout_with_workflow_markdown(&profile_scoped_workflow_markdown("pubfi")); + let repo_root = config.repo_root(); + + checkout_new_branch(repo_root, "config-subset"); + commit_worktree_change( + repo_root, + "config/new-surface.toml", + "name = \"new-surface\"\n", + "config subset change", + ); + + let selection = + orchestrator::select_repo_gate_for_worktree(workflow.frontmatter().execution(), repo_root); + + assert_eq!(selection.profile_name(), None); + assert_eq!(selection.canonicalize_commands(), ["cargo make fmt", "cargo make lint"]); + assert_eq!(selection.verify_commands(), ["cargo make check"]); +} + +#[test] +fn repo_gate_shell_falls_back_to_non_login_posix_sh_for_missing_absolute_shell() { + let (shell, shell_flag) = orchestrator::repo_gate_shell_from_env(Some(OsString::from( + "/definitely-missing-shell-for-tests", + ))); + + assert_eq!(Path::new(&shell), Path::new("/bin/sh")); + assert_eq!(shell_flag, "-c"); +} + +#[test] +fn repo_gate_shell_uses_non_login_mode_when_shell_is_bin_sh() { + let (shell, shell_flag) = + orchestrator::repo_gate_shell_from_env(Some(OsString::from("/bin/sh"))); + + assert_eq!(Path::new(&shell), Path::new("/bin/sh")); + assert_eq!(shell_flag, "-c"); +} + +#[test] +fn repo_gate_shell_keeps_login_mode_for_other_configured_shells() { + let (shell, shell_flag) = + orchestrator::repo_gate_shell_from_env(Some(OsString::from("/bin/bash"))); + + assert_eq!(Path::new(&shell), Path::new("/bin/bash")); + assert_eq!(shell_flag, "-lc"); +} diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs new file mode 100644 index 00000000..0d0b5a9b --- /dev/null +++ b/apps/decodex/src/orchestrator/types.rs @@ -0,0 +1,1396 @@ +use crate::tracker; + +trait PullRequestReviewStateInspector { + fn inspect_review_state( + &self, + cwd: &Path, + pr_url: &str, + ) -> crate::prelude::Result; +} + +/// One bounded run invocation and its optional daemon-planned overrides. +pub(crate) struct RunOnceRequest<'a> { + pub(crate) config_path: Option<&'a Path>, + pub(crate) dry_run: bool, + pub(crate) preferred_issue_id: Option<&'a str>, + pub(crate) preferred_issue_state: Option<&'a str>, + pub(crate) preferred_initial_issue_state: Option<&'a str>, + pub(crate) preferred_lease_acquired: bool, + pub(crate) preferred_issue_claim_fd: Option, + pub(crate) preferred_dispatch_slot_fd: Option, + pub(crate) preferred_dispatch_slot_index: Option, + pub(crate) preferred_dispatch_mode: Option, + pub(crate) preferred_run_id: Option<&'a str>, + pub(crate) preferred_attempt_number: Option, + pub(crate) preferred_retry_budget_base: Option, + pub(crate) preferred_workflow_snapshot: Option<&'a str>, +} + +/// Multi-project local control-plane daemon request. +pub(crate) struct ServeRequest<'a> { + pub(crate) config_path: Option<&'a Path>, + pub(crate) poll_interval: Duration, + pub(crate) listen_address: &'a str, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RunSummary { + project_id: String, + issue_id: String, + issue_identifier: String, + issue_state: String, + initial_issue_state: String, + #[cfg(test)] + retry_project_slug: String, + dispatch_mode: IssueDispatchMode, + branch_name: String, + worktree_path: PathBuf, + attempt_number: i64, + run_id: String, + continuation_pending: bool, +} + +struct MaterializedDaemonSpawnState { + worktree: WorktreeSpec, + retry_budget_base: i64, +} + +#[derive(Clone, Debug)] +struct IssueRunPlan { + issue: TrackerIssue, + issue_state: String, + initial_issue_state: String, + worktree: WorktreeSpec, + #[allow(dead_code)] + #[cfg(test)] + retry_project_slug: String, + dispatch_mode: IssueDispatchMode, + attempt_number: i64, + run_id: String, + retry_budget_base: i64, +} + +#[derive(Default)] +struct RecoveredRuntimeState { + active_issues: Vec, +} + +#[derive(Clone, Copy)] +struct RunCycleRequest<'a> { + config_path: &'a Path, + state_store: &'a StateStore, + dry_run: bool, + preferred_issue_id: Option<&'a str>, + preferred_issue_state: Option<&'a str>, + preferred_initial_issue_state: Option<&'a str>, + preferred_lease_acquired: bool, + preferred_issue_claim_fd: Option, + preferred_dispatch_slot_fd: Option, + preferred_dispatch_slot_index: Option, + preferred_dispatch_mode: Option, + preferred_run_identity: Option>, + preferred_retry_budget_base: Option, + preferred_workflow_snapshot: Option<&'a str>, +} + +struct SpawnRunOnceChildRequest<'a> { + config_path: &'a Path, + preferred_issue_id: &'a str, + preferred_issue_state: &'a str, + preferred_initial_issue_state: Option<&'a str>, + dispatch_mode: IssueDispatchMode, + preferred_run_id: &'a str, + preferred_attempt_number: i64, + preferred_retry_budget_base: i64, + workflow: &'a WorkflowDocument, + issue_claim_handoff: Option<&'a File>, + dispatch_slot_handoff: Option<&'a File>, + dispatch_slot_index_handoff: Option, +} + +#[derive(Clone, Copy)] +struct PrepareIssueRunContext<'a, T> { + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, + worktree_manager: &'a WorktreeManager, + dry_run: bool, + lease_preacquired: bool, + dispatch_mode: IssueDispatchMode, + preferred_issue_state: Option<&'a str>, + preferred_initial_issue_state: Option<&'a str>, + preferred_run_identity: Option>, + preferred_retry_budget_base: Option, +} + +struct IssueTurnContinuationGuard<'a, T> { + tracker: &'a T, + tracker_tool_bridge: &'a TrackerToolBridge<'a>, + workflow: &'a WorkflowDocument, + service_id: &'a str, + issue_id: &'a str, + issue_identifier: &'a str, + initial_issue_state: &'a str, + #[allow(dead_code)] + #[cfg(test)] + retry_project_slug: &'a str, + dispatch_mode: IssueDispatchMode, + review_state_inspector: Option<&'a dyn PullRequestReviewStateInspector>, +} +impl IssueTurnContinuationGuard<'_, T> +where + T: IssueTracker, +{ + fn issue_has_service_ownership(&self, issue: &TrackerIssue) -> crate::prelude::Result { + tracker::issue_has_label_with_server_confirmation( + self.tracker, + issue, + &tracker::automation_active_label(self.service_id), + ) + } + + fn completed_closeout_pr_is_merged(&self) -> crate::prelude::Result { + let Some(review_state_inspector) = self.review_state_inspector else { + return Ok(false); + }; + let Some(review_context) = self.tracker_tool_bridge.review_context() else { + return Ok(false); + }; + let Some(pr_url) = review_context.recorded_pr_url.as_deref() else { + return Ok(false); + }; + + match retained_closeout_pr_merge_gate_with_inspector( + &review_context.cwd, + &review_context.branch_name, + pr_url, + review_state_inspector, + )? { + RetainedCloseoutPrMergeGate::Merged => Ok(true), + RetainedCloseoutPrMergeGate::NotMerged => Ok(false), + RetainedCloseoutPrMergeGate::PullRequestStateReadFailed => { + eyre::bail!( + "retained closeout PR state read failed while validating the continuation boundary" + ) + }, + } + } +} + +impl TurnContinuationGuard for IssueTurnContinuationGuard<'_, T> +where + T: IssueTracker, +{ + fn should_continue_turn(&self, _turn_count: u32) -> crate::prelude::Result { + let Some(issue) = refresh_issue(self.tracker, self.issue_id)? else { + return Ok(false); + }; + let tracker_policy = self.workflow.frontmatter().tracker(); + + if !self.issue_has_service_ownership(&issue)? { + return Ok(false); + } + if self.dispatch_mode == IssueDispatchMode::ReviewRepair { + return Ok(issue.state.name == tracker_policy.success_state() + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label())); + } + if self.dispatch_mode == IssueDispatchMode::Closeout { + let completed_state = tracker_policy.resolved_completed_state(); + + return Ok((issue.state.name == tracker_policy.success_state() + || (issue.state.name == completed_state + && self.completed_closeout_pr_is_merged()?)) + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label())); + } + + let issue_remains_active = issue.state.name == tracker_policy.in_progress_state() + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()); + + if issue_remains_active { + return Ok(true); + } + + let stale_startup_snapshot = + self.tracker_tool_bridge.startup_transition_succeeded_locally() + && issue.state.name == self.initial_issue_state + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()); + + Ok(stale_startup_snapshot) + } + + fn validate_continuation_boundary(&self, turn_count: u32) -> crate::prelude::Result<()> { + if self.dispatch_mode == IssueDispatchMode::ReviewRepair { + let Some(issue) = refresh_issue(self.tracker, self.issue_id)? else { + return Ok(()); + }; + let tracker_policy = self.workflow.frontmatter().tracker(); + + if issue.state.name == tracker_policy.success_state() + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()) + { + return Ok(()); + } + + eyre::bail!( + "Turn {} for issue `{}` ended without keeping the tracker issue in `{}`; a clean {} continuation boundary is only valid while the lane remains in its retained post-review state.", + turn_count, + self.issue_identifier, + tracker_policy.success_state(), + "retained review-repair", + ); + } + if self.dispatch_mode == IssueDispatchMode::Closeout { + let Some(issue) = refresh_issue(self.tracker, self.issue_id)? else { + return Ok(()); + }; + let tracker_policy = self.workflow.frontmatter().tracker(); + let completed_state = tracker_policy.resolved_completed_state(); + let issue_completed_with_merged_pr = issue.state.name == completed_state + && self.completed_closeout_pr_is_merged()?; + + if (issue.state.name == tracker_policy.success_state() + || issue_completed_with_merged_pr) + && !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()) + { + return Ok(()); + } + + let retained_states = + format!("`{}` or `{}`", tracker_policy.success_state(), completed_state); + + eyre::bail!( + "Turn {} for issue `{}` ended without keeping the tracker issue in {}; a clean retained closeout continuation boundary is only valid while the lane remains in its retained post-review state.", + turn_count, + self.issue_identifier, + retained_states, + ); + } + if turn_count != 1 { + return Ok(()); + } + if self.tracker_tool_bridge.startup_transition_succeeded_locally() { + let Some(issue) = refresh_issue(self.tracker, self.issue_id)? else { + return Ok(()); + }; + let tracker_policy = self.workflow.frontmatter().tracker(); + + if !issue.has_label(tracker_policy.opt_out_label()) + && !issue.has_label(tracker_policy.needs_attention_label()) + && (issue.state.name == tracker_policy.in_progress_state() + || issue.state.name == self.initial_issue_state) + { + return Ok(()); + } + } + + let Some(issue) = refresh_issue(self.tracker, self.issue_id)? else { + return Ok(()); + }; + let in_progress = self.workflow.frontmatter().tracker().in_progress_state(); + + if issue.state.name != in_progress { + eyre::bail!( + "Turn 1 for issue `{}` ended without moving the tracker issue to `{}`; a clean continuation boundary is only valid after the startup transition succeeds.", + self.issue_identifier, + in_progress + ); + } + + Ok(()) + } +} + +#[derive(Debug)] +struct ManualAttentionRequested { + issue_identifier: String, + label: String, + run_id: String, +} +impl Display for ManualAttentionRequested { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Run `{}` for issue `{}` requested human attention via label `{}`; stop automatic retries and hand off manually.", + self.run_id, self.issue_identifier, self.label + ) + } +} + +impl Error for ManualAttentionRequested {} + +#[derive(Debug)] +struct ReviewHandoffNeedsAttention { + issue_identifier: String, + run_id: String, +} +impl Display for ReviewHandoffNeedsAttention { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Run `{}` for issue `{}` partially applied review handoff writeback; stop retries and repair the issue manually.", + self.run_id, self.issue_identifier + ) + } +} + +impl Error for ReviewHandoffNeedsAttention {} + +#[derive(Debug)] +struct RetainedReviewNeedsAttention { + reason: String, +} +impl Display for RetainedReviewNeedsAttention { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Retained review orchestration requires operator attention: {}.", + self.reason + ) + } +} + +impl Error for RetainedReviewNeedsAttention {} + +#[derive(Debug)] +struct RetainedPartialProgress { + issue_identifier: String, + run_id: String, + worktree_path: String, +} +impl Display for RetainedPartialProgress { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Run `{}` for issue `{}` retained tracked worktree changes at `{}` after failing before terminal handoff; stop automatic retries and finish recovery manually.", + self.run_id, self.issue_identifier, self.worktree_path + ) + } +} + +impl Error for RetainedPartialProgress {} + +#[derive(Debug)] +struct AgentGitCredentialsUnavailable { + run_id: String, + token_env_var: String, +} +impl Display for AgentGitCredentialsUnavailable { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Run `{}` could not prepare noninteractive GitHub credentials from `{}`; stop automatic execution and repair the configured credential.", + self.run_id, self.token_env_var + ) + } +} + +impl Error for AgentGitCredentialsUnavailable {} + +#[derive(Debug)] +struct StalledRunNeedsAttention { + issue_identifier: String, + run_id: String, + idle_for: Duration, +} +impl Display for StalledRunNeedsAttention { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Run `{}` for issue `{}` stalled after {:?} without app-server activity; stop automatic execution and repair manually.", + self.run_id, self.issue_identifier, self.idle_for + ) + } +} + +impl Error for StalledRunNeedsAttention {} + +struct DaemonRunChild { + child: Child, + issue_id: String, + run_id: String, + attempt_number: i64, + initial_issue_state: String, + #[allow(dead_code)] + #[cfg(test)] + retry_project_slug: String, + dispatch_mode: IssueDispatchMode, + from_retry_queue: bool, + workflow: WorkflowDocument, +} + +#[derive(Clone, Copy)] +struct ChildRunRef<'a> { + issue_id: &'a str, + run_id: &'a str, + attempt_number: i64, +} + +#[derive(Clone, Copy)] +struct ActiveChildRunContext<'a> { + child: ChildRunRef<'a>, + workflow: &'a WorkflowDocument, + dispatch_mode: IssueDispatchMode, +} + +#[derive(Clone, Copy)] +struct PreferredRunIdentity<'a> { + run_id: &'a str, + attempt_number: i64, +} + +#[derive(Clone, Debug)] +struct RetryEntry { + issue_id: String, + #[allow(dead_code)] + #[cfg(test)] + retry_project_slug: String, + continuation_initial_issue_state: Option, + dispatch_mode: IssueDispatchMode, + kind: RetryKind, + attempt: u32, + ready_at: Instant, +} + +#[derive(Default)] +struct RetryQueue { + entries: HashMap, +} +impl RetryQueue { + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn upsert(&mut self, entry: RetryEntry) { + self.entries.insert(entry.issue_id.clone(), entry); + } + + fn release(&mut self, issue_id: &str) { + self.entries.remove(issue_id); + } + + fn next_entry(&self) -> Option<&RetryEntry> { + self.entries.values().min_by(|left, right| { + left.ready_at.cmp(&right.ready_at).then_with(|| left.issue_id.cmp(&right.issue_id)) + }) + } + + fn ordered_entries(&self) -> Vec { + let mut entries = self.entries.values().cloned().collect::>(); + + entries.sort_by(|left, right| { + left.ready_at.cmp(&right.ready_at).then_with(|| left.issue_id.cmp(&right.issue_id)) + }); + + entries + } +} + +struct DaemonTickContext { + config: ServiceConfig, + workflow: WorkflowDocument, + tracker: LinearClient, + worktree_manager: WorktreeManager, +} + +#[derive(Default)] +struct ProjectDaemonRuntime { + active_children: Vec, + retry_queue: RetryQueue, + tracker_backoff: Option, + workflow_cache: Option, +} + +#[derive(Clone, Debug)] +struct TrackerConnectorBackoff { + until: Instant, + reset_unix_epoch: i64, + reset_source: &'static str, + sync_phase: &'static str, +} + +struct OperatorStateEndpoint { + listen_address: SocketAddr, + snapshot: Arc>, + dashboard_events: DashboardEventHub, + shutdown_tx: Sender<()>, + activity_shutdown_tx: Sender<()>, + server_thread: Option>, + activity_thread: Option>, +} +impl OperatorStateEndpoint { + fn start( + listen_address: &str, + ready_stale_after: Duration, + state_store: Arc, + ) -> crate::prelude::Result { + let listener = TcpListener::bind(listen_address).map_err(|error| { + eyre::eyre!("Failed to bind operator state endpoint on `{listen_address}`: {error}") + })?; + let listen_address = listener.local_addr().map_err(|error| { + eyre::eyre!( + "Failed to resolve operator state endpoint address for `{listen_address}`: {error}" + ) + })?; + + listener + .set_nonblocking(true) + .map_err(|error| eyre::eyre!("Failed to configure operator state endpoint: {error}"))?; + + let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot::default())); + let dashboard_events = DashboardEventHub::default(); + let shared_snapshot = Arc::clone(&snapshot); + let server_dashboard_events = dashboard_events.clone(); + let server_state_store = Arc::clone(&state_store); + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let server_thread = thread::spawn(move || { + run_operator_state_endpoint( + listener, + shared_snapshot, + server_dashboard_events, + server_state_store, + ready_stale_after, + shutdown_rx, + ); + }); + let activity_dashboard_events = dashboard_events.clone(); + let (activity_shutdown_tx, activity_shutdown_rx) = mpsc::channel(); + let activity_thread = thread::spawn(move || { + run_operator_run_activity_websocket_broadcasts( + state_store, + activity_dashboard_events, + activity_shutdown_rx, + ); + }); + + Ok(Self { + listen_address, + snapshot, + dashboard_events, + shutdown_tx, + activity_shutdown_tx, + server_thread: Some(server_thread), + activity_thread: Some(activity_thread), + }) + } + + fn listen_address(&self) -> SocketAddr { + self.listen_address + } + + fn publish_snapshot(&self, snapshot: &OperatorStatusSnapshot) -> crate::prelude::Result<()> { + let snapshot_json = serde_json::to_vec(snapshot)?; + let snapshot_value = serde_json::to_value(snapshot)?; + let last_publish_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); + let mut guard = self + .snapshot + .lock() + .map_err(|error| eyre::eyre!("Operator state snapshot lock poisoned: {error}"))?; + + *guard = PublishedOperatorSnapshot { + snapshot_json: Some(snapshot_json), + last_publish_unix_epoch: Some(last_publish_unix_epoch), + }; + + drop(guard); + + self.dashboard_events.broadcast( + "snapshot", + json!({ + "snapshotPublishedAtUnixEpoch": last_publish_unix_epoch, + "snapshot": snapshot_value, + }), + ); + + Ok(()) + } +} + +impl Drop for OperatorStateEndpoint { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + let _ = self.activity_shutdown_tx.send(()); + + if let Some(server_thread) = self.server_thread.take() { + let _ = server_thread.join(); + } + if let Some(activity_thread) = self.activity_thread.take() { + let _ = activity_thread.join(); + } + } +} + +#[derive(Clone, Default)] +struct PublishedOperatorSnapshot { + snapshot_json: Option>, + last_publish_unix_epoch: Option, +} + +#[derive(Clone)] +struct CachedWorkflowDocument { + path: PathBuf, + document: WorkflowDocument, +} + +#[derive(Clone, Copy)] +struct ActiveWorkflowOverride<'a> { + child: ChildRunRef<'a>, + workflow: &'a WorkflowDocument, +} + +#[derive(Clone, Debug)] +struct ActiveRunReconciliation { + issue: TrackerIssue, + run_attempt: RunAttempt, + worktree_mapping: Option, + disposition: ActiveRunDisposition, + workflow: WorkflowDocument, +} + +struct TerminalFailureOutcome { + error_class: &'static str, + retry_guarded_by_state: bool, +} + +#[derive(Clone, Debug, Serialize)] +struct OperatorStatusSnapshot { + project_id: String, + run_limit: usize, + warnings: Vec, + connector_backoffs: Vec, + projects: Vec, + accounts: Vec, + active_runs: Vec, + recent_runs: Vec, + history_lanes: Vec, + queued_candidates: Vec, + worktrees: Vec, + post_review_lanes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorConnectorBackoffStatus { + project_id: String, + connector: String, + sync_phase: String, + quota_class: String, + reset_at: String, + reset_unix_epoch: i64, + reset_source: String, + retry_after_seconds: i64, + next_action: String, + warning: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorProjectStatus { + project_id: String, + config_path: String, + repo_root: String, + enabled: bool, + active_run_count: usize, + queued_candidate_count: usize, + post_review_lane_count: usize, + retained_worktree_count: usize, + waiting_lane_count: usize, + attention_count: usize, + connector_state: String, + last_activity_at: Option, + warning_count: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorHistoryLaneStatus { + project_id: String, + issue_id: String, + issue_identifier: Option, + title: Option, + issue_key: String, + attempt_count: usize, + ledger_outcome: OperatorHistoryLedgerOutcome, + latest_run: OperatorRunStatus, + attempts: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorHistoryLedgerOutcome { + ledger_status: String, + final_outcome: String, + final_event_type: Option, + final_event_at: Option, + summary: Option, + pr_url: Option, + commit_sha: Option, + branch: Option, + closeout_status: Option, + needs_attention_reason: Option, + lifecycle_started_at: Option, + lifecycle_finished_at: Option, + lifecycle_elapsed_seconds: Option, + record_count: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorRunStatus { + project_id: String, + run_id: String, + issue_id: String, + issue_identifier: Option, + title: Option, + attempt_number: i64, + status: String, + attempt_status: String, + phase: String, + wait_reason: Option, + current_operation: String, + thread_id: Option, + turn_id: Option, + thread_status: Option, + thread_active_flags: Vec, + interactive_requested: bool, + continuation_pending: bool, + active_lease: bool, + queue_lease_state: String, + execution_liveness: String, + updated_at: String, + last_run_activity_at: Option, + last_protocol_activity_at: Option, + last_progress_at: Option, + idle_for_seconds: Option, + protocol_idle_for_seconds: Option, + suspected_stall: bool, + last_event_type: Option, + last_event_at: Option, + event_count: i64, + process_id: Option, + process_alive: Option, + retry_kind: Option, + next_retry_at: Option, + effective_model: Option, + effective_model_provider: Option, + effective_cwd: Option, + effective_approval_policy: Option, + effective_approvals_reviewer: Option, + effective_sandbox_mode: Option, + child_agent_activity: Option, + protocol_activity: Option, + account: Option, + accounts: Vec, + branch_name: Option, + worktree_path: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorQueuedIssueStatus { + issue_id: String, + issue_identifier: String, + title: String, + state: String, + priority: Option, + created_at: String, + classification: String, + reason: String, + attention: Option, + blocker_identifiers: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorQueuedIssueAttentionStatus { + summary: String, + run_id: Option, + attempt_number: Option, + current_operation: Option, + thread_status: Option, + attempt_status: Option, + auto_retry_blocked_reason: Option, + attention_error_class: Option, + attention_next_action: Option, + retry_budget_attempt_count: Option, + retry_budget_max_attempts: i64, + last_activity_at: Option, + last_progress_at: Option, + last_event_type: Option, + event_count: i64, + process_alive: Option, + worktree_path: Option, + worktree_has_tracked_changes: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorWorktreeStatus { + issue_id: String, + issue_identifier: Option, + issue_state: Option, + branch_name: String, + worktree_path: String, + ownership: String, + ownership_reason: String, + hygiene: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorWorktreeHygieneStatus { + classification: String, + default_branch: String, + dirty: bool, + reason: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorPostReviewLaneStatus { + issue_id: String, + issue_identifier: String, + issue_state: String, + branch_name: String, + worktree_path: String, + classification: String, + reason: String, + pr_url: Option, + pr_state: Option, + review_decision: Option, + mergeable: Option, + check_state: Option, + unresolved_review_threads: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PostReviewLaneSnapshot { + issue: TrackerIssue, + worktree: WorktreeMapping, + review_handoff: Option, + local_branch_name: Option, + local_head_oid: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PullRequestReviewState { + url: String, + state: String, + is_draft: bool, + review_decision: Option, + merge_commit_allowed: bool, + pending_review_requests: usize, + mergeable: String, + merge_state_status: String, + head_ref_name: String, + head_ref_oid: String, + merge_commit_oid: Option, + head_repository_name: Option, + head_repository_owner: Option, + status_check_rollup_state: Option, + unresolved_review_threads: usize, + issue_description_external_review_thumbs_up_count: usize, + issue_comments: Vec, + reviews: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PostReviewLaneClassification { + decision: PostReviewLaneDecision, + reason: String, + pr_url: Option, + pr_state: Option, + review_decision: Option, + mergeable: Option, + check_state: Option, + unresolved_review_threads: Option, +} + +struct RetainedReviewLaneBlocked { + issue: TrackerIssue, + worktree: WorktreeMapping, + run_identity: RetainedReviewRunIdentity, + reason: String, +} + +struct RetainedReviewRunIdentity { + run_id: String, + attempt_number: i64, +} + +struct SelectedIssueRunCandidate { + issue: TrackerIssue, + dispatch_mode: IssueDispatchMode, + preferred_run_identity: Option, +} +impl SelectedIssueRunCandidate { + fn new(issue: TrackerIssue, dispatch_mode: IssueDispatchMode) -> Self { + Self { issue, dispatch_mode, preferred_run_identity: None } + } +} + +struct GhPullRequestReviewStateInspector { + github_token_env_var: Option, +} +impl PullRequestReviewStateInspector for GhPullRequestReviewStateInspector { + fn inspect_review_state( + &self, + cwd: &Path, + pr_url: &str, + ) -> crate::prelude::Result { + let github_token = resolve_configured_env_var( + "github.token_env_var", + self.github_token_env_var.as_deref(), + )?; + let locator = github::parse_pull_request_url(pr_url)?; + let mut review_threads_after: Option = None; + let mut review_state: Option = None; + let mut comments_after: Option = None; + + loop { + let repository = query_pull_request_review_state_page( + cwd, + &locator.owner, + &locator.repo, + locator.number, + review_threads_after.as_deref(), + pr_url, + github_token.as_str(), + )?; + let pull_request = repository.pull_request.as_ref().ok_or_else(|| { + eyre::eyre!("GitHub GraphQL response for `{pr_url}` did not include a pull request.") + })?; + let next_cursor = match &mut review_state { + Some(review_state) => + merge_pull_request_review_state_page(review_state, &repository, pull_request)?, + None => { + let next_cursor = next_pull_request_review_threads_cursor(pull_request)?; + + comments_after = + next_pull_request_issue_comments_cursor(&pull_request.comments, pr_url)?; + review_state = Some(pull_request_review_state_from_page(&repository, pull_request)); + + next_cursor + }, + }; + let Some(next_cursor) = next_cursor else { + break; + }; + + review_threads_after = Some(next_cursor); + } + + let mut review_state = review_state.ok_or_else(|| { + eyre::eyre!("GitHub GraphQL response for `{pr_url}` did not include a pull request.") + })?; + + while let Some(cursor) = comments_after.take() { + let pull_request = query_pull_request_issue_comments_page( + cwd, + &locator.owner, + &locator.repo, + locator.number, + &cursor, + pr_url, + github_token.as_str(), + )?; + + comments_after = merge_pull_request_issue_comment_page(&mut review_state, &pull_request)?; + } + + Ok(review_state) + } +} + +#[derive(Clone, Copy, Debug, Default)] +struct RetryIssueStateHint<'a> { + preferred_issue_state: Option<&'a str>, + preferred_initial_issue_state: Option<&'a str>, +} + +struct ChildExitRetryContext<'a, T> { + retry_queue: &'a mut RetryQueue, + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, +} + +#[derive(Clone, Copy)] +struct TargetIssueRunContext<'a, T> { + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, + issue_id: &'a str, + preferred_issue_state: Option<&'a str>, + preferred_initial_issue_state: Option<&'a str>, + dry_run: bool, + lease_preacquired: bool, + preferred_issue_claim_fd: Option, + preferred_dispatch_slot_fd: Option, + preferred_dispatch_slot_index: Option, + dispatch_mode: IssueDispatchMode, + preferred_run_identity: Option>, + preferred_retry_budget_base: Option, +} + +struct ConcurrencySnapshot { + total_active: usize, +} +impl ConcurrencySnapshot { + fn new(project_id: &str, state_store: &StateStore) -> crate::prelude::Result { + let leases = state_store.list_active_shared_leases(project_id)?; + + Ok(Self { total_active: leases.len() }) + } + + fn has_global_capacity(&self, execution: &WorkflowExecution) -> bool { + self.total_active < execution.max_concurrent_agents() as usize + } +} + +#[derive(Deserialize)] +struct PullRequestReviewStateResponse { + data: PullRequestReviewStateData, +} + +#[derive(Deserialize)] +struct PullRequestReviewStateData { + repository: Option, +} + +#[derive(Deserialize)] +struct PullRequestReviewStateRepository { + #[serde(rename = "mergeCommitAllowed")] + merge_commit_allowed: bool, + #[serde(rename = "pullRequest")] + pull_request: Option, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentsResponse { + data: PullRequestIssueCommentsData, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentsData { + repository: Option, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentsRepository { + #[serde(rename = "pullRequest")] + pull_request: Option, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentsNode { + url: String, + comments: PullRequestIssueCommentConnection, +} + +#[derive(Deserialize)] +struct PullRequestReviewStateNode { + url: String, + state: String, + #[serde(rename = "isDraft")] + is_draft: bool, + #[serde(rename = "reviewDecision")] + review_decision: Option, + #[serde(rename = "reviewRequests")] + review_requests: PullRequestReviewRequestConnection, + mergeable: String, + #[serde(rename = "mergeStateStatus")] + merge_state_status: String, + #[serde(rename = "headRefName")] + head_ref_name: String, + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "mergeCommit")] + merge_commit: Option, + #[serde(rename = "headRepository")] + head_repository: Option, + #[serde(rename = "headRepositoryOwner")] + head_repository_owner: Option, + #[serde(rename = "reactionGroups")] + reaction_groups: Vec, + comments: PullRequestIssueCommentConnection, + reviews: PullRequestReviewConnection, + #[serde(rename = "reviewThreads")] + review_threads: PullRequestReviewThreadConnection, + commits: PullRequestCommitConnection, +} + +#[derive(Deserialize)] +struct PullRequestRepositoryOwner { + login: String, +} + +#[derive(Deserialize)] +struct PullRequestMergeCommitNode { + oid: String, +} + +#[derive(Deserialize)] +struct PullRequestRepository { + name: String, +} + +#[derive(Deserialize)] +struct PullRequestReviewRequestConnection { + #[serde(rename = "totalCount")] + total_count: usize, +} + +#[derive(Deserialize)] +struct PullRequestReviewThreadConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PullRequestPageInfo, +} + +#[derive(Deserialize)] +struct PullRequestReviewThreadNode { + #[serde(rename = "isResolved")] + is_resolved: bool, + #[serde(rename = "isOutdated")] + is_outdated: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PullRequestIssueCommentState { + database_id: i64, + author_login: Option, + body: String, + created_at_unix_epoch: i64, + external_review_eyes_reaction_count: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PullRequestReviewSummaryState { + author_login: Option, + body: String, + state: String, + submitted_at_unix_epoch: i64, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PullRequestPageInfo, +} + +#[derive(Deserialize)] +struct PullRequestIssueCommentNode { + #[serde(rename = "databaseId")] + database_id: i64, + body: String, + #[serde(rename = "createdAt")] + created_at: String, + author: Option, + #[serde(rename = "reactionGroups")] + reaction_groups: Vec, +} + +#[derive(Deserialize)] +struct PullRequestReviewConnection { + nodes: Vec, +} + +#[derive(Deserialize)] +struct PullRequestReviewNode { + body: String, + state: String, + #[serde(rename = "submittedAt")] + submitted_at: Option, + author: Option, +} + +#[derive(Deserialize)] +struct PullRequestReactionGroup { + content: String, + users: PullRequestReactionUsersConnection, +} + +#[derive(Deserialize)] +struct PullRequestReactionUsersConnection { + nodes: Vec, +} + +#[derive(Deserialize)] +struct PullRequestActor { + login: String, +} + +#[derive(Deserialize)] +struct PullRequestPageInfo { + #[serde(rename = "hasNextPage")] + has_next_page: bool, + #[serde(rename = "endCursor")] + end_cursor: Option, +} + +#[derive(Deserialize)] +struct PullRequestCommitConnection { + nodes: Vec, +} + +#[derive(Deserialize)] +struct PullRequestCommitNode { + commit: PullRequestCommitPayload, +} + +#[derive(Deserialize)] +struct PullRequestCommitPayload { + #[serde(rename = "statusCheckRollup")] + status_check_rollup: Option, +} + +#[derive(Deserialize)] +struct PullRequestStatusCheckRollup { + state: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum IssueDispatchMode { + Normal, + Retry, + ReviewRepair, + Closeout, +} +impl IssueDispatchMode { + fn as_str(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::Retry => "retry", + Self::ReviewRepair => "review_repair", + Self::Closeout => "closeout", + } + } + + fn allows_issue( + self, + tracker: &dyn IssueTracker, + issue: &TrackerIssue, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + hint: RetryIssueStateHint<'_>, + ) -> crate::prelude::Result { + match self { + Self::Normal => { + let queue_label = tracker::automation_queue_label(project.service_id()); + + issue_passes_dispatch_policy(tracker, issue, workflow, &queue_label, false) + }, + Self::Retry => issue_passes_retry_dispatch_policy( + tracker, + issue, + project, + workflow, + state_store, + hint, + ), + Self::ReviewRepair => { + Ok(issue_passes_review_repair_dispatch_policy(tracker, issue, project, workflow)? + && !issue_retry_budget_exhausted(workflow, state_store, &issue.id)?) + }, + Self::Closeout => issue_passes_closeout_dispatch_policy( + tracker, + issue, + project, + workflow, + state_store, + ), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PostReviewLaneDecision { + Continue, + WaitForReview, + NeedsReviewRepair, + ReadyToLand, + CloseoutBlocked, + CleanupBlocked, + Block, +} +impl PostReviewLaneDecision { + fn as_str(self) -> &'static str { + match self { + Self::Continue => "continue", + Self::WaitForReview => "wait_for_review", + Self::NeedsReviewRepair => "needs_review_repair", + Self::ReadyToLand => "ready_to_land", + Self::CloseoutBlocked => "closeout_blocked", + Self::CleanupBlocked => "cleanup_blocked", + Self::Block => "blocked", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RetryKind { + Continuation, + Failure, +} + +pub(crate) enum RetryDispatchDecision { + Blocked { excluded_issue_ids: Vec }, + Dispatch(Box), + Continue, +} + +#[derive(Clone, Debug)] +pub(crate) enum ActiveRunDisposition { + RetainedReviewComplete, + Terminal, + NonActive, + Stalled { idle_for: Duration }, +} + +enum RetainedReviewLaneLoad { + Skip, + Ready(Box), + Blocked(Box), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ReviewOrchestrationPhase { + RequestPending, + WaitingForAck, + WaitingForResult, + RepairRequired, + PassWaitingForGates, + WaitingForMerge, +} +impl ReviewOrchestrationPhase { + fn as_str(self) -> &'static str { + match self { + Self::RequestPending => "request_pending", + Self::WaitingForAck => "waiting_for_ack", + Self::WaitingForResult => "waiting_for_result", + Self::RepairRequired => "repair_required", + Self::PassWaitingForGates => "pass_waiting_for_gates", + Self::WaitingForMerge => "waiting_for_merge", + } + } + + fn parse(value: &str) -> std::result::Result { + match value { + "request_pending" => Ok(Self::RequestPending), + "waiting_for_ack" => Ok(Self::WaitingForAck), + "waiting_for_result" => Ok(Self::WaitingForResult), + "repair_required" => Ok(Self::RepairRequired), + "pass_waiting_for_gates" => Ok(Self::PassWaitingForGates), + "waiting_for_merge" => Ok(Self::WaitingForMerge), + other => Err(format!( + "Unknown review orchestration phase `{other}` in retained review marker." + )), + } + } +} + +enum PostReviewLaneStateLoad { + Classification(PostReviewLaneClassification), + ReviewState(PullRequestReviewState), +} diff --git a/apps/decodex/src/pull_request.rs b/apps/decodex/src/pull_request.rs new file mode 100644 index 00000000..ae9a72ae --- /dev/null +++ b/apps/decodex/src/pull_request.rs @@ -0,0 +1,218 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PullRequestLandingState { + pub(crate) url: String, + pub(crate) state: String, + pub(crate) is_draft: bool, + pub(crate) review_decision: Option, + pub(crate) base_ref_name: String, + pub(crate) pending_review_requests: usize, + pub(crate) mergeable: String, + pub(crate) merge_state_status: String, + pub(crate) head_ref_name: String, + pub(crate) head_ref_oid: String, + pub(crate) status_check_rollup_state: Option, + pub(crate) unresolved_review_threads: usize, +} +impl PullRequestLandingState { + pub(crate) fn gate_view(&self) -> PullRequestLandingGateView<'_> { + PullRequestLandingGateView { + state: self.state.as_str(), + is_draft: self.is_draft, + review_decision: self.review_decision.as_deref(), + pending_review_requests: self.pending_review_requests, + mergeable: self.mergeable.as_str(), + merge_state_status: self.merge_state_status.as_str(), + status_check_rollup_state: self.status_check_rollup_state.as_deref(), + unresolved_review_threads: self.unresolved_review_threads, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct PullRequestLandingGateView<'a> { + pub(crate) state: &'a str, + pub(crate) is_draft: bool, + pub(crate) review_decision: Option<&'a str>, + pub(crate) pending_review_requests: usize, + pub(crate) mergeable: &'a str, + pub(crate) merge_state_status: &'a str, + pub(crate) status_check_rollup_state: Option<&'a str>, + pub(crate) unresolved_review_threads: usize, +} + +pub(crate) fn manual_landing_gates_satisfied(view: PullRequestLandingGateView<'_>) -> bool { + view.state == "OPEN" + && !view.is_draft + && view.pending_review_requests == 0 + && view.unresolved_review_threads == 0 + && view.review_decision != Some("CHANGES_REQUESTED") + && view.mergeable == "MERGEABLE" + && merge_state_allows_ready_to_land(view.merge_state_status) + && !checks_require_wait(view.status_check_rollup_state) + && !failed_checks_require_repair(view.status_check_rollup_state, view.merge_state_status) + && merge_state_requires_review_repair(view.mergeable, view.merge_state_status).is_none() +} + +pub(crate) fn retained_landing_gates_satisfied(view: PullRequestLandingGateView<'_>) -> bool { + view.state == "OPEN" + && !view.is_draft + && view.review_decision != Some("CHANGES_REQUESTED") + && view.mergeable == "MERGEABLE" + && merge_state_allows_ready_to_land(view.merge_state_status) + && !checks_require_wait(view.status_check_rollup_state) + && !failed_checks_require_repair(view.status_check_rollup_state, view.merge_state_status) + && merge_state_requires_review_repair(view.mergeable, view.merge_state_status).is_none() +} + +pub(crate) fn retained_clean_path_landing_gates_satisfied( + view: PullRequestLandingGateView<'_>, +) -> bool { + retained_landing_gates_satisfied(view) + && view.merge_state_status == "CLEAN" + && matches!(view.status_check_rollup_state, None | Some("SUCCESS")) +} + +pub(crate) fn retained_landing_requires_agent_fallback( + view: PullRequestLandingGateView<'_>, +) -> bool { + let review_and_check_gates_ready = view.state == "OPEN" + && !view.is_draft + && view.review_decision != Some("CHANGES_REQUESTED") + && view.unresolved_review_threads == 0 + && !checks_require_wait(view.status_check_rollup_state) + && !failed_checks_require_repair(view.status_check_rollup_state, view.merge_state_status); + + review_and_check_gates_ready + && ((retained_landing_gates_satisfied(view) + && !retained_clean_path_landing_gates_satisfied(view)) + || view.mergeable == "UNKNOWN" + || view.merge_state_status == "UNKNOWN") +} + +pub(crate) fn merge_state_allows_ready_to_land(merge_state_status: &str) -> bool { + matches!(merge_state_status, "CLEAN" | "HAS_HOOKS" | "UNSTABLE") +} + +pub(crate) fn checks_require_wait(check_state: Option<&str>) -> bool { + matches!(check_state, Some("EXPECTED" | "PENDING")) +} + +pub(crate) fn failed_checks_require_repair( + check_state: Option<&str>, + merge_state_status: &str, +) -> bool { + matches!(check_state, Some("ERROR" | "FAILURE")) && merge_state_status == "BLOCKED" +} + +pub(crate) fn merge_state_requires_review_repair( + mergeable: &str, + merge_state_status: &str, +) -> Option<&'static str> { + if mergeable == "CONFLICTING" { + return Some("pull_request_merge_conflict"); + } + if merge_state_status == "BEHIND" { + return Some("pull_request_branch_behind_base"); + } + + None +} + +#[cfg(test)] +mod tests { + use crate::pull_request::{self, PullRequestLandingGateView}; + + fn sample_gate_view() -> PullRequestLandingGateView<'static> { + PullRequestLandingGateView { + state: "OPEN", + is_draft: false, + review_decision: Some("APPROVED"), + pending_review_requests: 0, + mergeable: "MERGEABLE", + merge_state_status: "CLEAN", + status_check_rollup_state: Some("SUCCESS"), + unresolved_review_threads: 0, + } + } + + #[test] + fn landing_gates_handle_green_pending_and_review_request_cases() { + assert!(pull_request::manual_landing_gates_satisfied(sample_gate_view())); + + let mut view = sample_gate_view(); + + view.status_check_rollup_state = Some("PENDING"); + + assert!(!pull_request::manual_landing_gates_satisfied(view)); + assert!(pull_request::checks_require_wait(Some("PENDING"))); + + let mut view = sample_gate_view(); + + view.pending_review_requests = 2; + + assert!(pull_request::retained_landing_gates_satisfied(view)); + assert!(!pull_request::manual_landing_gates_satisfied(view)); + } + + #[test] + fn clean_path_landing_gates_only_allow_current_green_prs() { + assert!(pull_request::retained_clean_path_landing_gates_satisfied(sample_gate_view())); + + let mut view = sample_gate_view(); + + view.merge_state_status = "HAS_HOOKS"; + + assert!(pull_request::retained_landing_gates_satisfied(view)); + assert!(pull_request::retained_landing_requires_agent_fallback(view)); + assert!(!pull_request::retained_clean_path_landing_gates_satisfied(view)); + + let mut view = sample_gate_view(); + + view.merge_state_status = "UNSTABLE"; + view.status_check_rollup_state = Some("FAILURE"); + + assert!(pull_request::retained_landing_gates_satisfied(view)); + assert!(pull_request::retained_landing_requires_agent_fallback(view)); + assert!(!pull_request::retained_clean_path_landing_gates_satisfied(view)); + + let mut view = sample_gate_view(); + + view.mergeable = "UNKNOWN"; + + assert!(!pull_request::retained_landing_gates_satisfied(view)); + assert!(pull_request::retained_landing_requires_agent_fallback(view)); + + let mut view = sample_gate_view(); + + view.merge_state_status = "UNKNOWN"; + + assert!(!pull_request::retained_landing_gates_satisfied(view)); + assert!(pull_request::retained_landing_requires_agent_fallback(view)); + } + + #[test] + fn landing_gates_route_conflicts_and_branch_lag_to_repair() { + assert_eq!( + pull_request::merge_state_requires_review_repair("CONFLICTING", "CLEAN"), + Some("pull_request_merge_conflict") + ); + assert_eq!( + pull_request::merge_state_requires_review_repair("MERGEABLE", "BEHIND"), + Some("pull_request_branch_behind_base") + ); + } + + #[test] + fn merge_state_allows_ready_to_land_matches_existing_runtime_policy() { + assert!(pull_request::merge_state_allows_ready_to_land("CLEAN")); + assert!(pull_request::merge_state_allows_ready_to_land("HAS_HOOKS")); + assert!(pull_request::merge_state_allows_ready_to_land("UNSTABLE")); + assert!(!pull_request::merge_state_allows_ready_to_land("BLOCKED")); + } + + #[test] + fn failed_checks_require_repair_only_for_blocked_red_checks() { + assert!(pull_request::failed_checks_require_repair(Some("FAILURE"), "BLOCKED")); + assert!(!pull_request::failed_checks_require_repair(Some("FAILURE"), "CLEAN")); + } +} diff --git a/apps/decodex/src/runtime.rs b/apps/decodex/src/runtime.rs new file mode 100644 index 00000000..c796ae14 --- /dev/null +++ b/apps/decodex/src/runtime.rs @@ -0,0 +1,314 @@ +//! Local Decodex control-plane runtime paths and project registry helpers. + +use std::{ + cmp::Reverse, + env, fs, + path::{Path, PathBuf}, +}; + +use crate::{ + config::ServiceConfig, + prelude::{Result, eyre}, + state::{ProjectRegistration, StateStore}, +}; + +/// Resolve Decodex's local application state directory under the Codex home. +pub(crate) fn decodex_home_dir() -> Result { + let Some(home) = env::var_os("HOME") else { + eyre::bail!("Failed to resolve `$HOME` for the local Decodex runtime directory."); + }; + + Ok(decodex_home_dir_from(PathBuf::from(home))) +} + +/// Resolve the global operator config path. +pub(crate) fn global_config_path() -> Result { + Ok(decodex_home_dir()?.join("config.toml")) +} + +/// Resolve the directory that stores project contract directories managed outside repos. +pub(crate) fn project_config_dir() -> Result { + Ok(decodex_home_dir()?.join("projects")) +} + +/// Resolve Decodex's log directory. +pub(crate) fn log_dir() -> Result { + Ok(decodex_home_dir()?.join("logs")) +} + +/// Resolve the global single-machine runtime database path. +pub(crate) fn runtime_db_path() -> Result { + Ok(decodex_home_dir()?.join("runtime.sqlite3")) +} + +/// Open the global single-machine runtime database. +pub(crate) fn open_runtime_store() -> Result { + StateStore::open(runtime_db_path()?) +} + +/// Register or refresh one project config in the global runtime DB. +pub(crate) fn register_project_config( + state_store: &StateStore, + config_path: &Path, + enabled: bool, +) -> Result { + let config_path = ServiceConfig::resolve_project_config_path(config_path)?; + let config_path = fs::canonicalize(config_path)?; + let config = ServiceConfig::from_path(&config_path)?; + let registration = ProjectRegistration::from_config( + config.service_id(), + &config_path, + &config, + enabled, + &config_fingerprint(&config_path, config.workflow_path())?, + ); + + state_store.upsert_project(®istration)?; + + Ok(registration) +} + +/// Resolve the registered project config that owns a local working directory. +pub(crate) fn registered_config_path_for_cwd( + state_store: &StateStore, + cwd: &Path, +) -> Result> { + let cwd = fs::canonicalize(cwd)?; + let mut matches = Vec::new(); + + for project in state_store.list_projects()? { + let repo_root = fs::canonicalize(project.repo_root()) + .unwrap_or_else(|_| project.repo_root().to_path_buf()); + let worktree_root = fs::canonicalize(project.worktree_root()) + .unwrap_or_else(|_| project.worktree_root().to_path_buf()); + let matched_root = if cwd.starts_with(&worktree_root) { + Some(worktree_root) + } else if cwd.starts_with(&repo_root) { + Some(repo_root) + } else { + None + }; + + if let Some(matched_root) = matched_root { + matches.push((matched_root.components().count(), project)); + } + } + + matches.sort_by_key(|item| Reverse(item.0)); + + let Some((best_score, best_project)) = matches.first() else { + return Ok(None); + }; + let ambiguous = matches.iter().skip(1).any(|(score, project)| { + score == best_score && project.service_id() != best_project.service_id() + }); + + if ambiguous { + eyre::bail!( + "Current directory `{}` matches multiple registered Decodex projects; pass `--config `.", + cwd.display() + ); + } + + Ok(Some(best_project.config_path().to_path_buf())) +} + +fn decodex_home_dir_from(home: PathBuf) -> PathBuf { + home.join(".codex").join("decodex") +} + +fn config_fingerprint(config_path: &Path, workflow_path: &Path) -> Result { + let config_body = fs::read(config_path)?; + let workflow_body = fs::read(workflow_path)?; + let mut hash = 0xcbf29ce484222325_u64; + + for byte in config_path + .to_string_lossy() + .bytes() + .chain(config_body) + .chain(workflow_path.to_string_lossy().bytes()) + .chain(workflow_body) + { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x100000001b3); + } + + Ok(format!("{hash:016x}")) +} + +#[cfg(test)] +mod tests { + use std::{ + env, + ffi::OsString, + fs, + path::{Path, PathBuf}, + }; + + use tempfile::TempDir; + + use crate::{runtime, state::StateStore}; + + struct EnvVarGuard { + key: &'static str, + previous: Option, + } + + impl EnvVarGuard { + fn set(key: &'static str, value: &Path) -> Self { + let previous = env::var_os(key); + + unsafe { + env::set_var(key, value); + } + + Self { key, previous } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(previous) => unsafe { env::set_var(self.key, previous) }, + None => unsafe { env::remove_var(self.key) }, + } + } + } + + #[test] + fn runtime_paths_live_under_codex_decodex_home() { + let home = PathBuf::from("/tmp/decodex-home-test"); + + assert_eq!( + runtime::decodex_home_dir_from(home), + PathBuf::from("/tmp/decodex-home-test/.codex/decodex") + ); + } + + #[test] + fn project_config_registration_requires_explicit_repo_root() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let _home_guard = EnvVarGuard::set("HOME", temp_dir.path()); + let state_store = StateStore::open(temp_dir.path().join("runtime.sqlite3")) + .expect("state store should open"); + let config_dir = + runtime::project_config_dir().expect("project config dir should resolve").join("pubfi"); + let config_path = config_dir.join("project.toml"); + + fs::create_dir_all(&config_dir).expect("project config dir should exist"); + + write_workflow(&config_dir); + write_config_without_repo_root(&config_path); + + let error = runtime::register_project_config(&state_store, &config_dir, true) + .expect_err("centralized project config without repo_root should fail"); + + assert!( + error.to_string().contains("paths.repo_root"), + "error should explain the missing explicit repo root: {error:?}" + ); + } + + #[test] + fn registered_config_path_for_cwd_matches_repo_and_worktree_roots() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let repo_root = temp_dir.path().join("target-repo"); + let worktree_root = repo_root.join(".worktrees"); + let lane_root = worktree_root.join("XY-380"); + let state_store = StateStore::open(temp_dir.path().join("runtime.sqlite3")) + .expect("state store should open"); + let config_dir = temp_dir.path().join("projects/pubfi"); + let config_path = config_dir.join("project.toml"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + fs::create_dir_all(&lane_root).expect("lane root should exist"); + fs::create_dir_all(&config_dir).expect("project config dir should exist"); + + write_workflow(&config_dir); + write_config_body(&config_path, &repo_root); + + let registration = runtime::register_project_config(&state_store, &config_dir, true) + .expect("project config should register"); + let canonical_config = fs::canonicalize(&config_path).expect("config should canonicalize"); + + assert_eq!(registration.config_path(), canonical_config.as_path()); + assert_eq!( + runtime::registered_config_path_for_cwd(&state_store, &repo_root) + .expect("repo cwd lookup should succeed"), + Some(canonical_config.clone()) + ); + assert_eq!( + runtime::registered_config_path_for_cwd(&state_store, &lane_root) + .expect("worktree cwd lookup should succeed"), + Some(canonical_config) + ); + } + + fn write_config_body(config_path: &Path, repo_root: &Path) { + fs::write( + config_path, + format!( + r#" +service_id = "pubfi" + +[tracker] +api_key_env_var = "HOME" + +[github] +token_env_var = "PATH" + +[paths] +repo_root = "{}" +"#, + repo_root.display() + ), + ) + .expect("config should write"); + } + + fn write_workflow(config_dir: &Path) { + fs::write( + config_dir.join("WORKFLOW.md"), + r#" ++++ +version = 1 +max_turns = 1 + +[tracker] +queued_state = "Todo" +in_progress_state = "In Progress" +success_state = "Done" +terminal_states = ["Done", "Canceled"] + +[tools] +comment = "issue_comment" +transition = "issue_transition" +label = "issue_label" +progress_checkpoint = "issue_progress_checkpoint" +review_checkpoint = "issue_review_checkpoint" +review_handoff = "issue_review_handoff" +terminal_finalize = "issue_terminal_finalize" ++++ + +Follow the project policy. +"#, + ) + .expect("workflow should write"); + } + + fn write_config_without_repo_root(config_path: &Path) { + fs::write( + config_path, + r#" +service_id = "pubfi" + +[tracker] +api_key_env_var = "HOME" + +[github] +token_env_var = "PATH" +"#, + ) + .expect("config should write"); + } +} diff --git a/apps/decodex/src/state.rs b/apps/decodex/src/state.rs new file mode 100644 index 00000000..c034359e --- /dev/null +++ b/apps/decodex/src/state.rs @@ -0,0 +1,44 @@ +//! Persistent single-machine runtime state for active Decodex execution. + +#[cfg(unix)] use std::os::fd::{AsRawFd, FromRawFd}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + fs::{self, File, OpenOptions, TryLockError}, + io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + process, + sync::{Mutex, MutexGuard}, + time::Duration, +}; + +use rusqlite::{Connection, Transaction, params}; +use serde::{Deserialize, Serialize}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + config::ServiceConfig, + prelude::{Result, eyre}, + tracker::records::{self, LinearExecutionEventRecord}, +}; + +include!("state/store.rs"); + +include!("state/models.rs"); + +include!("state/internal.rs"); + +pub(crate) const RUN_ACTIVITY_MARKER_FILE: &str = ".decodex-run-activity"; +pub(crate) const RUN_OPERATION_IDLE: &str = "idle"; +pub(crate) const RUN_OPERATION_GIT_CREDENTIALS: &str = "git_credentials"; +pub(crate) const RUN_OPERATION_APP_SERVER_PREFLIGHT: &str = "app_server_preflight"; +pub(crate) const RUN_OPERATION_AGENT_RUN: &str = "agent_run"; +pub(crate) const RUN_OPERATION_REPO_GATE: &str = "repo_gate"; +pub(crate) const RUN_OPERATION_REVIEW_WRITEBACK: &str = "review_writeback"; +pub(crate) const RUN_OPERATION_WAITING_EXTERNAL: &str = "waiting_external"; +pub(crate) const RUN_OPERATION_RECONCILIATION: &str = "reconciliation"; + +const DISPATCH_SLOT_LOCK_FILE_PREFIX: &str = ".decodex-dispatch-slot"; +const ISSUE_CLAIM_LOCK_FILE_PREFIX: &str = ".decodex-issue-claim"; + +#[cfg(test)] mod tests; diff --git a/apps/decodex/src/state/internal.rs b/apps/decodex/src/state/internal.rs new file mode 100644 index 00000000..1566e689 --- /dev/null +++ b/apps/decodex/src/state/internal.rs @@ -0,0 +1,2034 @@ +use libc::FD_CLOEXEC; +use libc::F_GETFD; +use libc::F_SETFD; + +pub(crate) struct EffectiveRuntimeMarker<'a> { + pub(crate) thread_id: Option<&'a str>, + pub(crate) turn_id: Option<&'a str>, + pub(crate) effective_model: &'a str, + pub(crate) effective_model_provider: &'a str, + pub(crate) effective_cwd: &'a str, + pub(crate) effective_approval_policy: &'a str, + pub(crate) effective_approvals_reviewer: &'a str, + pub(crate) effective_sandbox_mode: &'a str, +} + +pub(crate) struct ProtocolActivityMarker<'a> { + pub(crate) run_id: &'a str, + pub(crate) attempt_number: i64, + pub(crate) thread_id: Option<&'a str>, + pub(crate) turn_id: Option<&'a str>, + pub(crate) event_count: i64, + pub(crate) last_event_type: &'a str, + pub(crate) child_agent_activity: Option<&'a ChildAgentActivitySummary>, + pub(crate) protocol_activity: Option<&'a ProtocolActivitySummary>, +} + +pub(crate) struct CodexAccountMarker<'a> { + pub(crate) run_id: &'a str, + pub(crate) attempt_number: i64, + pub(crate) account: &'a CodexAccountActivitySummary, + pub(crate) accounts: &'a [CodexAccountActivitySummary], +} + +#[derive(Clone)] +struct DispatchSlotConfig { + root: PathBuf, + slot_limit: usize, +} + +struct IssueClaimGuard { + lock_file: File, + retention: GuardRetention, +} +impl IssueClaimGuard { + fn unlock(self) -> Result<()> { + self.lock_file.unlock()?; + + Ok(()) + } + + fn release_for_clear(self) -> Result<()> { + match self.retention { + GuardRetention::ParentAfterHandoff => Ok(()), + GuardRetention::Local | GuardRetention::AdoptingChild => self.unlock(), + } + } +} + +struct DispatchSlotGuard { + project_id: String, + slot_index: usize, + lock_file: File, + retention: GuardRetention, +} +impl DispatchSlotGuard { + fn release_for_clear(self) -> Result<()> { + match self.retention { + GuardRetention::ParentAfterHandoff => Ok(()), + GuardRetention::Local | GuardRetention::AdoptingChild => { + self.lock_file.unlock()?; + + Ok(()) + }, + } + } +} + +#[derive(Default)] +struct StateData { + projects: HashMap, + leases: HashMap, + run_attempts: HashMap, + events: HashMap>, + event_summaries: HashMap, + worktrees: HashMap, + linear_execution_events: HashMap, + review_handoffs: HashMap, + review_orchestrations: HashMap, + dispatch_slot_configs: HashMap, + issue_claim_guards: HashMap, + dispatch_slot_guards: HashMap, +} +impl StateData { + fn replace_durable_state(&mut self, loaded: Self) { + self.projects = loaded.projects; + self.leases = loaded.leases; + self.run_attempts = loaded.run_attempts; + self.events = loaded.events; + self.event_summaries = loaded.event_summaries; + self.worktrees = loaded.worktrees; + self.linear_execution_events = loaded.linear_execution_events; + self.review_handoffs = loaded.review_handoffs; + self.review_orchestrations = loaded.review_orchestrations; + } + + fn project_run_status( + &self, + project_id: &str, + attempt: &RunAttemptRecord, + ) -> Option { + let worktree = self.worktrees.get(&attempt.issue_id); + let active_lease = self + .leases + .get(&attempt.issue_id) + .is_some_and(|lease| lease.project_id == project_id && lease.run_id == attempt.run_id); + let remembered_project = attempt.project_id.as_deref() == Some(project_id); + let in_project = + remembered_project + || worktree.is_some_and(|mapping| mapping.project_id == project_id) + || active_lease; + + if !in_project { + return None; + } + + let event_summary = self.protocol_event_summary(&attempt.run_id); + + Some(ProjectRunStatus { + run_id: attempt.run_id.clone(), + issue_id: attempt.issue_id.clone(), + attempt_number: attempt.attempt_number, + status: attempt.status.clone(), + thread_id: attempt.thread_id.clone(), + turn_id: attempt.turn_id.clone(), + updated_at: attempt.updated_at.clone(), + branch_name: worktree.map(|mapping| mapping.branch_name.clone()), + worktree_path: worktree.map(|mapping| mapping.worktree_path.clone()), + active_lease, + event_count: event_summary.event_count, + last_event_type: event_summary.last_event_type, + last_event_at: event_summary.last_event_at, + }) + } + + fn protocol_event_summary(&self, run_id: &str) -> ProtocolEventSummaryRecord { + self.event_summaries + .get(run_id) + .cloned() + .or_else(|| self.events.get(run_id).map(|events| protocol_event_summary_from_events(events))) + .unwrap_or_default() + } + + fn project_id_for_run(&self, issue_id: &str, run_id: &str) -> Option { + self.leases + .get(issue_id) + .filter(|lease| lease.run_id == run_id) + .map(|lease| lease.project_id.clone()) + .or_else(|| self.worktrees.get(issue_id).map(|mapping| mapping.project_id.clone())) + } + + fn remember_run_project(&mut self, project_id: &str, issue_id: &str, run_id: Option<&str>) { + for attempt in self + .run_attempts + .values_mut() + .filter(|attempt| attempt.issue_id == issue_id) + .filter(|attempt| run_id.is_none_or(|run_id| attempt.run_id == run_id)) + { + attempt.project_id = Some(project_id.to_owned()); + } + } +} + +struct SqliteStateStore { + connection: Connection, +} +impl SqliteStateStore { + fn open(path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let connection = Connection::open(path)?; + + connection.busy_timeout(Duration::from_secs(5))?; + + let store = Self { connection }; + + store.bootstrap_schema()?; + + Ok(store) + } + + fn bootstrap_schema(&self) -> Result<()> { + self.connection.execute_batch( + r#" +PRAGMA foreign_keys = ON; +PRAGMA journal_mode = WAL; +CREATE TABLE IF NOT EXISTS projects ( + service_id TEXT PRIMARY KEY NOT NULL, + config_path TEXT NOT NULL, + repo_root TEXT NOT NULL, + worktree_root TEXT NOT NULL, + workflow_path TEXT NOT NULL, + tracker_api_key_env_var TEXT NOT NULL, + github_token_env_var TEXT NOT NULL, + enabled INTEGER NOT NULL, + config_fingerprint TEXT NOT NULL, + updated_at TEXT NOT NULL, + updated_at_unix INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS leases ( + issue_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL, + run_id TEXT NOT NULL, + issue_state TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS run_attempts ( + run_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT, + issue_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + status TEXT NOT NULL, + thread_id TEXT, + turn_id TEXT, + updated_at TEXT NOT NULL, + updated_at_unix INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS protocol_events ( + run_id TEXT NOT NULL, + sequence_number INTEGER NOT NULL, + event_type TEXT NOT NULL, + created_at TEXT NOT NULL, + created_at_unix INTEGER NOT NULL, + PRIMARY KEY (run_id, sequence_number) +); +CREATE TABLE IF NOT EXISTS worktrees ( + issue_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + worktree_path TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS linear_execution_events ( + idempotency_key TEXT PRIMARY KEY NOT NULL, + service_id TEXT NOT NULL, + issue_id TEXT NOT NULL, + event_type TEXT NOT NULL, + event_timestamp TEXT NOT NULL, + event_unix INTEGER, + payload_json TEXT NOT NULL, + recorded_at TEXT NOT NULL, + recorded_at_unix INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS linear_execution_events_issue_idx +ON linear_execution_events (service_id, issue_id, event_unix, recorded_at_unix); +CREATE TABLE IF NOT EXISTS review_handoffs ( + project_id TEXT NOT NULL, + issue_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + run_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + pr_url TEXT NOT NULL, + target_base_ref_name TEXT, + pr_head_ref_name TEXT NOT NULL, + pr_head_oid TEXT NOT NULL, + updated_at TEXT NOT NULL, + updated_at_unix INTEGER NOT NULL, + PRIMARY KEY (project_id, issue_id, branch_name) +); +CREATE TABLE IF NOT EXISTS review_orchestrations ( + project_id TEXT NOT NULL, + issue_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + run_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + pr_url TEXT NOT NULL, + head_sha TEXT NOT NULL, + phase TEXT NOT NULL, + request_comment_database_id INTEGER, + request_created_at_unix_epoch INTEGER, + request_description_thumbs_up_count INTEGER, + request_retry_count INTEGER NOT NULL, + external_round_count INTEGER NOT NULL, + auto_merge_enabled_at_unix_epoch INTEGER, + updated_at TEXT NOT NULL, + updated_at_unix INTEGER NOT NULL, + PRIMARY KEY (project_id, issue_id, branch_name, run_id, attempt_number) +); +CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +); +INSERT INTO schema_meta (key, value) +VALUES ('schema_version', '3') +ON CONFLICT(key) DO UPDATE SET value = excluded.value; +"#, + )?; + + Ok(()) + } + + fn load_state(&self) -> Result { + let mut state = StateData::default(); + + self.load_projects(&mut state)?; + self.load_leases(&mut state)?; + self.load_run_attempts(&mut state)?; + self.load_protocol_event_summaries(&mut state)?; + self.load_worktrees(&mut state)?; + self.load_linear_execution_events(&mut state)?; + self.load_review_handoffs(&mut state)?; + self.load_review_orchestrations(&mut state)?; + + Ok(state) + } + + fn persist_runtime_state(&mut self, state: &StateData) -> Result<()> { + let transaction = self.connection.transaction()?; + + persist_projects(&transaction, state)?; + persist_leases(&transaction, state)?; + persist_run_attempts(&transaction, state)?; + persist_protocol_events(&transaction, state)?; + persist_worktrees(&transaction, state)?; + persist_linear_execution_events(&transaction, state)?; + persist_review_handoffs(&transaction, state)?; + persist_review_orchestrations(&transaction, state)?; + + transaction.commit()?; + + Ok(()) + } + + fn upsert_run_attempt(&self, attempt: &RunAttemptRecord) -> Result<()> { + self.connection.execute( + "INSERT OR REPLACE INTO run_attempts ( + run_id, project_id, issue_id, attempt_number, status, thread_id, turn_id, + updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &attempt.run_id, + attempt.project_id.as_deref(), + &attempt.issue_id, + attempt.attempt_number, + &attempt.status, + attempt.thread_id.as_deref(), + attempt.turn_id.as_deref(), + &attempt.updated_at, + attempt.updated_at_unix, + ], + )?; + + Ok(()) + } + + fn append_protocol_event(&self, run_id: &str, event: &ProtocolEventRecord) -> Result { + let changed = self.connection.execute( + "INSERT OR IGNORE INTO protocol_events ( + run_id, sequence_number, event_type, created_at, created_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + run_id, + event.sequence_number, + &event.event_type, + &event.created_at, + event.created_at_unix, + ], + )?; + + Ok(changed == 1) + } + + fn upsert_linear_execution_event( + &self, + record: &LinearExecutionEventRuntimeRecord, + ) -> Result<()> { + let payload_json = serde_json::to_string(&record.record)?; + + self.connection.execute( + "INSERT OR REPLACE INTO linear_execution_events ( + idempotency_key, service_id, issue_id, event_type, event_timestamp, + event_unix, payload_json, recorded_at, recorded_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &record.record.idempotency_key, + &record.record.service_id, + &record.record.issue_id, + &record.record.event_type, + &record.record.event_timestamp, + record.event_unix, + payload_json, + &record.recorded_at, + record.recorded_at_unix, + ], + )?; + + Ok(()) + } + + fn delete_lease(&mut self, issue_id: &str) -> Result<()> { + self.connection + .execute("DELETE FROM leases WHERE issue_id = ?1", params![issue_id])?; + + Ok(()) + } + + fn delete_previous_issue_identity(&mut self, previous_issue_id: &str) -> Result<()> { + let transaction = self.connection.transaction()?; + + transaction.execute("DELETE FROM leases WHERE issue_id = ?1", params![previous_issue_id])?; + transaction.execute("DELETE FROM worktrees WHERE issue_id = ?1", params![previous_issue_id])?; + transaction.execute( + "DELETE FROM review_handoffs WHERE issue_id = ?1", + params![previous_issue_id], + )?; + transaction.execute( + "DELETE FROM review_orchestrations WHERE issue_id = ?1", + params![previous_issue_id], + )?; + transaction.commit()?; + + Ok(()) + } + + fn delete_worktree_and_review_markers(&mut self, issue_id: &str) -> Result<()> { + let transaction = self.connection.transaction()?; + + transaction.execute("DELETE FROM worktrees WHERE issue_id = ?1", params![issue_id])?; + transaction.execute("DELETE FROM review_handoffs WHERE issue_id = ?1", params![issue_id])?; + transaction.execute( + "DELETE FROM review_orchestrations WHERE issue_id = ?1", + params![issue_id], + )?; + transaction.commit()?; + + Ok(()) + } + + fn delete_review_markers(&mut self, issue_id: &str) -> Result<()> { + let transaction = self.connection.transaction()?; + + transaction.execute("DELETE FROM review_handoffs WHERE issue_id = ?1", params![issue_id])?; + transaction.execute( + "DELETE FROM review_orchestrations WHERE issue_id = ?1", + params![issue_id], + )?; + transaction.commit()?; + + Ok(()) + } + + fn load_projects(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT service_id, config_path, repo_root, worktree_root, workflow_path, \ + tracker_api_key_env_var, github_token_env_var, enabled, config_fingerprint, \ + updated_at, updated_at_unix FROM projects", + )?; + let rows = statement.query_map([], |row| { + let service_id: String = row.get(0)?; + + Ok(( + service_id.clone(), + ProjectRegistration { + service_id, + config_path: PathBuf::from(row.get::<_, String>(1)?), + repo_root: PathBuf::from(row.get::<_, String>(2)?), + worktree_root: PathBuf::from(row.get::<_, String>(3)?), + workflow_path: PathBuf::from(row.get::<_, String>(4)?), + tracker_api_key_env_var: row.get(5)?, + github_token_env_var: row.get(6)?, + enabled: row.get::<_, i64>(7)? != 0, + config_fingerprint: row.get(8)?, + updated_at: row.get(9)?, + updated_at_unix: row.get(10)?, + }, + )) + })?; + + for row in rows { + let (service_id, project) = row?; + + state.projects.insert(service_id, project); + } + + Ok(()) + } + + fn load_leases(&self, state: &mut StateData) -> Result<()> { + let mut statement = self + .connection + .prepare("SELECT issue_id, project_id, run_id, issue_state FROM leases")?; + let rows = statement.query_map([], |row| { + let issue_id: String = row.get(0)?; + + Ok(( + issue_id.clone(), + IssueLease { + issue_id, + project_id: row.get(1)?, + run_id: row.get(2)?, + issue_state: row.get(3)?, + }, + )) + })?; + + for row in rows { + let (issue_id, lease) = row?; + + state.leases.insert(issue_id, lease); + } + + Ok(()) + } + + fn load_run_attempts(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT run_id, project_id, issue_id, attempt_number, status, thread_id, turn_id, \ + updated_at, updated_at_unix FROM run_attempts", + )?; + let rows = statement.query_map([], |row| { + let run_id: String = row.get(0)?; + + Ok(( + run_id.clone(), + RunAttemptRecord { + run_id, + project_id: row.get(1)?, + issue_id: row.get(2)?, + attempt_number: row.get(3)?, + status: row.get(4)?, + thread_id: row.get(5)?, + turn_id: row.get(6)?, + updated_at: row.get(7)?, + updated_at_unix: row.get(8)?, + }, + )) + })?; + + for row in rows { + let (run_id, attempt) = row?; + + state.run_attempts.insert(run_id, attempt); + } + + Ok(()) + } + + fn load_protocol_event_summaries(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT totals.run_id, totals.event_count, totals.last_sequence_number, \ + last.event_type, last.created_at, last.created_at_unix \ + FROM ( + SELECT run_id, COUNT(*) AS event_count, MAX(sequence_number) AS last_sequence_number \ + FROM protocol_events GROUP BY run_id + ) totals \ + JOIN protocol_events last \ + ON last.run_id = totals.run_id \ + AND last.sequence_number = totals.last_sequence_number \ + ORDER BY totals.run_id", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + ProtocolEventSummaryRecord { + event_count: row.get(1)?, + last_sequence_number: Some(row.get(2)?), + last_event_type: Some(row.get(3)?), + last_event_at: Some(row.get(4)?), + last_event_at_unix: Some(row.get(5)?), + }, + )) + })?; + + for row in rows { + let (run_id, summary) = row?; + + state.event_summaries.insert(run_id, summary); + } + + Ok(()) + } + + fn load_worktrees(&self, state: &mut StateData) -> Result<()> { + let mut statement = self + .connection + .prepare("SELECT issue_id, project_id, branch_name, worktree_path FROM worktrees")?; + let rows = statement.query_map([], |row| { + let issue_id: String = row.get(0)?; + + Ok(( + issue_id.clone(), + WorktreeMappingRecord { + issue_id, + project_id: row.get(1)?, + branch_name: row.get(2)?, + worktree_path: PathBuf::from(row.get::<_, String>(3)?), + }, + )) + })?; + + for row in rows { + let (issue_id, mapping) = row?; + + state.worktrees.insert(issue_id, mapping); + } + + Ok(()) + } + + fn load_linear_execution_events(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT payload_json, event_unix, recorded_at, recorded_at_unix \ + FROM linear_execution_events", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + })?; + + for row in rows { + let (payload_json, event_unix, recorded_at, recorded_at_unix) = row?; + let record = serde_json::from_str::(&payload_json)?; + let record = LinearExecutionEventRuntimeRecord { + record, + event_unix, + recorded_at, + recorded_at_unix, + }; + + state + .linear_execution_events + .insert(record.record.idempotency_key.clone(), record); + } + + Ok(()) + } + + fn load_review_handoffs(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT project_id, issue_id, branch_name, run_id, attempt_number, pr_url, \ + target_base_ref_name, pr_head_ref_name, pr_head_oid, updated_at, updated_at_unix \ + FROM review_handoffs", + )?; + let rows = statement.query_map([], |row| { + let project_id: String = row.get(0)?; + let issue_id: String = row.get(1)?; + let branch_name: String = row.get(2)?; + let marker = ReviewHandoffMarker { + run_id: row.get(3)?, + attempt_number: row.get(4)?, + branch_name: branch_name.clone(), + pr_url: row.get(5)?, + target_base_ref_name: row.get(6)?, + pr_head_ref_name: row.get(7)?, + pr_head_oid: row.get(8)?, + }; + + Ok(( + ReviewMarkerKey::new(&project_id, &issue_id, &branch_name), + ReviewHandoffRuntimeRecord { + project_id, + issue_id, + branch_name, + marker, + updated_at: row.get(9)?, + updated_at_unix: row.get(10)?, + }, + )) + })?; + + for row in rows { + let (key, record) = row?; + + state.review_handoffs.insert(key, record); + } + + Ok(()) + } + + fn load_review_orchestrations(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT project_id, issue_id, branch_name, run_id, attempt_number, pr_url, head_sha, \ + phase, request_comment_database_id, request_created_at_unix_epoch, \ + request_description_thumbs_up_count, request_retry_count, external_round_count, \ + auto_merge_enabled_at_unix_epoch, updated_at, updated_at_unix \ + FROM review_orchestrations", + )?; + let rows = statement.query_map([], |row| { + let project_id: String = row.get(0)?; + let issue_id: String = row.get(1)?; + let branch_name: String = row.get(2)?; + let run_id: String = row.get(3)?; + let attempt_number: i64 = row.get(4)?; + let request_description_thumbs_up_count = row + .get::<_, Option>(10)? + .and_then(|count| usize::try_from(count).ok()); + let marker = ReviewOrchestrationMarker::new( + run_id.clone(), + attempt_number, + branch_name.clone(), + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + row.get(8)?, + row.get(9)?, + request_description_thumbs_up_count, + row.get(11)?, + row.get(12)?, + row.get(13)?, + ); + + Ok(( + ReviewOrchestrationKey::new( + &project_id, + &issue_id, + &branch_name, + &run_id, + attempt_number, + ), + ReviewOrchestrationRuntimeRecord { + project_id, + issue_id, + branch_name, + run_id, + attempt_number, + marker, + updated_at: row.get(14)?, + updated_at_unix: row.get(15)?, + }, + )) + })?; + + for row in rows { + let (key, record) = row?; + + state.review_orchestrations.insert(key, record); + } + + Ok(()) + } +} + +struct TimestampParts { + text: String, + unix: i64, +} + +#[derive(Clone, Debug)] +struct RunAttemptRecord { + run_id: String, + project_id: Option, + issue_id: String, + attempt_number: i64, + status: String, + thread_id: Option, + turn_id: Option, + updated_at: String, + updated_at_unix: i64, +} +impl RunAttemptRecord { + fn as_public(&self) -> RunAttempt { + RunAttempt { + run_id: self.run_id.clone(), + issue_id: self.issue_id.clone(), + attempt_number: self.attempt_number, + status: self.status.clone(), + thread_id: self.thread_id.clone(), + turn_id: self.turn_id.clone(), + } + } +} + +#[derive(Clone, Debug)] +struct ProtocolEventRecord { + sequence_number: i64, + event_type: String, + created_at: String, + created_at_unix: i64, +} + +#[derive(Clone, Debug, Default)] +struct ProtocolEventSummaryRecord { + event_count: i64, + last_sequence_number: Option, + last_event_type: Option, + last_event_at: Option, + last_event_at_unix: Option, +} +impl ProtocolEventSummaryRecord { + fn record_event(&mut self, event: &ProtocolEventRecord) { + self.event_count += 1; + + if self + .last_sequence_number + .is_none_or(|sequence_number| event.sequence_number >= sequence_number) + { + self.last_sequence_number = Some(event.sequence_number); + self.last_event_type = Some(event.event_type.clone()); + self.last_event_at = Some(event.created_at.clone()); + self.last_event_at_unix = Some(event.created_at_unix); + } + } +} + +#[derive(Clone, Debug)] +struct LinearExecutionEventRuntimeRecord { + record: LinearExecutionEventRecord, + event_unix: Option, + recorded_at: String, + recorded_at_unix: i64, +} + +#[derive(Clone, Debug)] +struct WorktreeMappingRecord { + project_id: String, + issue_id: String, + branch_name: String, + worktree_path: PathBuf, +} +impl WorktreeMappingRecord { + fn as_public(&self) -> WorktreeMapping { + WorktreeMapping { + project_id: self.project_id.clone(), + issue_id: self.issue_id.clone(), + branch_name: self.branch_name.clone(), + worktree_path: self.worktree_path.clone(), + } + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct ReviewMarkerKey { + project_id: String, + issue_id: String, + branch_name: String, +} +impl ReviewMarkerKey { + fn new(project_id: &str, issue_id: &str, branch_name: &str) -> Self { + Self { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: branch_name.to_owned(), + } + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct ReviewOrchestrationKey { + project_id: String, + issue_id: String, + branch_name: String, + run_id: String, + attempt_number: i64, +} +impl ReviewOrchestrationKey { + fn new( + project_id: &str, + issue_id: &str, + branch_name: &str, + run_id: &str, + attempt_number: i64, + ) -> Self { + Self { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: branch_name.to_owned(), + run_id: run_id.to_owned(), + attempt_number, + } + } +} + +#[derive(Clone, Debug)] +struct ReviewHandoffRuntimeRecord { + project_id: String, + issue_id: String, + branch_name: String, + marker: ReviewHandoffMarker, + updated_at: String, + updated_at_unix: i64, +} + +#[derive(Clone, Debug)] +struct ReviewOrchestrationRuntimeRecord { + project_id: String, + issue_id: String, + branch_name: String, + run_id: String, + attempt_number: i64, + marker: ReviewOrchestrationMarker, + updated_at: String, + updated_at_unix: i64, +} + +#[derive(Clone, Default)] +struct RunActivityMarkerRecord { + run_id: Option, + attempt_number: Option, + process_id: Option, + last_activity_unix_epoch: Option, + last_protocol_activity_unix_epoch: Option, + last_progress_unix_epoch: Option, + current_operation: Option, + thread_id: Option, + turn_id: Option, + thread_status: Option, + thread_active_flags: Vec, + event_count: Option, + last_event_type: Option, + effective_model: Option, + effective_model_provider: Option, + effective_cwd: Option, + effective_approval_policy: Option, + effective_approvals_reviewer: Option, + effective_sandbox_mode: Option, + child_agent_activity: Option, + protocol_activity: Option, + account: Option, + accounts: Vec, + retry_budget_attempt_count: Option, + retry_kind: Option, + retry_ready_at_unix_epoch: Option, + review_policy_phase: Option, + review_policy_status: Option, + review_policy_head_sha: Option, + review_policy_nonclean_rounds: Option, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum GuardRetention { + Local, + ParentAfterHandoff, + AdoptingChild, +} + +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) fn write_run_activity_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, +) -> Result<()> { + write_run_activity_marker_for_process(worktree_path, run_id, attempt_number, process::id()) +} + +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) fn write_run_activity_marker_for_process( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + process_id: u32, +) -> Result<()> { + write_run_activity_marker_at( + worktree_path, + run_id, + attempt_number, + process_id, + OffsetDateTime::now_utc().unix_timestamp(), + None, + ) +} + +pub(crate) fn write_run_operation_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + current_operation: &str, +) -> Result<()> { + write_run_operation_marker_for_process( + worktree_path, + run_id, + attempt_number, + process::id(), + current_operation, + ) +} + +pub(crate) fn write_run_operation_marker_for_process( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + process_id: u32, + current_operation: &str, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let now = OffsetDateTime::now_utc().unix_timestamp(); + let existing_marker = read_run_activity_marker_record(worktree_path)?; + let mut marker = run_activity_marker_record_for_attempt(existing_marker.as_ref(), run_id, attempt_number); + + marker.process_id = Some(process_id); + marker.last_activity_unix_epoch = Some(now); + marker.last_progress_unix_epoch = Some(now); + marker.current_operation = Some(current_operation.to_owned()); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_operation_marker_preserving_activity( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + current_operation: &str, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let existing_marker = read_run_activity_marker_record(worktree_path)?; + let mut marker = run_activity_marker_record_for_attempt(existing_marker.as_ref(), run_id, attempt_number); + + marker.current_operation = Some(current_operation.to_owned()); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_protocol_activity_marker( + worktree_path: &Path, + activity: &ProtocolActivityMarker<'_>, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let now = OffsetDateTime::now_utc().unix_timestamp(); + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(activity.run_id.to_owned()); + marker.attempt_number = Some(activity.attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.last_activity_unix_epoch = Some(now); + marker.last_protocol_activity_unix_epoch = Some(now); + marker.last_progress_unix_epoch = Some(now); + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.thread_id = activity.thread_id.map(str::to_owned).or(marker.thread_id); + marker.turn_id = activity.turn_id.map(str::to_owned).or(marker.turn_id); + marker.event_count = Some(activity.event_count); + marker.last_event_type = Some(activity.last_event_type.to_owned()); + marker.child_agent_activity = activity.child_agent_activity.cloned(); + marker.protocol_activity = activity.protocol_activity.cloned(); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_account_marker( + worktree_path: &Path, + account: &CodexAccountMarker<'_>, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(account.run_id.to_owned()); + marker.attempt_number = Some(account.attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.account = Some(account.account.clone()); + marker.accounts = normalize_accounts(account.account, account.accounts); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_thread_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + thread_id: &str, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.thread_id = Some(thread_id.to_owned()); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_turn_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + turn_id: &str, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.turn_id = Some(turn_id.to_owned()); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_thread_status_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + thread_id: Option<&str>, + turn_id: Option<&str>, + thread_status: &str, + thread_active_flags: &[String], +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.thread_id = thread_id.map(str::to_owned).or(marker.thread_id); + marker.turn_id = turn_id.map(str::to_owned).or(marker.turn_id); + marker.thread_status = Some(thread_status.to_owned()); + marker.thread_active_flags = thread_active_flags.to_vec(); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_effective_runtime_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + runtime: &EffectiveRuntimeMarker<'_>, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); + marker.thread_id = runtime.thread_id.map(str::to_owned).or(marker.thread_id); + marker.turn_id = runtime.turn_id.map(str::to_owned).or(marker.turn_id); + marker.effective_model = Some(runtime.effective_model.to_owned()); + marker.effective_model_provider = Some(runtime.effective_model_provider.to_owned()); + marker.effective_cwd = Some(runtime.effective_cwd.to_owned()); + marker.effective_approval_policy = Some(runtime.effective_approval_policy.to_owned()); + marker.effective_approvals_reviewer = Some(runtime.effective_approvals_reviewer.to_owned()); + marker.effective_sandbox_mode = Some(runtime.effective_sandbox_mode.to_owned()); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn read_run_activity_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, +) -> Result> { + let marker = read_run_activity_marker_record(worktree_path)?.filter(|marker| { + marker.run_id.as_deref() == Some(run_id) && marker.attempt_number == Some(attempt_number) + }); + + Ok(marker.and_then(|marker| marker.last_activity_unix_epoch)) +} + +pub(crate) fn read_run_protocol_activity_marker( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, +) -> Result> { + let marker = read_run_activity_marker_record(worktree_path)?.filter(|marker| { + marker.run_id.as_deref() == Some(run_id) && marker.attempt_number == Some(attempt_number) + }); + + Ok(marker.and_then(|marker| marker.last_protocol_activity_unix_epoch)) +} + +pub(crate) fn write_run_retry_budget_attempt_count( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + retry_budget_attempt_count: i64, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.retry_budget_attempt_count = Some(retry_budget_attempt_count); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_retry_schedule( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + retry_kind: &str, + retry_ready_at_unix_epoch: i64, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + marker.retry_kind = Some(retry_kind.to_owned()); + marker.retry_ready_at_unix_epoch = Some(retry_ready_at_unix_epoch); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn clear_run_retry_schedule(worktree_path: &Path) -> Result<()> { + let Some(mut marker) = read_run_activity_marker_record(worktree_path)? else { + return Ok(()); + }; + + marker.retry_kind = None; + marker.retry_ready_at_unix_epoch = None; + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn write_run_review_policy_state( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + review_policy_phase: &str, + review_policy_status: &str, + review_policy_head_sha: &str, + review_policy_nonclean_rounds: i64, +) -> Result<()> { + fs::create_dir_all(worktree_path)?; + + let mut marker = read_run_activity_marker_record(worktree_path)?.unwrap_or_default(); + + marker.run_id = Some(run_id.to_owned()); + marker.attempt_number = Some(attempt_number); + + marker.process_id.get_or_insert_with(process::id); + + marker.review_policy_phase = Some(review_policy_phase.to_owned()); + marker.review_policy_status = Some(review_policy_status.to_owned()); + marker.review_policy_head_sha = Some(review_policy_head_sha.to_owned()); + marker.review_policy_nonclean_rounds = Some(review_policy_nonclean_rounds); + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn clear_run_review_policy_state(worktree_path: &Path) -> Result<()> { + let Some(mut marker) = read_run_activity_marker_record(worktree_path)? else { + return Ok(()); + }; + + marker.review_policy_phase = None; + marker.review_policy_status = None; + marker.review_policy_head_sha = None; + marker.review_policy_nonclean_rounds = None; + + write_run_activity_marker_record(worktree_path, &marker)?; + + Ok(()) +} + +pub(crate) fn read_run_retry_budget_attempt_count(worktree_path: &Path) -> Result> { + Ok(read_run_activity_marker_record(worktree_path)? + .and_then(|marker| marker.retry_budget_attempt_count)) +} + +pub(crate) fn read_run_activity_marker_snapshot( + worktree_path: &Path, +) -> Result> { + Ok(read_run_activity_marker_record(worktree_path)?.and_then(|marker| { + let accounts = accounts_from_marker_record(&marker); + + Some(RunActivityMarker { + run_id: marker.run_id?, + attempt_number: marker.attempt_number?, + process_id: marker.process_id, + last_activity_unix_epoch: marker.last_activity_unix_epoch, + last_protocol_activity_unix_epoch: marker.last_protocol_activity_unix_epoch, + last_progress_unix_epoch: marker.last_progress_unix_epoch, + current_operation: marker.current_operation, + thread_id: marker.thread_id, + turn_id: marker.turn_id, + thread_status: marker.thread_status, + thread_active_flags: marker.thread_active_flags, + event_count: marker.event_count, + last_event_type: marker.last_event_type, + effective_model: marker.effective_model, + effective_model_provider: marker.effective_model_provider, + effective_cwd: marker.effective_cwd, + effective_approval_policy: marker.effective_approval_policy, + effective_approvals_reviewer: marker.effective_approvals_reviewer, + effective_sandbox_mode: marker.effective_sandbox_mode, + child_agent_activity: marker.child_agent_activity, + protocol_activity: marker.protocol_activity, + account: marker.account, + accounts, + retry_budget_attempt_count: marker.retry_budget_attempt_count, + retry_kind: marker.retry_kind, + retry_ready_at_unix_epoch: marker.retry_ready_at_unix_epoch, + review_policy_phase: marker.review_policy_phase, + review_policy_status: marker.review_policy_status, + review_policy_head_sha: marker.review_policy_head_sha, + review_policy_nonclean_rounds: marker.review_policy_nonclean_rounds, + }) + })) +} + +fn normalize_accounts( + selected: &CodexAccountActivitySummary, + accounts: &[CodexAccountActivitySummary], +) -> Vec { + let mut normalized = if accounts.is_empty() { + vec![selected.clone()] + } else { + accounts.to_vec() + }; + + if !normalized.iter().any(|account| { + account.account_fingerprint == selected.account_fingerprint + }) { + normalized.insert(0, selected.clone()); + } + + normalized +} + +fn accounts_from_marker_record( + marker: &RunActivityMarkerRecord, +) -> Vec { + if marker.accounts.is_empty() { + marker.account.iter().cloned().collect() + } else { + marker.accounts.clone() + } +} + +fn persist_projects(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for project in state.projects.values() { + transaction.execute( + "INSERT OR REPLACE INTO projects ( + service_id, config_path, repo_root, worktree_root, workflow_path, + tracker_api_key_env_var, github_token_env_var, enabled, config_fingerprint, + updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + project.service_id(), + project.config_path().to_string_lossy().as_ref(), + project.repo_root().to_string_lossy().as_ref(), + project.worktree_root().to_string_lossy().as_ref(), + project.workflow_path().to_string_lossy().as_ref(), + project.tracker_api_key_env_var(), + project.github_token_env_var(), + if project.enabled() { 1_i64 } else { 0_i64 }, + project.config_fingerprint(), + project.updated_at(), + project.updated_at_unix(), + ], + )?; + } + + Ok(()) +} + +fn persist_leases(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for lease in state.leases.values() { + transaction.execute( + "INSERT OR REPLACE INTO leases (issue_id, project_id, run_id, issue_state) \ + VALUES (?1, ?2, ?3, ?4)", + params![lease.issue_id(), lease.project_id(), lease.run_id(), lease.issue_state()], + )?; + } + + Ok(()) +} + +fn persist_run_attempts(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for attempt in state.run_attempts.values() { + transaction.execute( + "INSERT OR REPLACE INTO run_attempts ( + run_id, project_id, issue_id, attempt_number, status, thread_id, turn_id, + updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &attempt.run_id, + attempt.project_id.as_deref(), + &attempt.issue_id, + attempt.attempt_number, + &attempt.status, + attempt.thread_id.as_deref(), + attempt.turn_id.as_deref(), + &attempt.updated_at, + attempt.updated_at_unix, + ], + )?; + } + + Ok(()) +} + +fn persist_protocol_events( + transaction: &Transaction<'_>, + state: &StateData, +) -> Result<()> { + for (run_id, events) in &state.events { + for event in events { + transaction.execute( + "INSERT OR REPLACE INTO protocol_events ( + run_id, sequence_number, event_type, created_at, created_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + run_id, + event.sequence_number, + &event.event_type, + &event.created_at, + event.created_at_unix, + ], + )?; + } + } + + Ok(()) +} + +fn persist_worktrees(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for mapping in state.worktrees.values() { + transaction.execute( + "INSERT OR REPLACE INTO worktrees (issue_id, project_id, branch_name, worktree_path) \ + VALUES (?1, ?2, ?3, ?4)", + params![ + &mapping.issue_id, + &mapping.project_id, + &mapping.branch_name, + mapping.worktree_path.to_string_lossy().as_ref(), + ], + )?; + } + + Ok(()) +} + +fn persist_linear_execution_events( + transaction: &Transaction<'_>, + state: &StateData, +) -> Result<()> { + for record in state.linear_execution_events.values() { + let payload_json = serde_json::to_string(&record.record)?; + + transaction.execute( + "INSERT OR REPLACE INTO linear_execution_events ( + idempotency_key, service_id, issue_id, event_type, event_timestamp, + event_unix, payload_json, recorded_at, recorded_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &record.record.idempotency_key, + &record.record.service_id, + &record.record.issue_id, + &record.record.event_type, + &record.record.event_timestamp, + record.event_unix, + payload_json, + &record.recorded_at, + record.recorded_at_unix, + ], + )?; + } + + Ok(()) +} + +fn persist_review_handoffs(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for record in state.review_handoffs.values() { + transaction.execute( + "INSERT OR REPLACE INTO review_handoffs ( + project_id, issue_id, branch_name, run_id, attempt_number, pr_url, + target_base_ref_name, pr_head_ref_name, pr_head_oid, updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + record.project_id, + record.issue_id, + record.branch_name, + record.marker.run_id, + record.marker.attempt_number, + record.marker.pr_url, + record.marker.target_base_ref_name, + record.marker.pr_head_ref_name, + record.marker.pr_head_oid, + record.updated_at, + record.updated_at_unix, + ], + )?; + } + + Ok(()) +} + +fn persist_review_orchestrations(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { + for record in state.review_orchestrations.values() { + transaction.execute( + "INSERT OR REPLACE INTO review_orchestrations ( + project_id, issue_id, branch_name, run_id, attempt_number, pr_url, head_sha, + phase, request_comment_database_id, request_created_at_unix_epoch, + request_description_thumbs_up_count, request_retry_count, external_round_count, + auto_merge_enabled_at_unix_epoch, updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)", + params![ + record.project_id, + record.issue_id, + record.branch_name, + record.run_id, + record.attempt_number, + record.marker.pr_url, + record.marker.head_sha, + record.marker.phase, + record.marker.request_comment_database_id, + record.marker.request_created_at_unix_epoch, + record + .marker + .request_description_thumbs_up_count + .and_then(|count| i64::try_from(count).ok()), + record.marker.request_retry_count, + record.marker.external_round_count, + record.marker.auto_merge_enabled_at_unix_epoch, + record.updated_at, + record.updated_at_unix, + ], + )?; + } + + Ok(()) +} + +fn dispatch_slot_lock_path(root: &Path, slot_index: usize) -> PathBuf { + root.join(format!("{DISPATCH_SLOT_LOCK_FILE_PREFIX}.{slot_index}.lock")) +} + +fn issue_claim_lock_path(root: &Path, issue_id: &str) -> PathBuf { + root.join(format!("{ISSUE_CLAIM_LOCK_FILE_PREFIX}.{issue_id}.lock")) +} + +fn issue_claim_id_from_path(path: &Path) -> Option { + let file_name = path.file_name()?.to_str()?; + + file_name + .strip_prefix(&format!("{ISSUE_CLAIM_LOCK_FILE_PREFIX}.")) + .and_then(|suffix| suffix.strip_suffix(".lock")) + .map(str::to_owned) +} + +fn write_issue_claim_record( + lock_file: &mut File, + project_id: &str, + issue_id: &str, + run_id: &str, + issue_state: &str, +) -> Result<()> { + lock_file.set_len(0)?; + lock_file.seek(SeekFrom::Start(0))?; + + write!( + lock_file, + "project_id={project_id}\nissue_id={issue_id}\nrun_id={run_id}\nissue_state={issue_state}\n" + )?; + + lock_file.flush()?; + + Ok(()) +} + +fn read_issue_claim_record(path: &Path) -> Result> { + let mut body = String::new(); + let mut file = File::open(path)?; + + file.read_to_string(&mut body)?; + + if body.trim().is_empty() { + return Ok(None); + } + + let mut project_id = None; + let mut issue_id = None; + let mut run_id = None; + let mut issue_state = None; + + for line in body.lines().filter(|line| !line.trim().is_empty()) { + let (key, value) = line + .split_once('=') + .ok_or_else(|| eyre::eyre!("issue claim record `{}` is malformed", path.display()))?; + + match key { + "project_id" => project_id = Some(value.to_owned()), + "issue_id" => issue_id = Some(value.to_owned()), + "run_id" => run_id = Some(value.to_owned()), + "issue_state" => issue_state = Some(value.to_owned()), + _ => {}, + } + } + + let Some(project_id) = project_id else { + return Err(eyre::eyre!("issue claim record `{}` is missing project_id", path.display())); + }; + let Some(issue_id) = issue_id else { + return Err(eyre::eyre!("issue claim record `{}` is missing issue_id", path.display())); + }; + let Some(run_id) = run_id else { + return Err(eyre::eyre!("issue claim record `{}` is missing run_id", path.display())); + }; + let Some(issue_state) = issue_state else { + return Err(eyre::eyre!("issue claim record `{}` is missing issue_state", path.display())); + }; + + Ok(Some(IssueLease { project_id, issue_id, run_id, issue_state })) +} + +#[cfg_attr(not(test), allow(dead_code))] +fn write_run_activity_marker_at( + worktree_path: &Path, + run_id: &str, + attempt_number: i64, + process_id: u32, + last_activity_unix_epoch: i64, + last_protocol_activity_unix_epoch: Option, +) -> Result<()> { + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + let existing_marker = read_run_activity_marker_record(worktree_path)?; + let same_run_marker = existing_marker + .as_ref() + .filter(|marker| marker.run_id.as_deref() == Some(run_id) && marker.attempt_number == Some(attempt_number)); + let mut marker = run_activity_marker_record_for_attempt(existing_marker.as_ref(), run_id, attempt_number); + + marker.process_id = Some(process_id); + marker.last_activity_unix_epoch = Some(last_activity_unix_epoch); + marker.last_protocol_activity_unix_epoch = last_protocol_activity_unix_epoch + .or_else(|| same_run_marker.and_then(|marker| marker.last_protocol_activity_unix_epoch)); + + if let Some(same_run_marker) = same_run_marker { + marker.retry_kind = same_run_marker.retry_kind.clone(); + marker.retry_ready_at_unix_epoch = same_run_marker.retry_ready_at_unix_epoch; + } + + fs::write(marker_path, serialize_run_activity_marker_record(&marker))?; + + Ok(()) +} + +fn run_activity_marker_record_for_attempt( + existing_marker: Option<&RunActivityMarkerRecord>, + run_id: &str, + attempt_number: i64, +) -> RunActivityMarkerRecord { + let same_run_marker = existing_marker + .filter(|marker| marker.run_id.as_deref() == Some(run_id) && marker.attempt_number == Some(attempt_number)); + + RunActivityMarkerRecord { + run_id: Some(run_id.to_owned()), + attempt_number: Some(attempt_number), + process_id: same_run_marker.and_then(|marker| marker.process_id), + last_activity_unix_epoch: same_run_marker.and_then(|marker| marker.last_activity_unix_epoch), + last_protocol_activity_unix_epoch: same_run_marker + .and_then(|marker| marker.last_protocol_activity_unix_epoch), + last_progress_unix_epoch: same_run_marker.and_then(|marker| marker.last_progress_unix_epoch), + current_operation: same_run_marker.and_then(|marker| marker.current_operation.clone()), + thread_id: same_run_marker.and_then(|marker| marker.thread_id.clone()), + turn_id: same_run_marker.and_then(|marker| marker.turn_id.clone()), + thread_status: same_run_marker.and_then(|marker| marker.thread_status.clone()), + thread_active_flags: same_run_marker + .map(|marker| marker.thread_active_flags.clone()) + .unwrap_or_default(), + event_count: same_run_marker.and_then(|marker| marker.event_count), + last_event_type: same_run_marker.and_then(|marker| marker.last_event_type.clone()), + effective_model: same_run_marker.and_then(|marker| marker.effective_model.clone()), + effective_model_provider: same_run_marker + .and_then(|marker| marker.effective_model_provider.clone()), + effective_cwd: same_run_marker.and_then(|marker| marker.effective_cwd.clone()), + effective_approval_policy: same_run_marker + .and_then(|marker| marker.effective_approval_policy.clone()), + effective_approvals_reviewer: same_run_marker + .and_then(|marker| marker.effective_approvals_reviewer.clone()), + effective_sandbox_mode: same_run_marker + .and_then(|marker| marker.effective_sandbox_mode.clone()), + child_agent_activity: same_run_marker + .and_then(|marker| marker.child_agent_activity.clone()), + protocol_activity: same_run_marker.and_then(|marker| marker.protocol_activity.clone()), + account: same_run_marker.and_then(|marker| marker.account.clone()), + accounts: same_run_marker.map(|marker| marker.accounts.clone()).unwrap_or_default(), + retry_budget_attempt_count: existing_marker + .and_then(|marker| marker.retry_budget_attempt_count), + retry_kind: same_run_marker.and_then(|marker| marker.retry_kind.clone()), + retry_ready_at_unix_epoch: same_run_marker.and_then(|marker| marker.retry_ready_at_unix_epoch), + review_policy_phase: existing_marker.and_then(|marker| marker.review_policy_phase.clone()), + review_policy_status: existing_marker.and_then(|marker| marker.review_policy_status.clone()), + review_policy_head_sha: existing_marker.and_then(|marker| marker.review_policy_head_sha.clone()), + review_policy_nonclean_rounds: existing_marker.and_then(|marker| marker.review_policy_nonclean_rounds), + } +} + +fn read_run_activity_marker_record( + worktree_path: &Path, +) -> Result> { + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + let marker_body = match fs::read_to_string(&marker_path) { + Ok(body) => body, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let mut marker = RunActivityMarkerRecord::default(); + + for line in marker_body.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + + match key { + "run_id" => marker.run_id = Some(value.to_owned()), + "attempt_number" => marker.attempt_number = value.parse::().ok(), + "process_id" => marker.process_id = value.parse::().ok(), + "last_activity_unix_epoch" => + marker.last_activity_unix_epoch = value.parse::().ok(), + "last_protocol_activity_unix_epoch" => + marker.last_protocol_activity_unix_epoch = value.parse::().ok(), + "last_progress_unix_epoch" => + marker.last_progress_unix_epoch = value.parse::().ok(), + "current_operation" => marker.current_operation = Some(value.to_owned()), + "thread_id" => marker.thread_id = Some(value.to_owned()), + "turn_id" => marker.turn_id = Some(value.to_owned()), + "thread_status" => marker.thread_status = Some(value.to_owned()), + "thread_active_flags" => marker.thread_active_flags = parse_marker_list(value), + "event_count" => marker.event_count = value.parse::().ok(), + "last_event_type" => marker.last_event_type = Some(value.to_owned()), + "effective_model" => marker.effective_model = Some(value.to_owned()), + "effective_model_provider" => + marker.effective_model_provider = Some(value.to_owned()), + "effective_cwd" => marker.effective_cwd = Some(value.to_owned()), + "effective_approval_policy" => + marker.effective_approval_policy = Some(value.to_owned()), + "effective_approvals_reviewer" => + marker.effective_approvals_reviewer = Some(value.to_owned()), + "effective_sandbox_mode" => marker.effective_sandbox_mode = Some(value.to_owned()), + "child_agent_activity" => + marker.child_agent_activity = serde_json::from_str(value).ok(), + "protocol_activity" => marker.protocol_activity = serde_json::from_str(value).ok(), + "account" => marker.account = serde_json::from_str(value).ok(), + "accounts" => { + if let Ok(accounts) = serde_json::from_str(value) { + marker.accounts = accounts; + } + }, + "retry_budget_attempt_count" => + marker.retry_budget_attempt_count = value.parse::().ok(), + "retry_kind" => marker.retry_kind = Some(value.to_owned()), + "retry_ready_at_unix_epoch" => + marker.retry_ready_at_unix_epoch = value.parse::().ok(), + "review_policy_phase" => marker.review_policy_phase = Some(value.to_owned()), + "review_policy_status" => marker.review_policy_status = Some(value.to_owned()), + "review_policy_head_sha" => marker.review_policy_head_sha = Some(value.to_owned()), + "review_policy_nonclean_rounds" => + marker.review_policy_nonclean_rounds = value.parse::().ok(), + _ => {}, + } + } + + Ok(Some(marker)) +} + +fn write_run_activity_marker_record( + worktree_path: &Path, + marker: &RunActivityMarkerRecord, +) -> Result<()> { + fs::write( + worktree_path.join(RUN_ACTIVITY_MARKER_FILE), + serialize_run_activity_marker_record(marker), + )?; + + Ok(()) +} + +fn serialize_run_activity_marker_record(marker: &RunActivityMarkerRecord) -> String { + let mut body = String::new(); + + if let Some(run_id) = &marker.run_id { + body.push_str(&format!("run_id={run_id}\n")); + } + if let Some(attempt_number) = marker.attempt_number { + body.push_str(&format!("attempt_number={attempt_number}\n")); + } + if let Some(process_id) = marker.process_id { + body.push_str(&format!("process_id={process_id}\n")); + } + if let Some(last_activity_unix_epoch) = marker.last_activity_unix_epoch { + body.push_str(&format!("last_activity_unix_epoch={last_activity_unix_epoch}\n")); + } + if let Some(last_protocol_activity_unix_epoch) = marker.last_protocol_activity_unix_epoch { + body.push_str(&format!( + "last_protocol_activity_unix_epoch={last_protocol_activity_unix_epoch}\n" + )); + } + if let Some(last_progress_unix_epoch) = marker.last_progress_unix_epoch { + body.push_str(&format!("last_progress_unix_epoch={last_progress_unix_epoch}\n")); + } + if let Some(current_operation) = &marker.current_operation { + body.push_str(&format!("current_operation={current_operation}\n")); + } + if let Some(thread_id) = &marker.thread_id { + body.push_str(&format!("thread_id={thread_id}\n")); + } + if let Some(turn_id) = &marker.turn_id { + body.push_str(&format!("turn_id={turn_id}\n")); + } + if let Some(thread_status) = &marker.thread_status { + body.push_str(&format!("thread_status={thread_status}\n")); + } + + if !marker.thread_active_flags.is_empty() { + body.push_str(&format!( + "thread_active_flags={}\n", + marker.thread_active_flags.join(",") + )); + } + + if let Some(event_count) = marker.event_count { + body.push_str(&format!("event_count={event_count}\n")); + } + if let Some(last_event_type) = &marker.last_event_type { + body.push_str(&format!("last_event_type={last_event_type}\n")); + } + if let Some(effective_model) = &marker.effective_model { + body.push_str(&format!("effective_model={effective_model}\n")); + } + if let Some(effective_model_provider) = &marker.effective_model_provider { + body.push_str(&format!("effective_model_provider={effective_model_provider}\n")); + } + if let Some(effective_cwd) = &marker.effective_cwd { + body.push_str(&format!("effective_cwd={effective_cwd}\n")); + } + if let Some(effective_approval_policy) = &marker.effective_approval_policy { + body.push_str(&format!( + "effective_approval_policy={effective_approval_policy}\n" + )); + } + if let Some(effective_approvals_reviewer) = &marker.effective_approvals_reviewer { + body.push_str(&format!( + "effective_approvals_reviewer={effective_approvals_reviewer}\n" + )); + } + if let Some(effective_sandbox_mode) = &marker.effective_sandbox_mode { + body.push_str(&format!("effective_sandbox_mode={effective_sandbox_mode}\n")); + } + if let Some(child_agent_activity) = &marker.child_agent_activity + && let Ok(summary_json) = serde_json::to_string(child_agent_activity) + { + body.push_str(&format!("child_agent_activity={summary_json}\n")); + } + if let Some(protocol_activity) = &marker.protocol_activity + && let Ok(summary_json) = serde_json::to_string(protocol_activity) + { + body.push_str(&format!("protocol_activity={summary_json}\n")); + } + if let Some(account) = &marker.account + && let Ok(summary_json) = serde_json::to_string(account) + { + body.push_str(&format!("account={summary_json}\n")); + } + + if !marker.accounts.is_empty() + && let Ok(accounts_json) = serde_json::to_string(&marker.accounts) + { + body.push_str(&format!("accounts={accounts_json}\n")); + } + + if let Some(retry_budget_attempt_count) = marker.retry_budget_attempt_count { + body.push_str(&format!("retry_budget_attempt_count={retry_budget_attempt_count}\n")); + } + if let Some(retry_kind) = &marker.retry_kind { + body.push_str(&format!("retry_kind={retry_kind}\n")); + } + if let Some(retry_ready_at_unix_epoch) = marker.retry_ready_at_unix_epoch { + body.push_str(&format!("retry_ready_at_unix_epoch={retry_ready_at_unix_epoch}\n")); + } + if let Some(review_policy_phase) = &marker.review_policy_phase { + body.push_str(&format!("review_policy_phase={review_policy_phase}\n")); + } + if let Some(review_policy_status) = &marker.review_policy_status { + body.push_str(&format!("review_policy_status={review_policy_status}\n")); + } + if let Some(review_policy_head_sha) = &marker.review_policy_head_sha { + body.push_str(&format!("review_policy_head_sha={review_policy_head_sha}\n")); + } + if let Some(review_policy_nonclean_rounds) = marker.review_policy_nonclean_rounds { + body.push_str(&format!("review_policy_nonclean_rounds={review_policy_nonclean_rounds}\n")); + } + + body +} + +fn parse_marker_list(value: &str) -> Vec { + value + .split(',') + .filter(|part| !part.is_empty()) + .map(str::to_owned) + .collect() +} + +fn timestamp_parts() -> TimestampParts { + let now = OffsetDateTime::now_utc(); + + TimestampParts { + text: now.format(&Rfc3339).expect("timestamp formatting should succeed"), + unix: now.unix_timestamp(), + } +} + +fn parse_linear_execution_event_unix(record: &LinearExecutionEventRecord) -> Option { + OffsetDateTime::parse(&record.event_timestamp, &Rfc3339) + .ok() + .map(|timestamp| timestamp.unix_timestamp()) +} + +fn protocol_event_summary_from_events(events: &[ProtocolEventRecord]) -> ProtocolEventSummaryRecord { + let mut summary = ProtocolEventSummaryRecord::default(); + + for event in events { + summary.record_event(event); + } + + summary +} + +fn compare_attempt_records(left: &RunAttemptRecord, right: &RunAttemptRecord) -> Ordering { + left.attempt_number + .cmp(&right.attempt_number) + .then_with(|| left.updated_at_unix.cmp(&right.updated_at_unix)) + .then_with(|| left.run_id.cmp(&right.run_id)) +} + +fn compare_linear_execution_event_runtime_records( + left: &LinearExecutionEventRuntimeRecord, + right: &LinearExecutionEventRuntimeRecord, +) -> Ordering { + left.event_unix + .cmp(&right.event_unix) + .then_with(|| left.recorded_at_unix.cmp(&right.recorded_at_unix)) + .then_with(|| left.record.idempotency_key.cmp(&right.record.idempotency_key)) +} + +fn compare_project_run_status(left: &ProjectRunStatus, right: &ProjectRunStatus) -> Ordering { + right + .active_lease + .cmp(&left.active_lease) + .then_with(|| right.updated_at.cmp(&left.updated_at)) + .then_with(|| right.attempt_number.cmp(&left.attempt_number)) + .then_with(|| right.run_id.cmp(&left.run_id)) +} + +#[cfg(unix)] +fn clear_close_on_exec(file: &File) -> Result<()> { + let fd = file.as_raw_fd(); + let existing_flags = unsafe { libc::fcntl(fd, F_GETFD) }; + + if existing_flags == -1 { + return Err(Error::last_os_error().into()); + } + + let new_flags = existing_flags & !FD_CLOEXEC; + + if new_flags != existing_flags { + let result = unsafe { libc::fcntl(fd, F_SETFD, new_flags) }; + + if result == -1 { + return Err(Error::last_os_error().into()); + } + } + + Ok(()) +} + +#[cfg(unix)] +fn set_close_on_exec(file: &File) -> Result<()> { + let fd = file.as_raw_fd(); + let existing_flags = unsafe { libc::fcntl(fd, F_GETFD) }; + + if existing_flags == -1 { + return Err(Error::last_os_error().into()); + } + + let new_flags = existing_flags | FD_CLOEXEC; + + if new_flags != existing_flags { + let result = unsafe { libc::fcntl(fd, F_SETFD, new_flags) }; + + if result == -1 { + return Err(Error::last_os_error().into()); + } + } + + Ok(()) +} diff --git a/apps/decodex/src/state/models.rs b/apps/decodex/src/state/models.rs new file mode 100644 index 00000000..774acb95 --- /dev/null +++ b/apps/decodex/src/state/models.rs @@ -0,0 +1,678 @@ +/// Active lease for one issue. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct IssueLease { + project_id: String, + issue_id: String, + run_id: String, + issue_state: String, +} +impl IssueLease { + /// Local project identifier owning this lease. + pub fn project_id(&self) -> &str { + &self.project_id + } + + /// Issue identifier owning the lease. + pub fn issue_id(&self) -> &str { + &self.issue_id + } + + /// Run identifier holding the lease. + pub fn run_id(&self) -> &str { + &self.run_id + } + + /// Tracker state representing the dispatched run. + pub fn issue_state(&self) -> &str { + &self.issue_state + } +} + +/// Persistent run attempt metadata. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RunAttempt { + run_id: String, + issue_id: String, + attempt_number: i64, + status: String, + thread_id: Option, + turn_id: Option, +} +impl RunAttempt { + /// Stable run identifier. + pub fn run_id(&self) -> &str { + &self.run_id + } + + /// Issue identifier for the run. + pub fn issue_id(&self) -> &str { + &self.issue_id + } + + /// Attempt number for this run. + pub fn attempt_number(&self) -> i64 { + self.attempt_number + } + + /// Current local status for the run. + pub fn status(&self) -> &str { + &self.status + } + + /// Thread identifier returned by `app-server`, when known. + pub fn thread_id(&self) -> Option<&str> { + self.thread_id.as_deref() + } + + /// Latest turn identifier returned by `app-server`, when known. + pub fn turn_id(&self) -> Option<&str> { + self.turn_id.as_deref() + } +} + +/// Project-scoped operator view of one run attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectRunStatus { + run_id: String, + issue_id: String, + attempt_number: i64, + status: String, + thread_id: Option, + turn_id: Option, + updated_at: String, + branch_name: Option, + worktree_path: Option, + active_lease: bool, + event_count: i64, + last_event_type: Option, + last_event_at: Option, +} +impl ProjectRunStatus { + /// Stable run identifier. + pub fn run_id(&self) -> &str { + &self.run_id + } + + /// Issue identifier for the run. + pub fn issue_id(&self) -> &str { + &self.issue_id + } + + /// Attempt number for this run. + pub fn attempt_number(&self) -> i64 { + self.attempt_number + } + + /// Current local status for the run. + pub fn status(&self) -> &str { + &self.status + } + + /// Thread identifier returned by `app-server`, when known. + pub fn thread_id(&self) -> Option<&str> { + self.thread_id.as_deref() + } + + /// Latest turn identifier returned by `app-server`, when known. + pub fn turn_id(&self) -> Option<&str> { + self.turn_id.as_deref() + } + + /// Timestamp of the latest run-attempt status update. + pub fn updated_at(&self) -> &str { + &self.updated_at + } + + /// Branch name for the retained lane, when known. + pub fn branch_name(&self) -> Option<&str> { + self.branch_name.as_deref() + } + + /// Filesystem path to the retained worktree, when known. + pub fn worktree_path(&self) -> Option<&Path> { + self.worktree_path.as_deref() + } + + /// Whether this run still holds the active local lease. + pub fn active_lease(&self) -> bool { + self.active_lease + } + + /// Number of recorded protocol events for the run. + pub fn event_count(&self) -> i64 { + self.event_count + } + + /// Latest recorded protocol event type, when one exists. + pub fn last_event_type(&self) -> Option<&str> { + self.last_event_type.as_deref() + } + + /// Timestamp of the latest recorded protocol event, when one exists. + pub fn last_event_at(&self) -> Option<&str> { + self.last_event_at.as_deref() + } +} + +/// Worktree mapping for one issue lane. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WorktreeMapping { + project_id: String, + issue_id: String, + branch_name: String, + worktree_path: PathBuf, +} +impl WorktreeMapping { + /// Local project identifier owning this lane. + pub fn project_id(&self) -> &str { + &self.project_id + } + + /// Issue identifier for this lane. + pub fn issue_id(&self) -> &str { + &self.issue_id + } + + /// Branch name used for the lane. + pub fn branch_name(&self) -> &str { + &self.branch_name + } + + /// Filesystem path to the worktree checkout. + pub fn worktree_path(&self) -> &Path { + &self.worktree_path + } +} + +/// Unix file-descriptor handoff for a daemon-planned lease adopted by a child process. +pub struct PreacquiredLeaseGuards { + /// The inherited issue-claim lock fd that keeps one issue single-owned across processes. + pub issue_claim_fd: i32, + /// The inherited dispatch-slot lock fd that keeps one shared capacity slot occupied. + pub dispatch_slot_fd: i32, + /// The inherited shared dispatch-slot index used for local guard bookkeeping. + pub dispatch_slot_index: usize, +} + +/// Registered repo target managed by the local Decodex control plane. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProjectRegistration { + service_id: String, + config_path: PathBuf, + repo_root: PathBuf, + worktree_root: PathBuf, + workflow_path: PathBuf, + tracker_api_key_env_var: String, + github_token_env_var: String, + enabled: bool, + config_fingerprint: String, + updated_at: String, + updated_at_unix: i64, +} +impl ProjectRegistration { + /// Build a registry row from a Decodex project config. + pub(crate) fn from_config( + service_id: &str, + config_path: &Path, + config: &ServiceConfig, + enabled: bool, + config_fingerprint: &str, + ) -> Self { + let now = timestamp_parts(); + + Self { + service_id: service_id.to_owned(), + config_path: config_path.to_path_buf(), + repo_root: config.repo_root().to_path_buf(), + worktree_root: config.worktree_root().to_path_buf(), + workflow_path: config.workflow_path().to_path_buf(), + tracker_api_key_env_var: config.tracker().api_key_env_var().to_owned(), + github_token_env_var: config.github().token_env_var().to_owned(), + enabled, + config_fingerprint: config_fingerprint.to_owned(), + updated_at: now.text, + updated_at_unix: now.unix, + } + } + + /// Stable service id from the project config. + pub(crate) fn service_id(&self) -> &str { + &self.service_id + } + + /// Absolute config path registered for this project. + pub(crate) fn config_path(&self) -> &Path { + &self.config_path + } + + /// Absolute repository root for this project. + pub(crate) fn repo_root(&self) -> &Path { + &self.repo_root + } + + /// Absolute worktree root for this project. + pub(crate) fn worktree_root(&self) -> &Path { + &self.worktree_root + } + + /// Absolute workflow path registered for this project. + pub(crate) fn workflow_path(&self) -> &Path { + &self.workflow_path + } + + /// Environment variable name for the tracker API key. + pub(crate) fn tracker_api_key_env_var(&self) -> &str { + &self.tracker_api_key_env_var + } + + /// Environment variable name for the GitHub token. + pub(crate) fn github_token_env_var(&self) -> &str { + &self.github_token_env_var + } + + /// Whether the project participates in `decodex serve`. + pub(crate) fn enabled(&self) -> bool { + self.enabled + } + + /// Last config fingerprint registered for this project. + pub(crate) fn config_fingerprint(&self) -> &str { + &self.config_fingerprint + } + + /// Last registry update timestamp. + pub(crate) fn updated_at(&self) -> &str { + &self.updated_at + } + + /// Last registry update timestamp as Unix epoch seconds. + pub(crate) fn updated_at_unix(&self) -> i64 { + self.updated_at_unix + } + + /// Set whether the registered project is enabled. + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + + let now = timestamp_parts(); + + self.updated_at = now.text; + self.updated_at_unix = now.unix; + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ChildAgentActivitySummary { + pub(crate) buckets: Vec, + pub(crate) current_bucket: Option, + pub(crate) current_detail: Option, + pub(crate) current_started_unix_epoch: Option, + pub(crate) current_elapsed_seconds: Option, + pub(crate) wall_seconds: i64, + pub(crate) event_count: i64, + pub(crate) tool_call_count: i64, + pub(crate) input_tokens_current: Option, + pub(crate) input_tokens_max: Option, + pub(crate) input_tokens_cumulative: i64, + pub(crate) output_tokens_cumulative: i64, + pub(crate) largest_tool_output_bytes: Option, + pub(crate) largest_tool_output_tool: Option, + pub(crate) large_output_warnings: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ChildAgentActivityBucket { + pub(crate) name: String, + pub(crate) wall_seconds: i64, + pub(crate) event_count: i64, + pub(crate) tool_call_count: i64, + pub(crate) input_tokens: i64, + pub(crate) output_tokens: i64, + pub(crate) output_bytes: i64, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ProtocolActivitySummary { + pub(crate) turn_status: Option, + pub(crate) waiting_reason: Option, + pub(crate) rate_limit_status: Option, + pub(crate) recent_events: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ProtocolActivityEventSummary { + pub(crate) event_type: String, + pub(crate) category: String, + pub(crate) detail: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct CodexAccountActivitySummary { + pub(crate) account_fingerprint: String, + pub(crate) email: Option, + pub(crate) plan_type: Option, + pub(crate) status: String, + pub(crate) refresh_status: String, + pub(crate) checked_at_unix_epoch: Option, + pub(crate) selected_at_unix_epoch: Option, + pub(crate) primary_window_seconds: Option, + pub(crate) primary_remaining_percent: Option, + pub(crate) primary_resets_at_unix_epoch: Option, + pub(crate) secondary_window_seconds: Option, + pub(crate) secondary_remaining_percent: Option, + pub(crate) secondary_resets_at_unix_epoch: Option, + pub(crate) credits_has_credits: Option, + pub(crate) credits_unlimited: Option, + pub(crate) credits_balance: Option, + pub(crate) rate_limit_reached_type: Option, + pub(crate) cooldown_until_unix_epoch: Option, + pub(crate) note: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RunActivityMarker { + run_id: String, + attempt_number: i64, + process_id: Option, + last_activity_unix_epoch: Option, + last_protocol_activity_unix_epoch: Option, + last_progress_unix_epoch: Option, + current_operation: Option, + thread_id: Option, + turn_id: Option, + thread_status: Option, + thread_active_flags: Vec, + event_count: Option, + last_event_type: Option, + effective_model: Option, + effective_model_provider: Option, + effective_cwd: Option, + effective_approval_policy: Option, + effective_approvals_reviewer: Option, + effective_sandbox_mode: Option, + child_agent_activity: Option, + protocol_activity: Option, + account: Option, + accounts: Vec, + retry_budget_attempt_count: Option, + retry_kind: Option, + retry_ready_at_unix_epoch: Option, + review_policy_phase: Option, + review_policy_status: Option, + review_policy_head_sha: Option, + review_policy_nonclean_rounds: Option, +} +impl RunActivityMarker { + pub(crate) fn run_id(&self) -> &str { + &self.run_id + } + + pub(crate) fn attempt_number(&self) -> i64 { + self.attempt_number + } + + pub(crate) fn process_id(&self) -> Option { + self.process_id + } + + pub(crate) fn last_activity_unix_epoch(&self) -> Option { + self.last_activity_unix_epoch + } + + pub(crate) fn last_protocol_activity_unix_epoch(&self) -> Option { + self.last_protocol_activity_unix_epoch + } + + pub(crate) fn last_progress_unix_epoch(&self) -> Option { + self.last_progress_unix_epoch + } + + pub(crate) fn current_operation(&self) -> Option<&str> { + self.current_operation.as_deref() + } + + pub(crate) fn thread_id(&self) -> Option<&str> { + self.thread_id.as_deref() + } + + pub(crate) fn turn_id(&self) -> Option<&str> { + self.turn_id.as_deref() + } + + pub(crate) fn thread_status(&self) -> Option<&str> { + self.thread_status.as_deref() + } + + pub(crate) fn thread_active_flags(&self) -> &[String] { + &self.thread_active_flags + } + + pub(crate) fn event_count(&self) -> i64 { + self.event_count.unwrap_or(0) + } + + pub(crate) fn last_event_type(&self) -> Option<&str> { + self.last_event_type.as_deref() + } + + pub(crate) fn effective_model(&self) -> Option<&str> { + self.effective_model.as_deref() + } + + pub(crate) fn effective_model_provider(&self) -> Option<&str> { + self.effective_model_provider.as_deref() + } + + pub(crate) fn effective_cwd(&self) -> Option<&str> { + self.effective_cwd.as_deref() + } + + pub(crate) fn effective_approval_policy(&self) -> Option<&str> { + self.effective_approval_policy.as_deref() + } + + pub(crate) fn effective_approvals_reviewer(&self) -> Option<&str> { + self.effective_approvals_reviewer.as_deref() + } + + pub(crate) fn effective_sandbox_mode(&self) -> Option<&str> { + self.effective_sandbox_mode.as_deref() + } + + pub(crate) fn child_agent_activity(&self) -> Option<&ChildAgentActivitySummary> { + self.child_agent_activity.as_ref() + } + + pub(crate) fn protocol_activity(&self) -> Option<&ProtocolActivitySummary> { + self.protocol_activity.as_ref() + } + + pub(crate) fn account(&self) -> Option<&CodexAccountActivitySummary> { + self.account.as_ref() + } + + pub(crate) fn accounts(&self) -> &[CodexAccountActivitySummary] { + &self.accounts + } + + pub(crate) fn retry_kind(&self) -> Option<&str> { + self.retry_kind.as_deref() + } + + pub(crate) fn retry_ready_at_unix_epoch(&self) -> Option { + self.retry_ready_at_unix_epoch + } + + pub(crate) fn retry_budget_attempt_count(&self) -> Option { + self.retry_budget_attempt_count + } + + pub(crate) fn review_policy_phase(&self) -> Option<&str> { + self.review_policy_phase.as_deref() + } + + pub(crate) fn review_policy_status(&self) -> Option<&str> { + self.review_policy_status.as_deref() + } + + pub(crate) fn review_policy_head_sha(&self) -> Option<&str> { + self.review_policy_head_sha.as_deref() + } + + pub(crate) fn review_policy_nonclean_rounds(&self) -> Option { + self.review_policy_nonclean_rounds + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReviewHandoffMarker { + run_id: String, + attempt_number: i64, + branch_name: String, + pr_url: String, + target_base_ref_name: Option, + pr_head_ref_name: String, + pr_head_oid: String, +} +impl ReviewHandoffMarker { + pub(crate) fn new( + run_id: impl Into, + attempt_number: i64, + branch_name: impl Into, + pr_url: impl Into, + target_base_ref_name: impl Into, + pr_head_ref_name: impl Into, + pr_head_oid: impl Into, + ) -> Self { + Self { + run_id: run_id.into(), + attempt_number, + branch_name: branch_name.into(), + pr_url: pr_url.into(), + target_base_ref_name: Some(target_base_ref_name.into()), + pr_head_ref_name: pr_head_ref_name.into(), + pr_head_oid: pr_head_oid.into(), + } + } + + pub(crate) fn branch_name(&self) -> &str { + &self.branch_name + } + + pub(crate) fn run_id(&self) -> &str { + &self.run_id + } + + pub(crate) fn attempt_number(&self) -> i64 { + self.attempt_number + } + + pub(crate) fn pr_url(&self) -> &str { + &self.pr_url + } + + pub(crate) fn target_base_ref_name(&self) -> Option<&str> { + self.target_base_ref_name.as_deref() + } + + pub(crate) fn pr_head_oid(&self) -> &str { + &self.pr_head_oid + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReviewOrchestrationMarker { + run_id: String, + attempt_number: i64, + branch_name: String, + pr_url: String, + head_sha: String, + phase: String, + request_comment_database_id: Option, + request_created_at_unix_epoch: Option, + request_description_thumbs_up_count: Option, + request_retry_count: i64, + external_round_count: i64, + auto_merge_enabled_at_unix_epoch: Option, +} +impl ReviewOrchestrationMarker { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + run_id: impl Into, + attempt_number: i64, + branch_name: impl Into, + pr_url: impl Into, + head_sha: impl Into, + phase: impl Into, + request_comment_database_id: Option, + request_created_at_unix_epoch: Option, + request_description_thumbs_up_count: Option, + request_retry_count: i64, + external_round_count: i64, + auto_merge_enabled_at_unix_epoch: Option, + ) -> Self { + Self { + run_id: run_id.into(), + attempt_number, + branch_name: branch_name.into(), + pr_url: pr_url.into(), + head_sha: head_sha.into(), + phase: phase.into(), + request_comment_database_id, + request_created_at_unix_epoch, + request_description_thumbs_up_count, + request_retry_count, + external_round_count, + auto_merge_enabled_at_unix_epoch, + } + } + + pub(crate) fn branch_name(&self) -> &str { + &self.branch_name + } + + pub(crate) fn run_id(&self) -> &str { + &self.run_id + } + + pub(crate) fn attempt_number(&self) -> i64 { + self.attempt_number + } + + pub(crate) fn pr_url(&self) -> &str { + &self.pr_url + } + + pub(crate) fn head_sha(&self) -> &str { + &self.head_sha + } + + pub(crate) fn phase(&self) -> &str { + &self.phase + } + + pub(crate) fn request_comment_database_id(&self) -> Option { + self.request_comment_database_id + } + + pub(crate) fn request_created_at_unix_epoch(&self) -> Option { + self.request_created_at_unix_epoch + } + + #[cfg(test)] + pub(crate) fn request_description_thumbs_up_count(&self) -> Option { + self.request_description_thumbs_up_count + } + + pub(crate) fn request_retry_count(&self) -> i64 { + self.request_retry_count + } + + pub(crate) fn external_round_count(&self) -> i64 { + self.external_round_count + } + + pub(crate) fn auto_merge_enabled_at_unix_epoch(&self) -> Option { + self.auto_merge_enabled_at_unix_epoch + } +} diff --git a/apps/decodex/src/state/store.rs b/apps/decodex/src/state/store.rs new file mode 100644 index 00000000..63d0f707 --- /dev/null +++ b/apps/decodex/src/state/store.rs @@ -0,0 +1,1236 @@ +use std::mem; + +/// Local runtime store for leases, attempts, worktrees, and protocol events. +#[derive(Default)] +pub struct StateStore { + inner: Mutex, + sqlite: Option>, +} +impl StateStore { + /// Open the local persistent runtime store. + pub fn open(path: impl AsRef) -> Result { + let sqlite = SqliteStateStore::open(path.as_ref())?; + let state = sqlite.load_state()?; + + Ok(Self { inner: Mutex::new(state), sqlite: Some(Mutex::new(sqlite)) }) + } + + /// Open an in-memory runtime store for tests. + pub fn open_in_memory() -> Result { + Ok(Self::default()) + } + + /// Create or replace a registered project row in the local control-plane registry. + pub(crate) fn upsert_project(&self, registration: &ProjectRegistration) -> Result<()> { + let mut state = self.lock()?; + + state + .projects + .insert(registration.service_id().to_owned(), registration.clone()); + + self.persist_runtime_state_locked(&state) + } + + /// List all registered projects known to this local Decodex installation. + pub(crate) fn list_projects(&self) -> Result> { + let state = self.lock()?; + let mut projects = state.projects.values().cloned().collect::>(); + + projects.sort_by(|left, right| left.service_id().cmp(right.service_id())); + + Ok(projects) + } + + /// Enable or disable one registered project. + pub(crate) fn set_project_enabled(&self, service_id: &str, enabled: bool) -> Result<()> { + let mut state = self.lock()?; + let project = state + .projects + .get_mut(service_id) + .ok_or_else(|| eyre::eyre!("Decodex project `{service_id}` is not registered."))?; + + project.set_enabled(enabled); + + self.persist_runtime_state_locked(&state) + } + + /// Configure the shared cross-process dispatch-slot root for one project. + pub fn configure_dispatch_slot_root( + &self, + project_id: &str, + worktree_root: impl AsRef, + slot_limit: u32, + ) -> Result<()> { + let mut state = self.lock()?; + + state.dispatch_slot_configs.insert( + project_id.to_owned(), + DispatchSlotConfig { + root: worktree_root.as_ref().to_path_buf(), + slot_limit: usize::try_from(slot_limit) + .map_err(|_error| eyre::eyre!("dispatch slot limit overflowed usize"))?, + }, + ); + + Ok(()) + } + + /// Retarget runtime records from a visible issue identifier to the canonical tracker id. + pub fn canonicalize_issue_identity( + &self, + previous_issue_id: &str, + canonical_issue_id: &str, + ) -> Result<()> { + if previous_issue_id == canonical_issue_id { + return Ok(()); + } + + let mut state = self.lock()?; + + if let Some(mut lease) = state.leases.remove(previous_issue_id) { + lease.issue_id = canonical_issue_id.to_owned(); + + state.leases.entry(canonical_issue_id.to_owned()).or_insert(lease); + } + if let Some(mut mapping) = state.worktrees.remove(previous_issue_id) { + mapping.issue_id = canonical_issue_id.to_owned(); + + state.worktrees.entry(canonical_issue_id.to_owned()).or_insert(mapping); + } + + retarget_review_handoff_issue( + &mut state.review_handoffs, + previous_issue_id, + canonical_issue_id, + ); + retarget_review_orchestration_issue( + &mut state.review_orchestrations, + previous_issue_id, + canonical_issue_id, + ); + + if let Some(guard) = state.issue_claim_guards.remove(previous_issue_id) { + state.issue_claim_guards.entry(canonical_issue_id.to_owned()).or_insert(guard); + } + if let Some(guard) = state.dispatch_slot_guards.remove(previous_issue_id) { + state.dispatch_slot_guards.entry(canonical_issue_id.to_owned()).or_insert(guard); + } + + for attempt in state + .run_attempts + .values_mut() + .filter(|attempt| attempt.issue_id == previous_issue_id) + { + attempt.issue_id = canonical_issue_id.to_owned(); + } + + self.persist_runtime_state_locked(&state)?; + + self.delete_previous_issue_identity_locked(previous_issue_id) + } + + /// Create or replace the active lease for one issue. + pub fn upsert_lease( + &self, + project_id: &str, + issue_id: &str, + run_id: &str, + issue_state: &str, + ) -> Result<()> { + let mut state = self.lock()?; + + state.leases.insert( + issue_id.to_owned(), + IssueLease { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + run_id: run_id.to_owned(), + issue_state: issue_state.to_owned(), + }, + ); + state.remember_run_project(project_id, issue_id, Some(run_id)); + + self.persist_runtime_state_locked(&state) + } + + /// Try to acquire one issue claim plus one shared dispatch slot for one issue. + pub fn try_acquire_lease( + &self, + project_id: &str, + issue_id: &str, + run_id: &str, + issue_state: &str, + ) -> Result { + let mut state = self.lock()?; + + if state.leases.values().any(|lease| lease.issue_id == issue_id) { + return Ok(false); + } + + if let Some(dispatch_slot_config) = state.dispatch_slot_configs.get(project_id).cloned() { + fs::create_dir_all(&dispatch_slot_config.root)?; + + let issue_claim_lock_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(issue_claim_lock_path(&dispatch_slot_config.root, issue_id))?; + + match issue_claim_lock_file.try_lock() { + Ok(()) => {}, + Err(TryLockError::WouldBlock) => return Ok(false), + Err(TryLockError::Error(error)) => return Err(error.into()), + } + + let mut issue_claim_guard = + IssueClaimGuard { lock_file: issue_claim_lock_file, retention: GuardRetention::Local }; + + write_issue_claim_record( + &mut issue_claim_guard.lock_file, + project_id, + issue_id, + run_id, + issue_state, + )?; + + let held_slot_indexes = state + .dispatch_slot_guards + .values() + .filter(|guard| guard.project_id == project_id) + .map(|guard| guard.slot_index) + .collect::>(); + let mut acquired_guard = None; + + for slot_index in 0..dispatch_slot_config.slot_limit { + if held_slot_indexes.contains(&slot_index) { + continue; + } + + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(dispatch_slot_lock_path(&dispatch_slot_config.root, slot_index))?; + + match lock_file.try_lock() { + Ok(()) => { + acquired_guard = Some(DispatchSlotGuard { + project_id: project_id.to_owned(), + slot_index, + lock_file, + retention: GuardRetention::Local, + }); + + break; + }, + Err(TryLockError::WouldBlock) => continue, + Err(TryLockError::Error(error)) => return Err(error.into()), + } + } + + let Some(dispatch_slot_guard) = acquired_guard else { + issue_claim_guard.unlock()?; + + return Ok(false); + }; + + state.issue_claim_guards.insert(issue_id.to_owned(), issue_claim_guard); + state.dispatch_slot_guards.insert(issue_id.to_owned(), dispatch_slot_guard); + } + + state.leases.insert( + issue_id.to_owned(), + IssueLease { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + run_id: run_id.to_owned(), + issue_state: issue_state.to_owned(), + }, + ); + state.remember_run_project(project_id, issue_id, Some(run_id)); + self.persist_runtime_state_locked(&state)?; + + Ok(true) + } + + /// Read the active lease for one issue. + pub fn lease_for_issue(&self, issue_id: &str) -> Result> { + let state = self.lock()?; + + Ok(state.leases.get(issue_id).cloned()) + } + + /// List all active leases. + pub fn list_leases(&self, project_id: &str) -> Result> { + let state = self.lock()?; + let mut leases = state + .leases + .values() + .filter(|lease| lease.project_id == project_id) + .cloned() + .collect::>(); + + leases.sort_by(|left, right| left.issue_id.cmp(&right.issue_id)); + + Ok(leases) + } + + /// List all active shared leases by combining local claims with other processes' issue claims. + pub fn list_active_shared_leases(&self, project_id: &str) -> Result> { + let (mut leases_by_issue, dispatch_slot_config) = { + let state = self.lock()?; + let leases = state + .leases + .values() + .filter(|lease| lease.project_id == project_id) + .cloned() + .map(|lease| (lease.issue_id.clone(), lease)) + .collect::>(); + + (leases, state.dispatch_slot_configs.get(project_id).cloned()) + }; + let Some(dispatch_slot_config) = dispatch_slot_config else { + let mut leases = leases_by_issue.into_values().collect::>(); + + leases.sort_by(|left, right| left.issue_id.cmp(&right.issue_id)); + + return Ok(leases); + }; + let read_dir = match fs::read_dir(&dispatch_slot_config.root) { + Ok(read_dir) => read_dir, + Err(error) if error.kind() == ErrorKind::NotFound => { + let mut leases = leases_by_issue.into_values().collect::>(); + + leases.sort_by(|left, right| left.issue_id.cmp(&right.issue_id)); + + return Ok(leases); + }, + Err(error) => return Err(error.into()), + }; + + for entry in read_dir { + let entry = entry?; + let path = entry.path(); + let Some(issue_id) = issue_claim_id_from_path(&path) else { + continue; + }; + + if leases_by_issue.contains_key(&issue_id) { + continue; + } + + let claim_lock_file = match OpenOptions::new() + .read(true) + .write(true) + .create(false) + .truncate(false) + .open(&path) + { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => continue, + Err(error) => return Err(error.into()), + }; + + match claim_lock_file.try_lock() { + Ok(()) => claim_lock_file.unlock()?, + Err(TryLockError::WouldBlock) => { + if let Some(lease) = read_issue_claim_record(&path)? + && lease.project_id == project_id + { + leases_by_issue.insert(issue_id, lease); + } + }, + Err(TryLockError::Error(error)) => return Err(error.into()), + } + } + + let mut leases = leases_by_issue.into_values().collect::>(); + + leases.sort_by(|left, right| left.issue_id.cmp(&right.issue_id)); + + Ok(leases) + } + + /// Report whether one issue is actively claimed by this or another process. + pub fn issue_has_active_shared_claim(&self, project_id: &str, issue_id: &str) -> Result { + let state = self.lock()?; + + if state.leases.contains_key(issue_id) { + return Ok(true); + } + + let Some(dispatch_slot_config) = state.dispatch_slot_configs.get(project_id).cloned() + else { + return Ok(false); + }; + + drop(state); + + let path = issue_claim_lock_path(&dispatch_slot_config.root, issue_id); + let claim_lock_file = match OpenOptions::new() + .read(true) + .write(true) + .create(false) + .truncate(false) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + + match claim_lock_file.try_lock() { + Ok(()) => { + claim_lock_file.unlock()?; + + Ok(false) + }, + Err(TryLockError::WouldBlock) => Ok(true), + Err(TryLockError::Error(error)) => Err(error.into()), + } + } + + /// Remove the active lease for one issue. + pub fn clear_lease(&self, issue_id: &str) -> Result<()> { + let mut state = self.lock()?; + let removed_lease = state.leases.remove(issue_id).is_some(); + + if let Some(guard) = state.issue_claim_guards.remove(issue_id) { + guard.release_for_clear()?; + } + if let Some(guard) = state.dispatch_slot_guards.remove(issue_id) { + guard.release_for_clear()?; + } + + if removed_lease { + self.persist_runtime_state_locked(&state)?; + } + + self.delete_lease_locked(issue_id) + } + + /// Drop the current process-local dispatch-slot guard while keeping the local lease record. + pub fn release_dispatch_slot(&self, issue_id: &str) -> Result<()> { + let mut state = self.lock()?; + + state.dispatch_slot_guards.remove(issue_id); + + Ok(()) + } + + /// Drop process-local lock guards after another process inherited them. + pub fn release_handed_off_guards(&self, issue_id: &str) -> Result<()> { + let mut state = self.lock()?; + + state.issue_claim_guards.remove(issue_id); + state.dispatch_slot_guards.remove(issue_id); + + Ok(()) + } + + /// Duplicate the held dispatch-slot lock so a spawned child can inherit it across exec. + #[cfg(unix)] + pub fn clone_issue_claim_for_child(&self, issue_id: &str) -> Result { + let mut state = self.lock()?; + let guard = state + .issue_claim_guards + .get_mut(issue_id) + .ok_or_else(|| eyre::eyre!("issue `{issue_id}` does not hold an issue-claim guard"))?; + + guard.retention = GuardRetention::ParentAfterHandoff; + + let child_lock = guard.lock_file.try_clone()?; + + clear_close_on_exec(&child_lock)?; + + Ok(child_lock) + } + + /// Duplicate the held dispatch-slot lock so a spawned child can inherit it across exec. + #[cfg(unix)] + pub fn clone_dispatch_slot_for_child(&self, issue_id: &str) -> Result<(File, usize)> { + let mut state = self.lock()?; + let guard = state + .dispatch_slot_guards + .get_mut(issue_id) + .ok_or_else(|| eyre::eyre!("issue `{issue_id}` does not hold a dispatch-slot guard"))?; + + guard.retention = GuardRetention::ParentAfterHandoff; + + let child_lock = guard.lock_file.try_clone()?; + + clear_close_on_exec(&child_lock)?; + + Ok((child_lock, guard.slot_index)) + } + + /// Adopt an inherited dispatch-slot fd and local lease for a daemon child process. + #[cfg(unix)] + pub fn adopt_preacquired_lease( + &self, + project_id: &str, + issue_id: &str, + run_id: &str, + issue_state: &str, + guards: PreacquiredLeaseGuards, + ) -> Result<()> { + let issue_claim_lock_file = unsafe { File::from_raw_fd(guards.issue_claim_fd) }; + let lock_file = unsafe { File::from_raw_fd(guards.dispatch_slot_fd) }; + + set_close_on_exec(&issue_claim_lock_file)?; + set_close_on_exec(&lock_file)?; + + let mut state = self.lock()?; + + state.issue_claim_guards.insert( + issue_id.to_owned(), + IssueClaimGuard { + lock_file: issue_claim_lock_file, + retention: GuardRetention::AdoptingChild, + }, + ); + state.dispatch_slot_guards.insert( + issue_id.to_owned(), + DispatchSlotGuard { + project_id: project_id.to_owned(), + slot_index: guards.dispatch_slot_index, + lock_file, + retention: GuardRetention::AdoptingChild, + }, + ); + state.leases.insert( + issue_id.to_owned(), + IssueLease { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + run_id: run_id.to_owned(), + issue_state: issue_state.to_owned(), + }, + ); + state.remember_run_project(project_id, issue_id, Some(run_id)); + + self.persist_runtime_state_locked(&state) + } + + /// Insert or update a run attempt record. + pub fn record_run_attempt( + &self, + run_id: &str, + issue_id: &str, + attempt_number: i64, + status: &str, + ) -> Result<()> { + let now = timestamp_parts(); + let mut state = self.lock_without_refresh()?; + let project_id = state.project_id_for_run(issue_id, run_id); + + match state.run_attempts.get_mut(run_id) { + Some(existing) => { + let retained_project_id = + (existing.issue_id == issue_id).then(|| existing.project_id.clone()).flatten(); + + existing.issue_id = issue_id.to_owned(); + existing.project_id = project_id.or(retained_project_id); + existing.attempt_number = attempt_number; + existing.status = status.to_owned(); + existing.updated_at = now.text.clone(); + existing.updated_at_unix = now.unix; + }, + None => { + state.run_attempts.insert( + run_id.to_owned(), + RunAttemptRecord { + run_id: run_id.to_owned(), + project_id, + issue_id: issue_id.to_owned(), + attempt_number, + status: status.to_owned(), + thread_id: None, + turn_id: None, + updated_at: now.text, + updated_at_unix: now.unix, + }, + ); + }, + } + + let attempt = state + .run_attempts + .get(run_id) + .ok_or_else(|| eyre::eyre!("Run attempt `{run_id}` was not recorded."))? + .clone(); + + self.upsert_run_attempt_locked(&attempt) + } + + /// Compute the next attempt number for one issue. + pub fn next_attempt_number(&self, issue_id: &str) -> Result { + let state = self.lock()?; + let next_attempt = state + .run_attempts + .values() + .filter(|attempt| attempt.issue_id == issue_id) + .map(|attempt| attempt.attempt_number) + .max() + .unwrap_or(0) + + 1; + + Ok(next_attempt) + } + + /// Count attempts that consume the retry budget for one issue. + pub fn retry_budget_attempt_count(&self, issue_id: &str) -> Result { + let state = self.lock()?; + let retry_budget_attempts = state + .run_attempts + .values() + .filter(|attempt| { + attempt.issue_id == issue_id + && matches!( + attempt.status.as_str(), + "failed" | "interrupted" | "terminal_guarded" + ) + }) + .count() as i64; + + Ok(retry_budget_attempts) + } + + /// Return whether a later attempt for one issue consumed retry budget. + pub fn issue_has_retry_budget_attempt_after( + &self, + issue_id: &str, + attempt_number: i64, + ) -> Result { + let state = self.lock()?; + + Ok(state.run_attempts.values().any(|attempt| { + attempt.issue_id == issue_id + && attempt.attempt_number > attempt_number + && matches!( + attempt.status.as_str(), + "failed" | "interrupted" | "terminal_guarded" + ) + })) + } + + /// Attach the active thread identifier to a run attempt. + pub fn update_run_thread(&self, run_id: &str, thread_id: &str) -> Result<()> { + let now = timestamp_parts(); + let mut state = self.lock_without_refresh()?; + + if let Some(attempt) = state.run_attempts.get_mut(run_id) { + attempt.thread_id = Some(thread_id.to_owned()); + attempt.updated_at = now.text; + attempt.updated_at_unix = now.unix; + + let attempt = attempt.clone(); + + return self.upsert_run_attempt_locked(&attempt); + } + + Ok(()) + } + + /// Attach the active turn identifier to a run attempt. + pub fn update_run_turn(&self, run_id: &str, turn_id: &str) -> Result<()> { + let now = timestamp_parts(); + let mut state = self.lock_without_refresh()?; + + if let Some(attempt) = state.run_attempts.get_mut(run_id) { + attempt.turn_id = Some(turn_id.to_owned()); + attempt.updated_at = now.text; + attempt.updated_at_unix = now.unix; + + let attempt = attempt.clone(); + + return self.upsert_run_attempt_locked(&attempt); + } + + Ok(()) + } + + /// Update the status for one run attempt. + pub fn update_run_status(&self, run_id: &str, status: &str) -> Result<()> { + let now = timestamp_parts(); + let mut state = self.lock_without_refresh()?; + + if let Some(attempt) = state.run_attempts.get_mut(run_id) { + attempt.status = status.to_owned(); + attempt.updated_at = now.text; + attempt.updated_at_unix = now.unix; + + let attempt = attempt.clone(); + + return self.upsert_run_attempt_locked(&attempt); + } + + Ok(()) + } + + /// Mark all active run attempts for one issue as succeeded. + pub fn succeed_active_run_attempts_for_issue(&self, issue_id: &str) -> Result { + let now = timestamp_parts(); + let mut state = self.lock()?; + let mut updated_count = 0; + + for attempt in state + .run_attempts + .values_mut() + .filter(|attempt| attempt.issue_id == issue_id) + .filter(|attempt| active_run_attempt_status(&attempt.status)) + { + attempt.status = "succeeded".to_owned(); + attempt.updated_at = now.text.clone(); + attempt.updated_at_unix = now.unix; + updated_count += 1; + } + + if updated_count > 0 { + self.persist_runtime_state_locked(&state)?; + } + + Ok(updated_count) + } + + /// Read one run attempt. + pub fn run_attempt(&self, run_id: &str) -> Result> { + let state = self.lock()?; + + Ok(state.run_attempts.get(run_id).map(RunAttemptRecord::as_public)) + } + + /// Read one run attempt by issue and attempt number. + pub fn run_attempt_for_issue_attempt( + &self, + issue_id: &str, + attempt_number: i64, + ) -> Result> { + let state = self.lock()?; + let attempt = state + .run_attempts + .values() + .filter(|attempt| { + attempt.issue_id == issue_id && attempt.attempt_number == attempt_number + }) + .max_by(|left, right| compare_attempt_records(left, right)) + .map(RunAttemptRecord::as_public); + + Ok(attempt) + } + + /// Read the latest run attempt for one issue. + pub fn latest_run_attempt_for_issue(&self, issue_id: &str) -> Result> { + let state = self.lock()?; + let attempt = state + .run_attempts + .values() + .filter(|attempt| attempt.issue_id == issue_id) + .max_by(|left, right| compare_attempt_records(left, right)) + .map(RunAttemptRecord::as_public); + + Ok(attempt) + } + + /// List recent run attempts for one project, including lease and protocol summary fields. + pub fn list_recent_runs( + &self, + project_id: &str, + limit: usize, + ) -> Result> { + let state = self.lock()?; + let mut runs = state + .run_attempts + .values() + .filter_map(|attempt| state.project_run_status(project_id, attempt)) + .collect::>(); + + runs.sort_by(compare_project_run_status); + runs.truncate(limit); + + Ok(runs) + } + + /// List all active leased runs for one project without applying the recent-run limit. + pub fn list_active_runs(&self, project_id: &str) -> Result> { + let state = self.lock()?; + let mut runs = state + .run_attempts + .values() + .filter_map(|attempt| { + let status = state.project_run_status(project_id, attempt)?; + + status.active_lease.then_some(status) + }) + .collect::>(); + + runs.sort_by(compare_project_run_status); + + Ok(runs) + } + + /// Append one protocol event to the journal for a run. + pub fn append_event( + &self, + run_id: &str, + sequence_number: i64, + event_type: &str, + _payload: &str, + ) -> Result<()> { + let mut state = self.lock_without_refresh()?; + + if state + .events + .get(run_id) + .is_some_and(|events| events.iter().any(|event| event.sequence_number == sequence_number)) + { + eyre::bail!( + "Protocol event `{run_id}` sequence `{sequence_number}` already exists in the runtime journal." + ); + } + + let now = timestamp_parts(); + let event = ProtocolEventRecord { + sequence_number, + event_type: event_type.to_owned(), + created_at: now.text, + created_at_unix: now.unix, + }; + + if !self.append_protocol_event_locked(run_id, &event)? { + eyre::bail!( + "Protocol event `{run_id}` sequence `{sequence_number}` already exists in the runtime journal." + ); + } + + state + .event_summaries + .entry(run_id.to_owned()) + .or_default() + .record_event(&event); + + let events = state.events.entry(run_id.to_owned()).or_default(); + + events.push(event); + events.sort_by_key(|event| event.sequence_number); + + Ok(()) + } + + /// Persist a locally known Linear execution event in the runtime store. + pub(crate) fn record_linear_execution_event( + &self, + record: &LinearExecutionEventRecord, + ) -> Result { + records::validate_linear_execution_event_record(record).map_err(|error| eyre::eyre!(error))?; + + let now = timestamp_parts(); + let mut state = self.lock_without_refresh()?; + let idempotency_key = record.idempotency_key.clone(); + let is_new = !state.linear_execution_events.contains_key(&idempotency_key); + let runtime_record = LinearExecutionEventRuntimeRecord { + record: record.clone(), + event_unix: parse_linear_execution_event_unix(record), + recorded_at: now.text, + recorded_at_unix: now.unix, + }; + + state.linear_execution_events.insert(idempotency_key, runtime_record.clone()); + self.upsert_linear_execution_event_locked(&runtime_record)?; + + Ok(is_new) + } + + /// List locally cached Linear execution events for one issue lane. + pub(crate) fn list_linear_execution_events( + &self, + service_id: &str, + issue_id: &str, + ) -> Result> { + let state = self.lock()?; + let mut records = state + .linear_execution_events + .values() + .filter(|record| { + record.record.service_id == service_id && record.record.issue_id == issue_id + }) + .cloned() + .collect::>(); + + records.sort_by(compare_linear_execution_event_runtime_records); + + Ok(records.into_iter().map(|record| record.record).collect()) + } + + /// Count protocol journal records for one run. + pub fn event_count(&self, run_id: &str) -> Result { + let state = self.lock()?; + + Ok(state.protocol_event_summary(run_id).event_count) + } + + /// Read the latest recorded activity timestamp for one run as a Unix epoch. + pub fn last_run_activity_unix_epoch(&self, run_id: &str) -> Result> { + let state = self.lock()?; + let last_activity = state.run_attempts.get(run_id).map(|attempt| attempt.updated_at_unix); + let last_event = state.protocol_event_summary(run_id).last_event_at_unix; + + Ok(match (last_activity, last_event) { + (Some(run_activity), Some(event_activity)) => Some(run_activity.max(event_activity)), + (Some(run_activity), None) => Some(run_activity), + (None, Some(event_activity)) => Some(event_activity), + (None, None) => None, + }) + } + + /// Read the latest recorded protocol-event timestamp for one run as a Unix epoch. + pub fn last_protocol_activity_unix_epoch(&self, run_id: &str) -> Result> { + let state = self.lock()?; + + Ok(state.protocol_event_summary(run_id).last_event_at_unix) + } + + /// Create or replace the worktree mapping for one issue. + pub fn upsert_worktree( + &self, + project_id: &str, + issue_id: &str, + branch_name: &str, + worktree_path: &str, + ) -> Result<()> { + let mut state = self.lock()?; + + state.worktrees.insert( + issue_id.to_owned(), + WorktreeMappingRecord { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: branch_name.to_owned(), + worktree_path: PathBuf::from(worktree_path), + }, + ); + state.remember_run_project(project_id, issue_id, None); + + self.persist_runtime_state_locked(&state) + } + + /// Create or replace the retained review handoff marker for one issue lane. + pub(crate) fn upsert_review_handoff_marker( + &self, + project_id: &str, + issue_id: &str, + marker: &ReviewHandoffMarker, + ) -> Result<()> { + let now = timestamp_parts(); + let key = ReviewMarkerKey::new(project_id, issue_id, marker.branch_name()); + let mut state = self.lock()?; + + state.review_handoffs.insert( + key, + ReviewHandoffRuntimeRecord { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: marker.branch_name().to_owned(), + marker: marker.clone(), + updated_at: now.text, + updated_at_unix: now.unix, + }, + ); + + self.persist_runtime_state_locked(&state) + } + + /// Read the retained review handoff marker for one issue branch from the runtime DB. + pub(crate) fn review_handoff_marker( + &self, + project_id: &str, + issue_id: &str, + branch_name: &str, + ) -> Result> { + let state = self.lock()?; + let key = ReviewMarkerKey::new(project_id, issue_id, branch_name); + + Ok(state.review_handoffs.get(&key).map(|record| record.marker.clone())) + } + + /// Create or replace the retained review orchestration marker for one issue lane. + pub(crate) fn upsert_review_orchestration_marker( + &self, + project_id: &str, + issue_id: &str, + marker: &ReviewOrchestrationMarker, + ) -> Result<()> { + let now = timestamp_parts(); + let key = ReviewOrchestrationKey::new( + project_id, + issue_id, + marker.branch_name(), + marker.run_id(), + marker.attempt_number(), + ); + let mut state = self.lock()?; + + state.review_orchestrations.insert( + key, + ReviewOrchestrationRuntimeRecord { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: marker.branch_name().to_owned(), + run_id: marker.run_id().to_owned(), + attempt_number: marker.attempt_number(), + marker: marker.clone(), + updated_at: now.text, + updated_at_unix: now.unix, + }, + ); + + self.persist_runtime_state_locked(&state) + } + + /// Read retained review orchestration for the current handoff identity. + pub(crate) fn review_orchestration_marker( + &self, + project_id: &str, + issue_id: &str, + review_handoff: &ReviewHandoffMarker, + ) -> Result> { + let state = self.lock()?; + let key = ReviewOrchestrationKey::new( + project_id, + issue_id, + review_handoff.branch_name(), + review_handoff.run_id(), + review_handoff.attempt_number(), + ); + + Ok(state.review_orchestrations.get(&key).map(|record| record.marker.clone())) + } + + /// Remove retained review markers for one issue without clearing its worktree mapping. + pub(crate) fn clear_review_markers(&self, issue_id: &str) -> Result<()> { + let mut state = self.lock()?; + + state.review_handoffs.retain(|key, _record| key.issue_id != issue_id); + state + .review_orchestrations + .retain(|key, _record| key.issue_id != issue_id); + self.persist_runtime_state_locked(&state)?; + + self.delete_review_markers_locked(issue_id) + } + + /// Read the worktree mapping for one issue. + pub fn worktree_for_issue(&self, issue_id: &str) -> Result> { + let state = self.lock()?; + + Ok(state.worktrees.get(issue_id).map(WorktreeMappingRecord::as_public)) + } + + /// List all known worktree mappings. + pub fn list_worktrees(&self, project_id: &str) -> Result> { + let state = self.lock()?; + let mut mappings = state + .worktrees + .values() + .filter(|mapping| mapping.project_id == project_id) + .map(WorktreeMappingRecord::as_public) + .collect::>(); + + mappings.sort_by(|left, right| left.issue_id.cmp(&right.issue_id)); + + Ok(mappings) + } + + /// Remove the worktree mapping for one issue. + pub fn clear_worktree(&self, issue_id: &str) -> Result<()> { + let mut state = self.lock()?; + + state.worktrees.remove(issue_id); + state.review_handoffs.retain(|key, _record| key.issue_id != issue_id); + state + .review_orchestrations + .retain(|key, _record| key.issue_id != issue_id); + self.persist_runtime_state_locked(&state)?; + + self.delete_worktree_and_review_markers_locked(issue_id) + } + + fn lock_without_refresh(&self) -> Result> { + self.inner.lock().map_err(|_| eyre::eyre!("StateStore mutex is poisoned.")) + } + + fn lock(&self) -> Result> { + let mut state = self.lock_without_refresh()?; + + self.refresh_runtime_state_locked(&mut state)?; + + Ok(state) + } + + fn refresh_runtime_state_locked(&self, state: &mut StateData) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + let loaded = sqlite.load_state()?; + + state.replace_durable_state(loaded); + + Ok(()) + } + + fn persist_runtime_state_locked(&self, state: &StateData) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.persist_runtime_state(state) + } + + fn upsert_run_attempt_locked(&self, attempt: &RunAttemptRecord) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.upsert_run_attempt(attempt) + } + + fn append_protocol_event_locked( + &self, + run_id: &str, + event: &ProtocolEventRecord, + ) -> Result { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(true); + }; + let sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.append_protocol_event(run_id, event) + } + + fn upsert_linear_execution_event_locked( + &self, + record: &LinearExecutionEventRuntimeRecord, + ) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.upsert_linear_execution_event(record) + } + + fn delete_lease_locked(&self, issue_id: &str) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_lease(issue_id) + } + + fn delete_previous_issue_identity_locked(&self, previous_issue_id: &str) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_previous_issue_identity(previous_issue_id) + } + + fn delete_worktree_and_review_markers_locked(&self, issue_id: &str) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_worktree_and_review_markers(issue_id) + } + + fn delete_review_markers_locked(&self, issue_id: &str) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_review_markers(issue_id) + } +} + +fn retarget_review_handoff_issue( + records: &mut HashMap, + previous_issue_id: &str, + canonical_issue_id: &str, +) { + let existing = mem::take(records); + + for (key, mut record) in existing { + let next_issue_id = if key.issue_id == previous_issue_id { + canonical_issue_id + } else { + key.issue_id.as_str() + }; + + record.issue_id = next_issue_id.to_owned(); + + records.insert( + ReviewMarkerKey::new(&key.project_id, next_issue_id, &key.branch_name), + record, + ); + } +} + +fn active_run_attempt_status(status: &str) -> bool { + matches!(status, "starting" | "running") +} + +fn retarget_review_orchestration_issue( + records: &mut HashMap, + previous_issue_id: &str, + canonical_issue_id: &str, +) { + let existing = mem::take(records); + + for (key, mut record) in existing { + let next_issue_id = if key.issue_id == previous_issue_id { + canonical_issue_id + } else { + key.issue_id.as_str() + }; + + record.issue_id = next_issue_id.to_owned(); + + records.insert( + ReviewOrchestrationKey::new( + &key.project_id, + next_issue_id, + &key.branch_name, + &key.run_id, + key.attempt_number, + ), + record, + ); + } +} diff --git a/apps/decodex/src/state/tests.rs b/apps/decodex/src/state/tests.rs new file mode 100644 index 00000000..f3a54529 --- /dev/null +++ b/apps/decodex/src/state/tests.rs @@ -0,0 +1,1661 @@ +#[cfg(unix)] use std::os::fd::{AsRawFd, IntoRawFd}; +use std::{ + fs, + path::Path, + process, slice, + sync::{Arc, Barrier}, + thread, +}; + +#[cfg(unix)] use libc::{F_GETFD, FD_CLOEXEC}; +use tempfile::TempDir; + +use crate::{ + state::{ + self, ChildAgentActivitySummary, CodexAccountActivitySummary, CodexAccountMarker, + EffectiveRuntimeMarker, PreacquiredLeaseGuards, ProjectRegistration, + ProtocolActivityMarker, ProtocolActivitySummary, RUN_ACTIVITY_MARKER_FILE, + RUN_OPERATION_REPO_GATE, ReviewHandoffMarker, ReviewOrchestrationMarker, StateStore, + }, + tracker::records::{LinearExecutionEventIdentity, LinearExecutionEventRecord}, +}; + +const IN_PROGRESS_STATE: &str = "In Progress"; + +#[cfg(unix)] +fn fd_has_close_on_exec(fd: i32) -> bool { + let flags = unsafe { libc::fcntl(fd, F_GETFD) }; + + assert_ne!(flags, -1, "fcntl(F_GETFD) should succeed for test fd {fd}"); + + flags & FD_CLOEXEC != 0 +} + +#[test] +fn review_markers_roundtrip_preserve_required_fields() { + let store = StateStore::open_in_memory().expect("state store should open"); + let handoff = ReviewHandoffMarker::new( + "run-1", + 2, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "main", + "x/decodex-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + ); + + store + .upsert_review_handoff_marker("pubfi", "PUB-101", &handoff) + .expect("review handoff marker should persist"); + + let restored_handoff = store + .review_handoff_marker("pubfi", "PUB-101", "x/decodex-pub-101") + .expect("review handoff marker should read") + .expect("review handoff marker should exist"); + + assert_eq!(restored_handoff, handoff); + + let orchestration = ReviewOrchestrationMarker::new( + "run-1", + 2, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "waiting_for_ack", + Some(1_234), + Some(1_775_200_000), + Some(3), + 1, + 2, + Some(1_775_200_900), + ); + + store + .upsert_review_orchestration_marker("pubfi", "PUB-101", &orchestration) + .expect("review orchestration marker should persist"); + + let restored_orchestration = store + .review_orchestration_marker("pubfi", "PUB-101", &handoff) + .expect("review orchestration marker should read") + .expect("review orchestration marker should exist"); + + assert_eq!(restored_orchestration, orchestration); +} + +#[test] +fn missing_review_markers_return_absent() { + let store = StateStore::open_in_memory().expect("state store should open"); + let handoff = ReviewHandoffMarker::new( + "run-1", + 2, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "main", + "x/decodex-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + ); + + assert!( + store + .review_handoff_marker("pubfi", "PUB-101", "x/decodex-pub-101") + .expect("review handoff marker should read") + .is_none() + ); + assert!( + store + .review_orchestration_marker("pubfi", "PUB-101", &handoff) + .expect("review orchestration marker should read") + .is_none() + ); +} + +#[test] +fn persistent_review_markers_survive_stale_store_persist_and_are_visible() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let observer = StateStore::open(&state_path).expect("observer state store should open"); + let writer = StateStore::open(&state_path).expect("writer state store should open"); + let handoff = ReviewHandoffMarker::new( + "run-1", + 1, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "main", + "x/decodex-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + ); + let orchestration = ReviewOrchestrationMarker::new( + "run-1", + 1, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "request_pending", + None, + None, + None, + 0, + 0, + None, + ); + + writer + .upsert_review_handoff_marker("pubfi", "PUB-101", &handoff) + .expect("handoff marker should persist"); + writer + .upsert_review_orchestration_marker("pubfi", "PUB-101", &orchestration) + .expect("orchestration marker should persist"); + + let observed_handoff = observer + .review_handoff_marker("pubfi", "PUB-101", "x/decodex-pub-101") + .expect("observer should read handoff marker") + .expect("observer should see marker written by another store"); + + assert_eq!(observed_handoff, handoff); + + observer + .record_run_attempt("run-2", "PUB-202", 1, "running") + .expect("stale observer should persist unrelated runtime state"); + + let reopened = StateStore::open(&state_path).expect("reopened state store should open"); + + assert_eq!( + reopened + .review_handoff_marker("pubfi", "PUB-101", "x/decodex-pub-101") + .expect("reopened store should read handoff marker"), + Some(handoff.clone()) + ); + assert_eq!( + reopened + .review_orchestration_marker("pubfi", "PUB-101", &handoff) + .expect("reopened store should read orchestration marker"), + Some(orchestration) + ); + assert!( + reopened.run_attempt("run-2").expect("run attempt should read").is_some(), + "unrelated stale-store persist should still keep its own update" + ); +} + +#[test] +fn persistent_event_appenders_can_write_distinct_runs_concurrently() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let first = StateStore::open(&state_path).expect("first state store should open"); + let second = StateStore::open(&state_path).expect("second state store should open"); + + first.record_run_attempt("run-a", "PUB-101", 1, "running").expect("first run should record"); + second.record_run_attempt("run-b", "PUB-102", 1, "running").expect("second run should record"); + + let barrier = Arc::new(Barrier::new(2)); + let first_barrier = Arc::clone(&barrier); + let first_writer = thread::spawn(move || { + first_barrier.wait(); + + for sequence_number in 1..=40 { + first + .append_event("run-a", sequence_number, "item/agentMessage/delta", "{}") + .expect("first event writer should append"); + } + }); + let second_writer = thread::spawn(move || { + barrier.wait(); + + for sequence_number in 1..=40 { + second + .append_event("run-b", sequence_number, "item/agentMessage/delta", "{}") + .expect("second event writer should append"); + } + }); + + first_writer.join().expect("first event writer should finish"); + second_writer.join().expect("second event writer should finish"); + + let reopened = StateStore::open(&state_path).expect("reopened state store should open"); + + assert_eq!(reopened.event_count("run-a").expect("first event count should load"), 40); + assert_eq!(reopened.event_count("run-b").expect("second event count should load"), 40); +} + +#[test] +fn persistent_append_event_does_not_refresh_full_event_journal() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let first = StateStore::open(&state_path).expect("first state store should open"); + let second = StateStore::open(&state_path).expect("second state store should open"); + + first.record_run_attempt("run-a", "PUB-101", 1, "running").expect("first run should record"); + second.record_run_attempt("run-b", "PUB-102", 1, "running").expect("second run should record"); + second + .append_event("run-b", 1, "item/agentMessage/delta", "{}") + .expect("second store should append an unrelated event"); + first + .append_event("run-a", 1, "item/agentMessage/delta", "{}") + .expect("first store should append without full journal refresh"); + + let state = first.inner.lock().expect("test should inspect the local cache"); + + assert!( + !state.events.contains_key("run-b"), + "append_event should not refresh the full persistent event journal into the local cache" + ); + + drop(state); + + let reopened = StateStore::open(&state_path).expect("reopened state store should open"); + + assert_eq!(reopened.event_count("run-a").expect("first event count should load"), 1); + assert_eq!(reopened.event_count("run-b").expect("second event count should load"), 1); +} + +#[test] +fn persistent_run_attempt_update_does_not_refresh_full_event_journal() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let first = StateStore::open(&state_path).expect("first state store should open"); + let second = StateStore::open(&state_path).expect("second state store should open"); + + second.record_run_attempt("run-b", "PUB-102", 1, "running").expect("second run should record"); + second + .append_event("run-b", 1, "item/agentMessage/delta", "{}") + .expect("second store should append an unrelated event"); + first.record_run_attempt("run-a", "PUB-101", 1, "running").expect("first run should record"); + first.update_run_thread("run-a", "thread-a").expect("first run thread should update"); + first.update_run_turn("run-a", "turn-a").expect("first run turn should update"); + first.update_run_status("run-a", "succeeded").expect("first run status should update"); + + let state = first.inner.lock().expect("test should inspect the local cache"); + + assert!( + !state.events.contains_key("run-b"), + "run attempt updates should not refresh the full persistent event journal into the local cache" + ); + + drop(state); + + let reopened = StateStore::open(&state_path).expect("reopened state store should open"); + let attempt = reopened + .run_attempt("run-a") + .expect("run attempt lookup should succeed") + .expect("run attempt should persist"); + + assert_eq!(attempt.status(), "succeeded"); + assert_eq!(attempt.thread_id(), Some("thread-a")); + assert_eq!(attempt.turn_id(), Some("turn-a")); + assert_eq!(reopened.event_count("run-b").expect("second event count should load"), 1); +} + +#[test] +fn persistent_project_run_listing_does_not_refresh_full_event_journal() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let observer = StateStore::open(&state_path).expect("observer state store should open"); + let writer = StateStore::open(&state_path).expect("writer state store should open"); + + observer + .record_run_attempt("run-a", "PUB-101", 1, "running") + .expect("observer run should record"); + observer + .upsert_lease("pubfi", "PUB-101", "run-a", IN_PROGRESS_STATE) + .expect("observer lease should record"); + observer + .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") + .expect("observer worktree should record"); + observer.append_event("run-a", 1, "item/started", "{}").expect("observer event should append"); + writer.record_run_attempt("run-b", "PUB-102", 1, "running").expect("writer run should record"); + writer + .append_event("run-b", 1, "item/agentMessage/delta", "{}") + .expect("writer event should append"); + + let runs = observer.list_active_runs("pubfi").expect("active runs should load"); + + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].run_id(), "run-a"); + assert_eq!(runs[0].event_count(), 1); + assert_eq!(runs[0].last_event_type(), Some("item/started")); + + let state = observer.inner.lock().expect("test should inspect the local cache"); + + assert!( + state.events.is_empty(), + "operator run listing should refresh event summaries without materializing event rows" + ); + assert_eq!( + state + .event_summaries + .get("run-b") + .expect("unrelated persistent run should have a summary") + .event_count, + 1 + ); +} + +#[test] +fn manages_issue_leases() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("lease should be inserted"); + + let lease = store + .lease_for_issue("PUB-101") + .expect("lease read should succeed") + .expect("lease should exist"); + + assert_eq!(lease.issue_id(), "PUB-101"); + assert_eq!(lease.run_id(), "run-1"); + assert_eq!(lease.project_id(), "pubfi"); + assert_eq!(lease.issue_state(), IN_PROGRESS_STATE); + + store.clear_lease("PUB-101").expect("lease should be deleted"); + + assert!(store.lease_for_issue("PUB-101").expect("lease lookup should succeed").is_none()); +} + +#[test] +fn tracks_issue_specific_leases_without_project_limit() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + assert!( + store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first lease acquisition should succeed") + ); + assert!( + store + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("second lease acquisition should succeed for another issue") + ); + assert!( + !store + .try_acquire_lease("pubfi", "PUB-101", "run-3", IN_PROGRESS_STATE) + .expect("duplicate issue acquisition should be rejected") + ); + assert!( + store + .try_acquire_lease("other", "PUB-201", "run-4", IN_PROGRESS_STATE) + .expect("other project should still acquire its own slot") + ); +} + +#[test] +fn shared_dispatch_slots_honor_configured_limit_across_process_local_stores() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let store_one = StateStore::open_in_memory().expect("first store should open"); + let store_two = StateStore::open_in_memory().expect("second store should open"); + let store_three = StateStore::open_in_memory().expect("third store should open"); + + store_one + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("first store should configure dispatch slot root"); + store_two + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("second store should configure dispatch slot root"); + store_three + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("third store should configure dispatch slot root"); + + assert!( + store_one + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first shared lease acquisition should succeed") + ); + assert!( + store_two + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("second store should acquire the second shared slot") + ); + assert!( + !store_three + .try_acquire_lease("pubfi", "PUB-103", "run-3", IN_PROGRESS_STATE) + .expect("third store should observe the configured shared slots as busy") + ); + + store_one.clear_lease("PUB-101").expect("shared lease should clear"); + + assert!( + store_three + .try_acquire_lease("pubfi", "PUB-103", "run-3", IN_PROGRESS_STATE) + .expect("shared slot should reopen after one of the configured leases clears") + ); +} + +#[test] +fn failed_shared_slot_attempt_releases_issue_claim_before_retry() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let store_one = StateStore::open_in_memory().expect("first store should open"); + let store_two = StateStore::open_in_memory().expect("second store should open"); + + store_one + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("first store should configure dispatch slot root"); + store_two + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("second store should configure dispatch slot root"); + + assert!( + store_one + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first store should acquire the only shared slot") + ); + assert!( + !store_two + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("second store should fail while the only slot is busy") + ); + + store_one.clear_lease("PUB-101").expect("shared lease should clear"); + + assert!( + store_two + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("retry should succeed after the failed contender releases its issue claim") + ); +} + +#[test] +fn shared_issue_claim_blocks_duplicate_issue_across_process_local_stores() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let store_one = StateStore::open_in_memory().expect("first store should open"); + let store_two = StateStore::open_in_memory().expect("second store should open"); + + store_one + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("first store should configure dispatch slot root"); + store_two + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("second store should configure dispatch slot root"); + + assert!( + store_one + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first issue claim should succeed") + ); + assert!( + !store_two + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("duplicate issue claim should be rejected across processes") + ); + assert!( + store_two + .try_acquire_lease("pubfi", "PUB-102", "run-3", IN_PROGRESS_STATE) + .expect("another issue should still be able to use the remaining slot") + ); +} + +#[test] +fn shared_issue_claim_reopens_same_issue_after_clear_across_process_local_stores() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let store_one = StateStore::open_in_memory().expect("first store should open"); + let store_two = StateStore::open_in_memory().expect("second store should open"); + + store_one + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("first store should configure dispatch slot root"); + store_two + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("second store should configure dispatch slot root"); + + assert!( + store_one + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first issue claim should succeed") + ); + assert!( + !store_two + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("duplicate issue claim should be rejected while the first lease is active") + ); + + store_one.clear_lease("PUB-101").expect("shared issue claim should clear"); + + assert!( + store_two + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("same issue claim should reopen after the first lease clears") + ); +} + +#[test] +fn shared_issue_claim_listing_reports_other_process_state() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let remote_store = StateStore::open_in_memory().expect("remote store should open"); + let observer_store = StateStore::open_in_memory().expect("observer store should open"); + + remote_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("remote store should configure dispatch slot root"); + observer_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("observer store should configure dispatch slot root"); + + assert!( + remote_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("remote issue claim should succeed") + ); + + let leases = observer_store + .list_active_shared_leases("pubfi") + .expect("shared claim listing should succeed"); + + assert_eq!(leases.len(), 1); + assert_eq!(leases[0].issue_id(), "PUB-101"); + assert_eq!(leases[0].run_id(), "run-1"); + assert_eq!(leases[0].issue_state(), IN_PROGRESS_STATE); +} + +#[cfg(unix)] +#[test] +fn adopted_dispatch_slot_blocks_after_parent_releases_local_guard() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let parent_store = StateStore::open_in_memory().expect("parent store should open"); + let child_store = StateStore::open_in_memory().expect("child store should open"); + let contender_store = StateStore::open_in_memory().expect("contender store should open"); + + parent_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("parent store should configure dispatch slot root"); + child_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("child store should configure dispatch slot root"); + contender_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("contender store should configure dispatch slot root"); + + assert!( + parent_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("parent should acquire the shared slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child("PUB-101") + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child("PUB-101") + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + "pubfi", + "PUB-101", + "run-1", + IN_PROGRESS_STATE, + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + parent_store + .release_dispatch_slot("PUB-101") + .expect("parent should release its local guard after handoff"); + + assert!( + !contender_store + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("child-held guard should keep the slot busy") + ); + + child_store.clear_lease("PUB-101").expect("child lease should clear"); +} + +#[cfg(unix)] +#[test] +fn adopted_issue_claim_blocks_same_issue_after_parent_clears_local_guard() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let parent_store = StateStore::open_in_memory().expect("parent store should open"); + let child_store = StateStore::open_in_memory().expect("child store should open"); + let contender_store = StateStore::open_in_memory().expect("contender store should open"); + + parent_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("parent store should configure dispatch slot root"); + child_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("child store should configure dispatch slot root"); + contender_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("contender store should configure dispatch slot root"); + + assert!( + parent_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("parent should acquire the shared issue claim") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child("PUB-101") + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child("PUB-101") + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + "pubfi", + "PUB-101", + "run-1", + IN_PROGRESS_STATE, + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + parent_store + .clear_lease("PUB-101") + .expect("parent should drop its local lease without unlocking the child handoff"); + + assert!( + !contender_store + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("same issue should stay claimed while the child still holds the handoff fd") + ); + + child_store.clear_lease("PUB-101").expect("child lease should clear"); +} + +#[cfg(unix)] +#[test] +fn parent_can_release_handed_off_guards_without_dropping_runtime_lease() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let parent_store = StateStore::open_in_memory().expect("parent store should open"); + let child_store = StateStore::open_in_memory().expect("child store should open"); + let contender_store = StateStore::open_in_memory().expect("contender store should open"); + + parent_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("parent store should configure dispatch slot root"); + child_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("child store should configure dispatch slot root"); + contender_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 2) + .expect("contender store should configure dispatch slot root"); + + assert!( + parent_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("parent should acquire the shared issue claim") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child("PUB-101") + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child("PUB-101") + .expect("child should inherit the shared dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + "pubfi", + "PUB-101", + "run-1", + IN_PROGRESS_STATE, + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + parent_store + .release_handed_off_guards("PUB-101") + .expect("parent should release process-local guards after handoff"); + + assert!( + parent_store + .lease_for_issue("PUB-101") + .expect("parent lease lookup should succeed") + .is_some(), + "parent must keep the runtime lease visible after dropping local fd guards" + ); + assert!( + !contender_store + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("same issue should stay claimed by the child handoff") + ); + assert!( + contender_store + .try_acquire_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("another issue should acquire the second dispatch slot") + ); + + child_store.clear_lease("PUB-101").expect("child lease should clear"); +} + +#[cfg(unix)] +#[test] +fn adopted_preacquired_lease_restores_close_on_exec_on_inherited_fds() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let parent_store = StateStore::open_in_memory().expect("parent store should open"); + let child_store = StateStore::open_in_memory().expect("child store should open"); + + parent_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("parent store should configure dispatch slot root"); + child_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("child store should configure dispatch slot root"); + + assert!( + parent_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("parent should acquire the shared slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child("PUB-101") + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child("PUB-101") + .expect("child should inherit the shared dispatch-slot fd"); + let issue_claim_fd = child_issue_claim.as_raw_fd(); + let dispatch_slot_fd = child_guard.as_raw_fd(); + + assert!( + !fd_has_close_on_exec(issue_claim_fd), + "handoff issue-claim fd should clear close-on-exec before exec" + ); + assert!( + !fd_has_close_on_exec(dispatch_slot_fd), + "handoff dispatch-slot fd should clear close-on-exec before exec" + ); + + child_store + .adopt_preacquired_lease( + "pubfi", + "PUB-101", + "run-1", + IN_PROGRESS_STATE, + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + + assert!( + fd_has_close_on_exec(issue_claim_fd), + "adopted issue-claim fd must restore close-on-exec before spawning grandchildren" + ); + assert!( + fd_has_close_on_exec(dispatch_slot_fd), + "adopted dispatch-slot fd must restore close-on-exec before spawning grandchildren" + ); + + child_store.clear_lease("PUB-101").expect("child lease should clear"); + parent_store.clear_lease("PUB-101").expect("parent lease should clear"); +} + +#[cfg(unix)] +#[test] +fn adopted_child_clear_releases_lock_when_descendant_keeps_inherited_fds_open() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let parent_store = StateStore::open_in_memory().expect("parent store should open"); + let child_store = StateStore::open_in_memory().expect("child store should open"); + let contender_store = StateStore::open_in_memory().expect("contender store should open"); + + parent_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("parent store should configure dispatch slot root"); + child_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("child store should configure dispatch slot root"); + contender_store + .configure_dispatch_slot_root("pubfi", temp_dir.path(), 1) + .expect("contender store should configure dispatch slot root"); + + assert!( + parent_store + .try_acquire_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("parent should acquire the shared slot") + ); + + let child_issue_claim = parent_store + .clone_issue_claim_for_child("PUB-101") + .expect("child should inherit the shared issue-claim fd"); + let (child_guard, child_slot_index) = parent_store + .clone_dispatch_slot_for_child("PUB-101") + .expect("child should inherit the shared dispatch-slot fd"); + let _descendant_issue_claim = + child_issue_claim.try_clone().expect("descendant should inherit the issue-claim fd"); + let _descendant_guard = + child_guard.try_clone().expect("descendant should inherit the dispatch-slot fd"); + + child_store + .adopt_preacquired_lease( + "pubfi", + "PUB-101", + "run-1", + IN_PROGRESS_STATE, + PreacquiredLeaseGuards { + issue_claim_fd: child_issue_claim.into_raw_fd(), + dispatch_slot_fd: child_guard.into_raw_fd(), + dispatch_slot_index: child_slot_index, + }, + ) + .expect("child should adopt the inherited lease guard"); + parent_store.clear_lease("PUB-101").expect("parent should drop its local handoff guard"); + child_store.clear_lease("PUB-101").expect("child lease should clear"); + + assert!( + contender_store + .try_acquire_lease("pubfi", "PUB-101", "run-2", IN_PROGRESS_STATE) + .expect("descendant-held fds must not keep the cleared lease claimed"), + "clearing an adopted child lease must release the shared claim and slot even if a descendant still holds inherited fds" + ); +} + +#[test] +fn records_run_attempts_and_events() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .record_run_attempt("run-1", "PUB-101", 1, "running") + .expect("run attempt should be recorded"); + store.update_run_thread("run-1", "thread-1").expect("thread id should be attached"); + store + .append_event("run-1", 1, "turn/started", "{\"turn\":\"1\"}") + .expect("event should be recorded"); + + let run_attempt = store + .run_attempt("run-1") + .expect("run attempt query should succeed") + .expect("run attempt should exist"); + + assert_eq!(run_attempt.issue_id(), "PUB-101"); + assert_eq!(run_attempt.attempt_number(), 1); + assert_eq!(run_attempt.status(), "running"); + assert_eq!(run_attempt.thread_id(), Some("thread-1")); + assert_eq!(store.event_count("run-1").expect("event count should succeed"), 1); + assert_eq!(store.next_attempt_number("PUB-101").expect("next attempt should load"), 2); + assert_eq!( + store.retry_budget_attempt_count("PUB-101").expect("retry budget count should load"), + 0 + ); + + store.update_run_status("run-1", "interrupted").expect("status should update"); + + let updated = store + .run_attempt("run-1") + .expect("run attempt query should succeed") + .expect("run attempt should exist"); + + assert_eq!(updated.status(), "interrupted"); + assert!( + store + .last_run_activity_unix_epoch("run-1") + .expect("last activity lookup should succeed") + .is_some() + ); +} + +#[test] +fn run_activity_marker_round_trips_marker_surfaces() { + assert_run_activity_marker_round_trips_clearable_auxiliary_fields(); + assert_run_activity_marker_round_trips_thread_and_protocol_summary_fields(); + assert_run_activity_marker_round_trips_child_agent_activity_summary(); + assert_run_activity_marker_round_trips_account_summary(); +} + +fn assert_run_activity_marker_round_trips_clearable_auxiliary_fields() { + let temp_dir = TempDir::new().expect("tempdir should create"); + + state::write_run_activity_marker_for_process(temp_dir.path(), "run-1", 1, process::id()) + .expect("activity marker should write"); + state::write_run_retry_schedule(temp_dir.path(), "run-1", 1, "failure", 12_345) + .expect("retry schedule should write"); + state::write_run_review_policy_state( + temp_dir.path(), + "run-1", + 1, + "handoff", + "findings", + "abc123", + 2, + ) + .expect("review policy state should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.run_id(), "run-1"); + assert_eq!(marker.attempt_number(), 1); + assert_eq!(marker.retry_kind(), Some("failure")); + assert_eq!(marker.retry_ready_at_unix_epoch(), Some(12_345)); + assert_eq!(marker.review_policy_phase(), Some("handoff")); + assert_eq!(marker.review_policy_status(), Some("findings")); + assert_eq!(marker.review_policy_head_sha(), Some("abc123")); + assert_eq!(marker.review_policy_nonclean_rounds(), Some(2)); + + state::clear_run_retry_schedule(temp_dir.path()).expect("retry schedule should clear"); + + let retry_cleared = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should reload") + .expect("marker snapshot should still exist"); + + assert_eq!(retry_cleared.retry_kind(), None); + assert_eq!(retry_cleared.retry_ready_at_unix_epoch(), None); + assert_eq!(retry_cleared.review_policy_phase(), Some("handoff")); + + state::clear_run_review_policy_state(temp_dir.path()) + .expect("review policy state should clear"); + + let policy_cleared = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should reload") + .expect("marker snapshot should still exist"); + + assert_eq!(policy_cleared.review_policy_phase(), None); + assert_eq!(policy_cleared.review_policy_status(), None); + assert_eq!(policy_cleared.review_policy_head_sha(), None); + assert_eq!(policy_cleared.review_policy_nonclean_rounds(), None); +} + +fn assert_run_activity_marker_round_trips_thread_and_protocol_summary_fields() { + let temp_dir = TempDir::new().expect("tempdir should create"); + + state::write_run_activity_marker_for_process(temp_dir.path(), "run-1", 1, process::id()) + .expect("activity marker should write"); + state::write_run_thread_marker(temp_dir.path(), "run-1", 1, "thread-1") + .expect("thread marker should write"); + state::write_run_turn_marker(temp_dir.path(), "run-1", 1, "turn-1") + .expect("turn marker should write"); + state::write_run_thread_status_marker( + temp_dir.path(), + "run-1", + 1, + Some("thread-1"), + Some("turn-1"), + "active", + &[String::from("waitingOnApproval")], + ) + .expect("thread status marker should write"); + state::write_run_effective_runtime_marker( + temp_dir.path(), + "run-1", + 1, + &EffectiveRuntimeMarker { + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + effective_model: "gpt-5.4", + effective_model_provider: "openai", + effective_cwd: "/tmp/worktree", + effective_approval_policy: "never", + effective_approvals_reviewer: "human", + effective_sandbox_mode: "workspaceWrite", + }, + ) + .expect("effective runtime marker should write"); + + let protocol_activity = ProtocolActivitySummary { + turn_status: Some(String::from("completed")), + waiting_reason: Some(String::from("model_execution")), + rate_limit_status: Some(String::from("usageLimitExceeded")), + recent_events: vec![ + state::ProtocolActivityEventSummary { + event_type: String::from("turn/started"), + category: String::from("turn"), + detail: Some(String::from("running")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("turn/completed"), + category: String::from("turn"), + detail: Some(String::from("completed")), + }, + ], + }; + + state::write_run_protocol_activity_marker( + temp_dir.path(), + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 3, + last_event_type: "turn/completed", + child_agent_activity: None, + protocol_activity: Some(&protocol_activity), + }, + ) + .expect("protocol summary should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.thread_id(), Some("thread-1")); + assert_eq!(marker.turn_id(), Some("turn-1")); + assert_eq!(marker.thread_status(), Some("active")); + assert_eq!(marker.thread_active_flags(), &[String::from("waitingOnApproval")]); + assert_eq!(marker.event_count(), 3); + assert_eq!(marker.last_event_type(), Some("turn/completed")); + assert_eq!(marker.effective_model(), Some("gpt-5.4")); + assert_eq!(marker.effective_model_provider(), Some("openai")); + assert_eq!(marker.effective_cwd(), Some("/tmp/worktree")); + assert_eq!(marker.effective_approval_policy(), Some("never")); + assert_eq!(marker.effective_approvals_reviewer(), Some("human")); + assert_eq!(marker.effective_sandbox_mode(), Some("workspaceWrite")); + assert_eq!(marker.protocol_activity(), Some(&protocol_activity)); + assert!(marker.last_protocol_activity_unix_epoch().is_some()); + assert_eq!(marker.current_operation(), Some(state::RUN_OPERATION_AGENT_RUN)); + assert!(marker.last_progress_unix_epoch().is_some()); +} + +fn assert_run_activity_marker_round_trips_child_agent_activity_summary() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let summary = ChildAgentActivitySummary { + buckets: vec![ + state::ChildAgentActivityBucket { + name: String::from("Model"), + wall_seconds: 693, + event_count: 12, + tool_call_count: 0, + input_tokens: 4_270_000, + output_tokens: 12_000, + output_bytes: 0, + }, + state::ChildAgentActivityBucket { + name: String::from("Browser/Image"), + wall_seconds: 41, + event_count: 6, + tool_call_count: 3, + input_tokens: 0, + output_tokens: 0, + output_bytes: 180_000, + }, + ], + current_bucket: Some(String::from("Model")), + current_detail: Some(String::from("waiting after tool output")), + current_started_unix_epoch: Some(1_800_000_000), + current_elapsed_seconds: Some(9), + wall_seconds: 734, + event_count: 18, + tool_call_count: 3, + input_tokens_current: Some(105_000), + input_tokens_max: Some(105_000), + input_tokens_cumulative: 4_270_000, + output_tokens_cumulative: 12_000, + largest_tool_output_bytes: Some(180_000), + largest_tool_output_tool: Some(String::from("view_image")), + large_output_warnings: vec![String::from( + "view_image repeated 3 large outputs; largest 180000 bytes", + )], + }; + + state::write_run_protocol_activity_marker( + temp_dir.path(), + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 18, + last_event_type: "item/tool/call/response", + child_agent_activity: Some(&summary), + protocol_activity: None, + }, + ) + .expect("protocol summary should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.child_agent_activity(), Some(&summary)); +} + +fn assert_run_activity_marker_round_trips_account_summary() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let summary = CodexAccountActivitySummary { + account_fingerprint: String::from("acct_...cdef"), + email: Some(String::from("account@example.com")), + plan_type: Some(String::from("pro")), + status: String::from("selected"), + refresh_status: String::from("not_needed"), + checked_at_unix_epoch: Some(1_800_000_010), + selected_at_unix_epoch: Some(1_800_000_011), + primary_window_seconds: Some(18_000), + primary_remaining_percent: Some(72), + primary_resets_at_unix_epoch: Some(1_800_018_000), + secondary_window_seconds: Some(604_800), + secondary_remaining_percent: Some(91), + secondary_resets_at_unix_epoch: Some(1_800_604_800), + credits_has_credits: Some(true), + credits_unlimited: Some(false), + credits_balance: Some(String::from("9.99")), + rate_limit_reached_type: None, + cooldown_until_unix_epoch: None, + note: Some(String::from("usage probe ok")), + }; + + state::write_run_account_marker( + temp_dir.path(), + &CodexAccountMarker { + run_id: "run-1", + attempt_number: 1, + account: &summary, + accounts: slice::from_ref(&summary), + }, + ) + .expect("account summary should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.account(), Some(&summary)); + assert_eq!(marker.accounts(), slice::from_ref(&summary)); + + let body = fs::read_to_string(temp_dir.path().join(RUN_ACTIVITY_MARKER_FILE)) + .expect("marker body should read"); + + assert!(body.contains("account=")); + assert!(body.contains("accounts=")); + assert!(!body.contains("codex_account=")); + assert!(!body.contains("codex_accounts=")); +} + +#[test] +fn run_operation_marker_resets_stale_per_attempt_fields_on_new_attempt() { + let temp_dir = TempDir::new().expect("tempdir should create"); + + state::write_run_activity_marker_for_process(temp_dir.path(), "run-1", 1, process::id()) + .expect("first activity marker should write"); + state::write_run_thread_marker(temp_dir.path(), "run-1", 1, "thread-1") + .expect("thread marker should write"); + state::write_run_turn_marker(temp_dir.path(), "run-1", 1, "turn-1") + .expect("turn marker should write"); + state::write_run_thread_status_marker( + temp_dir.path(), + "run-1", + 1, + Some("thread-1"), + Some("turn-1"), + "active", + &[String::from("waitingOnUserInput")], + ) + .expect("thread status should write"); + state::write_run_effective_runtime_marker( + temp_dir.path(), + "run-1", + 1, + &EffectiveRuntimeMarker { + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + effective_model: "gpt-5.4", + effective_model_provider: "openai", + effective_cwd: "/tmp/worktree", + effective_approval_policy: "never", + effective_approvals_reviewer: "human", + effective_sandbox_mode: "dangerFullAccess", + }, + ) + .expect("effective runtime should write"); + state::write_run_protocol_activity_marker( + temp_dir.path(), + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 3, + last_event_type: "turn/completed", + child_agent_activity: None, + protocol_activity: None, + }, + ) + .expect("protocol summary should write"); + state::write_run_retry_schedule(temp_dir.path(), "run-1", 1, "failure", 123) + .expect("retry schedule should write"); + state::write_run_retry_budget_attempt_count(temp_dir.path(), "run-1", 1, 2) + .expect("retry budget should write"); + state::write_run_review_policy_state( + temp_dir.path(), + "run-1", + 1, + "repair", + "findings", + "def456", + 2, + ) + .expect("review policy should write"); + state::write_run_operation_marker(temp_dir.path(), "run-2", 2, RUN_OPERATION_REPO_GATE) + .expect("next attempt operation marker should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.run_id(), "run-2"); + assert_eq!(marker.attempt_number(), 2); + assert_eq!(marker.current_operation(), Some(state::RUN_OPERATION_REPO_GATE)); + assert!(marker.last_progress_unix_epoch().is_some()); + assert_eq!(marker.thread_id(), None); + assert_eq!(marker.turn_id(), None); + assert_eq!(marker.thread_status(), None); + assert!(marker.thread_active_flags().is_empty()); + assert_eq!(marker.event_count(), 0); + assert_eq!(marker.last_event_type(), None); + assert_eq!(marker.protocol_activity(), None); + assert_eq!(marker.effective_model(), None); + assert_eq!(marker.effective_model_provider(), None); + assert_eq!(marker.effective_cwd(), None); + assert_eq!(marker.effective_approval_policy(), None); + assert_eq!(marker.effective_approvals_reviewer(), None); + assert_eq!(marker.effective_sandbox_mode(), None); + assert_eq!(marker.last_protocol_activity_unix_epoch(), None); + assert_eq!(marker.retry_kind(), None); + assert_eq!(marker.retry_ready_at_unix_epoch(), None); + assert_eq!( + state::read_run_retry_budget_attempt_count(temp_dir.path()) + .expect("retry budget count should load"), + Some(2) + ); + assert_eq!(marker.review_policy_phase(), Some("repair")); + assert_eq!(marker.review_policy_status(), Some("findings")); + assert_eq!(marker.review_policy_head_sha(), Some("def456")); + assert_eq!(marker.review_policy_nonclean_rounds(), Some(2)); +} + +#[test] +fn counts_retry_budget_attempts_per_issue() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store.record_run_attempt("run-1", "PUB-101", 1, "succeeded").expect("first run should record"); + store.record_run_attempt("run-2", "PUB-101", 2, "failed").expect("second run should record"); + store + .record_run_attempt("run-3", "PUB-101", 3, "interrupted") + .expect("third run should record"); + store + .record_run_attempt("run-5", "PUB-101", 4, "terminal_guarded") + .expect("guarded run should record"); + store + .record_run_attempt("run-4", "PUB-102", 1, "failed") + .expect("other issue run should record"); + + assert_eq!( + store.retry_budget_attempt_count("PUB-101").expect("retry budget count should load"), + 3 + ); + assert_eq!( + store.retry_budget_attempt_count("PUB-102").expect("retry budget count should load"), + 1 + ); +} + +#[test] +fn loads_latest_run_attempt_for_issue() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store.record_run_attempt("run-1", "PUB-101", 1, "failed").expect("first run should record"); + store + .record_run_attempt("run-2", "PUB-101", 2, "terminal_guarded") + .expect("latest run should record"); + + let attempt = store + .latest_run_attempt_for_issue("PUB-101") + .expect("latest run lookup should succeed") + .expect("latest run should exist"); + + assert_eq!(attempt.run_id(), "run-2"); + assert_eq!(attempt.attempt_number(), 2); + assert_eq!(attempt.status(), "terminal_guarded"); +} + +#[test] +fn manages_worktree_mappings() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .upsert_worktree("pubfi", "PUB-101", "x/pub-101", "/tmp/worktrees/pub-101") + .expect("worktree mapping should be recorded"); + + let mapping = store + .worktree_for_issue("PUB-101") + .expect("mapping lookup should succeed") + .expect("mapping should exist"); + + assert_eq!(mapping.issue_id(), "PUB-101"); + assert_eq!(mapping.branch_name(), "x/pub-101"); + assert_eq!(mapping.worktree_path(), Path::new("/tmp/worktrees/pub-101")); + assert_eq!(mapping.project_id(), "pubfi"); + assert_eq!(store.list_worktrees("pubfi").expect("list should succeed").len(), 1); + + store.clear_worktree("PUB-101").expect("mapping should be deleted"); + + assert!(store.worktree_for_issue("PUB-101").expect("lookup should succeed").is_none()); +} + +#[test] +fn persistent_clear_worktree_deletes_review_markers() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.sqlite3"); + let store = StateStore::open(&state_path).expect("state store should open"); + let handoff = ReviewHandoffMarker::new( + "run-1", + 1, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "main", + "x/decodex-pub-101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + ); + let orchestration = ReviewOrchestrationMarker::new( + "run-1", + 1, + "x/decodex-pub-101", + "https://github.com/hack-ink/decodex/pull/101", + "08a20f7dfb9526e7421a5f095b1c6adec84e52d6", + "request_pending", + None, + None, + None, + 0, + 0, + None, + ); + + store + .upsert_worktree("pubfi", "PUB-101", "x/decodex-pub-101", "/tmp/worktrees/pub-101") + .expect("worktree mapping should be recorded"); + store + .upsert_review_handoff_marker("pubfi", "PUB-101", &handoff) + .expect("handoff marker should persist"); + store + .upsert_review_orchestration_marker("pubfi", "PUB-101", &orchestration) + .expect("orchestration marker should persist"); + store.clear_worktree("PUB-101").expect("worktree cleanup should persist"); + + let reopened = StateStore::open(&state_path).expect("reopened store should open"); + + assert!( + reopened.worktree_for_issue("PUB-101").expect("worktree lookup should succeed").is_none() + ); + assert!( + reopened + .review_handoff_marker("pubfi", "PUB-101", "x/decodex-pub-101") + .expect("handoff lookup should succeed") + .is_none() + ); + assert!( + reopened + .review_orchestration_marker("pubfi", "PUB-101", &handoff) + .expect("orchestration lookup should succeed") + .is_none() + ); +} + +#[test] +fn lists_issue_leases() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("first lease should be inserted"); + store + .upsert_lease("pubfi", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("second lease should be inserted"); + + let leases = store.list_leases("pubfi").expect("lease listing should succeed"); + + assert_eq!(leases.len(), 2); + assert_eq!(leases[0].project_id(), "pubfi"); + assert_eq!(leases[0].issue_id(), "PUB-101"); + assert_eq!(leases[1].issue_id(), "PUB-102"); +} + +#[test] +fn lists_recent_project_runs_with_protocol_summary() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .record_run_attempt("run-2", "PUB-102", 2, "failed") + .expect("older run attempt should be recorded"); + store + .record_run_attempt("run-1", "PUB-101", 1, "running") + .expect("active run attempt should be recorded"); + store.update_run_thread("run-1", "thread-1").expect("thread id should attach"); + store + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("lease should record"); + store + .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") + .expect("active worktree should record"); + store + .upsert_worktree("pubfi", "PUB-102", "x/pubfi-pub-102", "/tmp/worktrees/pub-102") + .expect("retained worktree should record"); + store + .append_event("run-1", 1, "turn/started", "{\"turn\":\"1\"}") + .expect("event should record"); + store + .append_event("run-1", 2, "turn/completed", "{\"turn\":\"1\"}") + .expect("second event should record"); + + let runs = store.list_recent_runs("pubfi", 10).expect("recent project runs should load"); + + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].run_id(), "run-1"); + assert!(runs[0].active_lease()); + assert_eq!(runs[0].thread_id(), Some("thread-1")); + assert_eq!(runs[0].event_count(), 2); + assert_eq!(runs[0].last_event_type(), Some("turn/completed")); + assert_eq!(runs[0].branch_name(), Some("x/pubfi-pub-101")); + assert_eq!(runs[0].worktree_path(), Some(Path::new("/tmp/worktrees/pub-101"))); + assert_eq!(runs[1].run_id(), "run-2"); + assert!(!runs[1].active_lease()); + assert_eq!(runs[1].event_count(), 0); +} + +#[test] +fn lists_recent_project_runs_after_terminal_lane_cleanup() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store + .record_run_attempt("run-1", "PUB-101", 1, "running") + .expect("run attempt should record before project ownership is known"); + store + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("lease should record project ownership"); + store + .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") + .expect("worktree should record project ownership"); + store.update_run_status("run-1", "succeeded").expect("terminal status should update"); + store.clear_lease("PUB-101").expect("terminal cleanup should clear active lease"); + store.clear_worktree("PUB-101").expect("terminal cleanup should clear worktree mapping"); + + let runs = store.list_recent_runs("pubfi", 10).expect("recent project runs should load"); + + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].run_id(), "run-1"); + assert_eq!(runs[0].status(), "succeeded"); + assert!(!runs[0].active_lease()); + assert_eq!(runs[0].branch_name(), None); + assert_eq!(runs[0].worktree_path(), None); + assert!( + store.list_recent_runs("other", 10).expect("other project lookup should load").is_empty(), + "remembered run ownership must stay scoped to the original project" + ); +} + +#[test] +fn lists_active_project_runs_only() { + let store = StateStore::open_in_memory().expect("in-memory state store should open"); + + store.record_run_attempt("run-1", "PUB-101", 1, "running").expect("first run should record"); + store.record_run_attempt("run-2", "PUB-102", 1, "running").expect("second run should record"); + store + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("lease should record"); + store + .upsert_lease("other", "PUB-102", "run-2", IN_PROGRESS_STATE) + .expect("other-project lease should record"); + store + .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") + .expect("first worktree should record"); + store + .upsert_worktree("other", "PUB-102", "x/other-pub-102", "/tmp/worktrees/pub-102") + .expect("second worktree should record"); + + let runs = store.list_active_runs("pubfi").expect("active project runs should load"); + + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].run_id(), "run-1"); + assert!(runs[0].active_lease()); +} + +#[test] +fn state_store_open_persists_runtime_history_across_instances() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.db"); + let first = StateStore::open(&state_path).expect("first state store should open"); + + first + .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) + .expect("lease should persist"); + first.record_run_attempt("run-1", "PUB-101", 1, "running").expect("run attempt should record"); + first.update_run_thread("run-1", "thread-1").expect("thread should persist"); + first.append_event("run-1", 1, "thread/run/created", "{}").expect("event should persist"); + first + .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") + .expect("worktree should persist"); + + let mut ledger_record = LinearExecutionEventRecord::new( + LinearExecutionEventIdentity { + service_id: "pubfi", + issue_id: "PUB-101", + issue_identifier: "PUB-101", + run_id: "run-1", + attempt_number: 1, + }, + "closeout", + String::from("2026-04-29T10:10:00Z"), + "closeout", + ); + + ledger_record.pr_url = Some(String::from("https://github.com/hack-ink/decodex/pull/101")); + ledger_record.commit_sha = Some(String::from("1111111111111111111111111111111111111111")); + ledger_record.summary = Some(String::from("Completed retained closeout.")); + + first + .record_linear_execution_event(&ledger_record) + .expect("linear execution event should persist"); + + assert!(state_path.exists(), "persistent runtime DB should be created"); + + let second = StateStore::open(&state_path).expect("second state store should open"); + let latest = second + .latest_run_attempt_for_issue("PUB-101") + .expect("latest run lookup should succeed") + .expect("persistent store should recover run history"); + + assert_eq!(latest.run_id(), "run-1"); + assert_eq!(latest.thread_id(), Some("thread-1")); + assert_eq!(second.event_count("run-1").expect("event count should load"), 1); + assert!( + second.lease_for_issue("PUB-101").expect("lease lookup should succeed").is_some(), + "persistent store should recover active leases" + ); + assert!( + second.worktree_for_issue("PUB-101").expect("worktree lookup should succeed").is_some(), + "persistent store should recover retained worktree mappings" + ); + + let ledger_records = second + .list_linear_execution_events("pubfi", "PUB-101") + .expect("linear execution events should load"); + + assert_eq!(ledger_records, vec![ledger_record]); +} + +#[test] +fn state_store_open_refreshes_pubfi_project_registry_across_instances() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let state_path = temp_dir.path().join("runtime.db"); + let initial_config_path = temp_dir.path().join("stale/project.toml"); + let initial_repo_root = temp_dir.path().join("stale/repo"); + let initial_worktree_root = temp_dir.path().join("stale/repo/.worktrees"); + let initial_workflow_path = temp_dir.path().join("stale/repo/WORKFLOW.md"); + let refreshed_config_path = temp_dir.path().join("current/project.toml"); + let refreshed_repo_root = temp_dir.path().join("current/repo"); + let refreshed_worktree_root = temp_dir.path().join("current/repo/.worktrees"); + let refreshed_workflow_path = temp_dir.path().join("current/repo/WORKFLOW.md"); + let store = StateStore::open(&state_path).expect("state store should open"); + let registration = ProjectRegistration { + service_id: String::from("pubfi"), + config_path: initial_config_path, + repo_root: initial_repo_root, + worktree_root: initial_worktree_root, + workflow_path: initial_workflow_path, + tracker_api_key_env_var: String::from("LINEAR_API_KEY_HACKINK"), + github_token_env_var: String::from("GITHUB_PAT_Y"), + enabled: true, + config_fingerprint: String::from("abc123"), + updated_at: String::from("2026-04-29T00:00:00Z"), + updated_at_unix: 1_777_392_000, + }; + let refreshed_registration = ProjectRegistration { + service_id: String::from("pubfi"), + config_path: refreshed_config_path.clone(), + repo_root: refreshed_repo_root.clone(), + worktree_root: refreshed_worktree_root.clone(), + workflow_path: refreshed_workflow_path.clone(), + tracker_api_key_env_var: String::from("LINEAR_API_KEY_HACKINK"), + github_token_env_var: String::from("GITHUB_PAT_Y"), + enabled: true, + config_fingerprint: String::from("def456"), + updated_at: String::from("2026-04-30T00:00:00Z"), + updated_at_unix: 1_777_478_400, + }; + + store.upsert_project(®istration).expect("project should persist"); + store.set_project_enabled("pubfi", false).expect("project should disable"); + store.upsert_project(&refreshed_registration).expect("project should refresh"); + + let reopened = StateStore::open(&state_path).expect("state store should reopen"); + let projects = reopened.list_projects().expect("project registry should load"); + + assert_eq!(projects.len(), 1, "pubfi refresh should keep one scoped registry row"); + + let project = &projects[0]; + + assert_eq!( + project.service_id(), + "pubfi", + "pubfi refresh should stay scoped to the same service id" + ); + assert!(project.enabled(), "pubfi refresh should replace the previously disabled row"); + assert_eq!( + project.config_fingerprint(), + "def456", + "pubfi refresh should replace the stale config fingerprint" + ); + assert_eq!( + project.config_path(), + refreshed_config_path.as_path(), + "pubfi refresh should replace the stale config path" + ); + assert_eq!( + project.repo_root(), + refreshed_repo_root.as_path(), + "pubfi refresh should replace the stale repo root" + ); + assert_eq!( + project.worktree_root(), + refreshed_worktree_root.as_path(), + "pubfi refresh should replace the stale worktree root" + ); + assert_eq!( + project.workflow_path(), + refreshed_workflow_path.as_path(), + "pubfi refresh should replace the stale workflow path" + ); +} diff --git a/apps/decodex/src/test_support.rs b/apps/decodex/src/test_support.rs new file mode 100644 index 00000000..15366146 --- /dev/null +++ b/apps/decodex/src/test_support.rs @@ -0,0 +1,47 @@ +use std::{ + env, + ffi::OsString, + sync::{Mutex, MutexGuard, OnceLock}, +}; + +pub(crate) struct TestEnvVarGuard { + _lock: MutexGuard<'static, ()>, + key: String, + previous: Option, +} +impl TestEnvVarGuard { + pub(crate) fn set(key: impl Into, value: &str) -> Self { + let lock = test_env_mutex().lock().expect("test env mutex should not be poisoned"); + let key = key.into(); + let previous = env::var_os(&key); + + unsafe { env::set_var(&key, value) }; + + Self { _lock: lock, key, previous } + } + + pub(crate) fn lock() -> TestEnvLockGuard { + TestEnvLockGuard { + _lock: test_env_mutex().lock().expect("test env mutex should not be poisoned"), + } + } +} + +impl Drop for TestEnvVarGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(previous) => unsafe { env::set_var(&self.key, previous) }, + None => unsafe { env::remove_var(&self.key) }, + } + } +} + +pub(crate) struct TestEnvLockGuard { + _lock: MutexGuard<'static, ()>, +} + +fn test_env_mutex() -> &'static Mutex<()> { + static TEST_ENV_MUTEX: OnceLock> = OnceLock::new(); + + TEST_ENV_MUTEX.get_or_init(|| Mutex::new(())) +} diff --git a/apps/decodex/src/tracker.rs b/apps/decodex/src/tracker.rs new file mode 100644 index 00000000..eb0e978e --- /dev/null +++ b/apps/decodex/src/tracker.rs @@ -0,0 +1,248 @@ +pub(crate) mod linear; +pub(crate) mod records; + +use std::slice; + +use color_eyre::Report; + +use crate::prelude::{Result, eyre}; +use records::LinearExecutionEventRecord; + +pub(crate) trait IssueTracker { + fn list_issues_with_label(&self, label_name: &str) -> Result>; + fn find_team_label_id(&self, team_id: &str, label_name: &str) -> Result>; + fn get_issue_by_identifier(&self, issue_identifier: &str) -> Result>; + fn refresh_issues(&self, issue_ids: &[String]) -> Result>; + fn list_comments(&self, issue_id: &str) -> Result>; + fn update_issue_state(&self, issue_id: &str, state_id: &str) -> Result<()>; + fn add_issue_labels(&self, issue_id: &str, label_ids: &[String]) -> Result<()>; + fn remove_issue_labels(&self, issue_id: &str, label_ids: &[String]) -> Result<()>; + fn create_comment(&self, issue_id: &str, body: &str) -> Result<()>; +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerComment { + pub(crate) body: String, + pub(crate) created_at: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerIssue { + pub(crate) id: String, + pub(crate) identifier: String, + #[cfg(test)] + pub(crate) project_slug: Option, + pub(crate) title: String, + pub(crate) description: String, + pub(crate) priority: Option, + pub(crate) created_at: String, + pub(crate) updated_at: String, + pub(crate) state: TrackerState, + pub(crate) team: TrackerTeam, + pub(crate) labels_complete: bool, + pub(crate) labels: Vec, + pub(crate) blockers: Vec, +} +impl TrackerIssue { + pub(crate) fn has_label(&self, label_name: &str) -> bool { + self.labels.iter().any(|label| label.name == label_name) + } + + pub(crate) fn state_id_for_name(&self, state_name: &str) -> Option<&str> { + self.team + .states + .iter() + .find(|state| state.name == state_name) + .map(|state| state.id.as_str()) + } + + pub(crate) fn label_id_for_name(&self, label_name: &str) -> Option<&str> { + self.team + .labels + .iter() + .find(|label| label.name == label_name) + .map(|label| label.id.as_str()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerIssueBlocker { + pub(crate) id: String, + pub(crate) identifier: String, + pub(crate) state: TrackerState, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerState { + pub(crate) id: String, + pub(crate) name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerLabel { + pub(crate) id: String, + pub(crate) name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrackerTeam { + pub(crate) id: String, + pub(crate) name: String, + pub(crate) states: Vec, + pub(crate) labels: Vec, +} + +pub(crate) fn automation_queue_label(service_id: &str) -> String { + format!("decodex:queued:{service_id}") +} + +pub(crate) fn automation_active_label(service_id: &str) -> String { + format!("decodex:active:{service_id}") +} + +pub(crate) fn issue_has_label_with_server_confirmation( + tracker: &T, + issue: &TrackerIssue, + label_name: &str, +) -> Result +where + T: IssueTracker + ?Sized, +{ + if issue.has_label(label_name) { + return Ok(true); + } + if issue.labels_complete { + return Ok(false); + } + + Ok(tracker + .list_issues_with_label(label_name)? + .into_iter() + .any(|candidate| candidate.id == issue.id)) +} + +pub(crate) fn issue_team_label_id_with_server_confirmation( + tracker: &T, + issue: &TrackerIssue, + label_name: &str, +) -> Result> +where + T: IssueTracker + ?Sized, +{ + if let Some(label_id) = issue.label_id_for_name(label_name) { + return Ok(Some(label_id.to_owned())); + } + + tracker.find_team_label_id(&issue.team.id, label_name) +} + +pub(crate) fn set_issue_label_presence( + tracker: &T, + issue: &TrackerIssue, + label_name: &str, + present: bool, +) -> Result +where + T: IssueTracker + ?Sized, +{ + let label_present = issue_has_label_with_server_confirmation(tracker, issue, label_name)?; + + if label_present == present { + return Ok(false); + } + + let Some(label_id) = issue_team_label_id_with_server_confirmation(tracker, issue, label_name)? + else { + eyre::bail!( + "Issue `{}` does not expose required label `{}` on its team.", + issue.identifier, + label_name + ); + }; + + if present { + tracker.add_issue_labels(&issue.id, &[label_id])?; + } else if let Err(error) = tracker.remove_issue_labels(&issue.id, &[label_id]) { + if label_not_on_issue_error(&error) { + return Ok(false); + } + + return Err(error); + } + + Ok(true) +} + +pub(crate) fn label_not_on_issue_error(error: &Report) -> bool { + error + .chain() + .any(|source| source.to_string().to_ascii_lowercase().contains("label not on issue")) +} + +pub(crate) fn clear_automation_lane_labels( + tracker: &T, + issue: &TrackerIssue, + service_id: &str, +) -> Result<()> +where + T: IssueTracker + ?Sized, +{ + let mut refreshed_issues = tracker.refresh_issues(slice::from_ref(&issue.id))?; + let current_issue = refreshed_issues.pop().unwrap_or_else(|| issue.clone()); + let lane_labels = [automation_active_label(service_id), automation_queue_label(service_id)]; + + for label_name in lane_labels { + if !issue_has_label_with_server_confirmation(tracker, ¤t_issue, &label_name)? { + continue; + } + + let Some(label_id) = + issue_team_label_id_with_server_confirmation(tracker, ¤t_issue, &label_name)? + else { + eyre::bail!( + "Issue `{}` does not expose required label `{}` on its team.", + current_issue.identifier, + label_name + ); + }; + + if let Err(error) = tracker.remove_issue_labels(¤t_issue.id, &[label_id]) { + if label_not_on_issue_error(&error) { + continue; + } + + return Err(error); + } + } + + Ok(()) +} + +pub(crate) fn create_linear_execution_event_comment( + tracker: &T, + issue_id: &str, + body: &str, + record: &LinearExecutionEventRecord, +) -> Result +where + T: IssueTracker + ?Sized, +{ + records::validate_linear_execution_event_record(record).map_err(|error| eyre::eyre!(error))?; + + let comments = tracker.list_comments(issue_id)?; + + if records::has_linear_execution_event_record( + &comments, + &record.service_id, + &record.issue_id, + &record.idempotency_key, + ) { + return Ok(false); + } + + let comment_body = records::append_structured_comment_record(body, record)?; + + tracker.create_comment(issue_id, &comment_body)?; + + Ok(true) +} diff --git a/apps/decodex/src/tracker/linear.rs b/apps/decodex/src/tracker/linear.rs new file mode 100644 index 00000000..066c882b --- /dev/null +++ b/apps/decodex/src/tracker/linear.rs @@ -0,0 +1,1045 @@ +use reqwest::blocking::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + prelude::{Result, eyre}, + tracker::{ + IssueTracker, TrackerComment, TrackerIssue, TrackerIssueBlocker, TrackerLabel, + TrackerState, TrackerTeam, + }, +}; + +const LINEAR_GRAPHQL_URL: &str = "https://api.linear.app/graphql"; +const ISSUES_WITH_LABEL_QUERY: &str = r#" +query IssuesWithLabel($labelName: String!, $after: String) { + issues(filter: { labels: { name: { eq: $labelName } } }, first: 50, after: $after) { + nodes { + id + identifier + title + description + priority + createdAt + updatedAt + state { + id + name + } + team { + id + name + states(first: 50) { + nodes { + id + name + } + } + labels(first: 100) { + nodes { + id + name + } + } + } + labels(first: 50) { + nodes { + id + name + } + pageInfo { + hasNextPage + endCursor + } + } + inverseRelations(first: 50) { + nodes { + type + issue { + id + identifier + state { + id + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + pageInfo { + hasNextPage + endCursor + } + } +} +"#; +const ISSUE_BY_IDENTIFIER_QUERY: &str = r#" +query IssueByIdentifier($issueIdentifier: String!) { + issue(id: $issueIdentifier) { + id + identifier + title + description + priority + createdAt + updatedAt + state { + id + name + } + team { + id + name + states(first: 50) { + nodes { + id + name + } + } + labels(first: 100) { + nodes { + id + name + } + } + } + labels(first: 50) { + nodes { + id + name + } + pageInfo { + hasNextPage + endCursor + } + } + inverseRelations(first: 50) { + nodes { + type + issue { + id + identifier + state { + id + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +"#; +const ISSUES_BY_IDS_QUERY: &str = r#" +query IssuesByIds($issueIds: [ID!], $after: String) { + issues(filter: { id: { in: $issueIds } }, first: 50, after: $after) { + nodes { + id + identifier + title + description + priority + createdAt + updatedAt + state { + id + name + } + team { + id + name + states(first: 50) { + nodes { + id + name + } + } + labels(first: 100) { + nodes { + id + name + } + } + } + labels(first: 50) { + nodes { + id + name + } + pageInfo { + hasNextPage + endCursor + } + } + inverseRelations(first: 50) { + nodes { + type + issue { + id + identifier + state { + id + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + pageInfo { + hasNextPage + endCursor + } + } +} +"#; +const ISSUE_BLOCKERS_QUERY: &str = r#" +query IssueBlockers($issueId: String!, $after: String) { + issues(filter: { id: { eq: $issueId } }, first: 1) { + nodes { + inverseRelations(first: 50, after: $after) { + nodes { + type + issue { + id + identifier + state { + id + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} +"#; +const ISSUE_COMMENTS_QUERY: &str = r#" +query IssueComments($issueId: String!, $after: String) { + issue(id: $issueId) { + comments(first: 100, after: $after) { + nodes { + body + createdAt + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +"#; +const ISSUE_UPDATE_MUTATION: &str = r#" +mutation UpdateIssue($id: String!, $input: IssueUpdateInput!) { + issueUpdate(id: $id, input: $input) { + success + } +} +"#; +const COMMENT_CREATE_MUTATION: &str = r#" +mutation CreateComment($input: CommentCreateInput!) { + commentCreate(input: $input) { + success + } +} +"#; +const ISSUE_ARCHIVE_MUTATION: &str = r#" +mutation ArchiveIssue($id: String!, $trash: Boolean) { + issueArchive(id: $id, trash: $trash) { + success + } +} +"#; +const TEAM_LABEL_BY_NAME_QUERY: &str = r#" +query TeamLabelByName($teamId: ID!, $labelName: String!) { + issueLabels(filter: { team: { id: { eq: $teamId } }, name: { eq: $labelName } }, first: 1) { + nodes { + id + name + } + } +} +"#; + +pub(crate) struct LinearClient { + api_token: String, + http: Client, +} +impl LinearClient { + pub(crate) fn new(api_token: String) -> Result { + Ok(Self { api_token, http: Client::builder().build()? }) + } + + pub(crate) fn archive_issue(&self, issue_id: &str) -> Result<()> { + let data = self.post::<_, IssueArchiveData>( + ISSUE_ARCHIVE_MUTATION, + &IssueArchiveVariables { id: issue_id, trash: false }, + )?; + + if !data.issue_archive.success { + eyre::bail!("Linear did not confirm the issue archive mutation."); + } + + Ok(()) + } + + fn post(&self, query: &str, variables: &V) -> Result + where + V: Serialize, + T: for<'de> Deserialize<'de>, + { + let response = self + .http + .post(LINEAR_GRAPHQL_URL) + .header("Authorization", &self.api_token) + .json(&GraphqlRequest { query, variables }) + .send()?; + let status = response.status(); + let body = response.text()?; + let payload = serde_json::from_str::>(&body).map_err(|error| { + if status.is_success() { + eyre::eyre!("Failed to parse Linear GraphQL response: {error}") + } else { + eyre::eyre!( + "Linear HTTP request failed with status `{}` and an unparseable GraphQL body: {error}", + status + ) + } + })?; + + if let Some(errors) = payload.errors { + if let Some(message) = rate_limited_error_message(&errors) { + eyre::bail!("{message}"); + } + + let messages = + errors.into_iter().map(|error| error.message).collect::>().join("; "); + + eyre::bail!("Linear GraphQL request failed: {messages}"); + } + + if !status.is_success() { + eyre::bail!("Linear HTTP request failed with status `{status}`."); + } + + payload.data.ok_or_else(|| eyre::eyre!("Linear GraphQL response did not include data.")) + } + + fn collect_issue_pages( + &self, + query: &str, + mut make_variables: F, + ) -> Result> + where + V: Serialize, + F: FnMut(Option) -> V, + { + let mut after = None; + let mut issues = Vec::new(); + + loop { + let data = + self.post::<_, IssueConnectionData>(query, &make_variables(after.clone()))?; + let connection = data.issues; + + for issue in connection.nodes { + let blockers = self.resolve_issue_blockers(&issue)?; + + issues.push(map_issue(issue, blockers)); + } + + if !connection.page_info.has_next_page { + break; + } + + after = Some(require_end_cursor( + connection.page_info, + "Linear issue pagination reported `hasNextPage = true` without an `endCursor`.", + )?); + } + + Ok(issues) + } + + fn resolve_issue_blockers(&self, issue: &LinearIssue) -> Result> { + let mut blockers = map_blockers(&issue.inverse_relations.nodes); + + if issue.state.name != "Todo" || !issue.inverse_relations.page_info.has_next_page { + return Ok(blockers); + } + + let mut after = Some(require_end_cursor( + issue.inverse_relations.page_info.clone(), + "Linear blocker pagination reported `hasNextPage = true` without an `endCursor`.", + )?); + + while let Some(cursor) = after { + let data = self.post::<_, IssueBlockersData>( + ISSUE_BLOCKERS_QUERY, + &IssueBlockersVariables { issue_id: issue.id.clone(), after: Some(cursor) }, + )?; + let Some(issue_page) = data.issues.nodes.into_iter().next() else { + eyre::bail!( + "Linear blocker pagination did not return the requested issue `{}`.", + issue.id + ); + }; + let blocker_page = issue_page.inverse_relations; + + blockers.extend(map_blockers(&blocker_page.nodes)); + + after = if blocker_page.page_info.has_next_page { + Some(require_end_cursor( + blocker_page.page_info, + "Linear blocker pagination reported `hasNextPage = true` without an `endCursor`.", + )?) + } else { + None + }; + } + + Ok(blockers) + } + + fn collect_issue_comments(&self, issue_id: &str) -> Result> { + let mut after = None; + let mut comments = Vec::new(); + + loop { + let data = self.post::<_, IssueCommentsData>( + ISSUE_COMMENTS_QUERY, + &IssueCommentsVariables { issue_id: issue_id.to_owned(), after: after.clone() }, + )?; + let Some(issue) = data.issue else { + eyre::bail!("Linear did not return issue `{issue_id}` while listing comments."); + }; + let connection = issue.comments; + + comments.extend(connection.nodes.into_iter().map(|comment| TrackerComment { + body: comment.body, + created_at: comment.created_at, + })); + + if !connection.page_info.has_next_page { + break; + } + + after = Some(require_end_cursor( + connection.page_info, + "Linear comment pagination reported `hasNextPage = true` without an `endCursor`.", + )?); + } + + Ok(comments) + } +} + +impl IssueTracker for LinearClient { + fn list_issues_with_label(&self, label_name: &str) -> Result> { + self.collect_issue_pages(ISSUES_WITH_LABEL_QUERY, |after| IssuesWithLabelVariables { + label_name: label_name.to_owned(), + after, + }) + } + + fn find_team_label_id(&self, team_id: &str, label_name: &str) -> Result> { + let data = self.post::<_, TeamLabelByNameData>( + TEAM_LABEL_BY_NAME_QUERY, + &TeamLabelByNameVariables { + team_id: team_id.to_owned(), + label_name: label_name.to_owned(), + }, + )?; + + Ok(data.issue_labels.nodes.into_iter().next().map(|label| label.id)) + } + + fn get_issue_by_identifier(&self, issue_identifier: &str) -> Result> { + let data = self.post::<_, IssueByIdentifierData>( + ISSUE_BY_IDENTIFIER_QUERY, + &IssueByIdentifierVariables { issue_identifier: issue_identifier.to_owned() }, + )?; + let Some(issue) = data.issue else { + return Ok(None); + }; + let blockers = self.resolve_issue_blockers(&issue)?; + + Ok(Some(map_issue(issue, blockers))) + } + + fn refresh_issues(&self, issue_ids: &[String]) -> Result> { + if issue_ids.is_empty() { + return Ok(Vec::new()); + } + + self.collect_issue_pages(ISSUES_BY_IDS_QUERY, |after| IssuesByIdsVariables { + issue_ids: issue_ids.to_vec(), + after, + }) + } + + fn list_comments(&self, issue_id: &str) -> Result> { + self.collect_issue_comments(issue_id) + } + + fn update_issue_state(&self, issue_id: &str, state_id: &str) -> Result<()> { + let data = self.post::<_, IssueUpdateData>( + ISSUE_UPDATE_MUTATION, + &IssueUpdateVariables { + id: issue_id, + input: IssueUpdateInput { + state_id: Some(state_id.to_owned()), + label_ids: None, + added_label_ids: None, + removed_label_ids: None, + }, + }, + )?; + + if !data.issue_update.success { + eyre::bail!("Linear did not confirm the issue state update."); + } + + Ok(()) + } + + fn add_issue_labels(&self, issue_id: &str, label_ids: &[String]) -> Result<()> { + let data = self.post::<_, IssueUpdateData>( + ISSUE_UPDATE_MUTATION, + &IssueUpdateVariables { + id: issue_id, + input: IssueUpdateInput { + state_id: None, + label_ids: None, + added_label_ids: Some(label_ids.to_vec()), + removed_label_ids: None, + }, + }, + )?; + + if !data.issue_update.success { + eyre::bail!("Linear did not confirm the issue label addition."); + } + + Ok(()) + } + + fn remove_issue_labels(&self, issue_id: &str, label_ids: &[String]) -> Result<()> { + let data = self.post::<_, IssueUpdateData>( + ISSUE_UPDATE_MUTATION, + &IssueUpdateVariables { + id: issue_id, + input: IssueUpdateInput { + state_id: None, + label_ids: None, + added_label_ids: None, + removed_label_ids: Some(label_ids.to_vec()), + }, + }, + )?; + + if !data.issue_update.success { + eyre::bail!("Linear did not confirm the issue label removal."); + } + + Ok(()) + } + + fn create_comment(&self, issue_id: &str, body: &str) -> Result<()> { + let data = self.post::<_, CommentCreateData>( + COMMENT_CREATE_MUTATION, + &CommentCreateVariables { + input: CommentCreateInput { body: body.to_owned(), issue_id: issue_id.to_owned() }, + }, + )?; + + if !data.comment_create.success { + eyre::bail!("Linear did not confirm the comment creation."); + } + + Ok(()) + } +} + +#[derive(Serialize)] +struct GraphqlRequest<'a, V> { + query: &'a str, + variables: V, +} + +#[derive(Deserialize)] +struct GraphqlResponse { + data: Option, + errors: Option>, +} + +#[derive(Deserialize)] +struct GraphqlError { + message: String, + extensions: Option, +} + +#[derive(Serialize)] +struct IssuesWithLabelVariables { + #[serde(rename = "labelName")] + label_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, +} + +#[derive(Serialize)] +struct IssueByIdentifierVariables { + #[serde(rename = "issueIdentifier")] + issue_identifier: String, +} + +#[derive(Serialize)] +struct IssuesByIdsVariables { + #[serde(rename = "issueIds")] + issue_ids: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, +} + +#[derive(Serialize)] +struct IssueBlockersVariables { + #[serde(rename = "issueId")] + issue_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, +} + +#[derive(Serialize)] +struct IssueCommentsVariables { + #[serde(rename = "issueId")] + issue_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, +} + +#[derive(Deserialize)] +struct IssueConnectionData { + issues: IssueConnection, +} + +#[derive(Deserialize)] +struct IssueByIdentifierData { + issue: Option, +} + +#[derive(Deserialize)] +struct IssueBlockersData { + issues: IssueBlockerConnection, +} + +#[derive(Deserialize)] +struct IssueCommentsData { + issue: Option, +} + +#[derive(Deserialize)] +struct IssueConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Deserialize)] +struct IssueBlockerConnection { + nodes: Vec, +} + +#[derive(Deserialize)] +struct LinearIssueBlockerPage { + #[serde(rename = "inverseRelations")] + inverse_relations: IssueRelationConnection, +} + +#[derive(Deserialize)] +struct LinearIssueComments { + comments: CommentConnection, +} + +#[derive(Deserialize)] +struct CommentConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Deserialize)] +struct LinearComment { + body: String, + #[serde(rename = "createdAt")] + created_at: String, +} + +#[derive(Clone, Deserialize)] +struct PageInfo { + #[serde(rename = "hasNextPage")] + has_next_page: bool, + #[serde(rename = "endCursor")] + end_cursor: Option, +} + +#[derive(Deserialize)] +struct LinearIssue { + id: String, + identifier: String, + title: String, + description: Option, + priority: Option, + #[serde(rename = "createdAt")] + created_at: String, + #[serde(rename = "updatedAt")] + updated_at: String, + state: LinearState, + team: LinearTeam, + labels: LabelConnection, + #[serde(rename = "inverseRelations")] + inverse_relations: IssueRelationConnection, +} + +#[derive(Deserialize)] +struct LinearTeam { + id: String, + name: String, + states: StateConnection, + labels: LabelConnection, +} + +#[derive(Deserialize)] +struct StateConnection { + nodes: Vec, +} + +#[derive(Deserialize)] +struct LabelConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: Option, +} + +#[derive(Deserialize)] +struct IssueRelationConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Deserialize)] +struct LinearIssueRelation { + #[serde(rename = "type")] + relation_type: String, + issue: LinearRelatedIssue, +} + +#[derive(Deserialize)] +struct LinearRelatedIssue { + id: String, + identifier: String, + state: LinearState, +} + +#[derive(Deserialize)] +struct LinearState { + id: String, + name: String, +} + +#[derive(Deserialize)] +struct LinearLabel { + id: String, + name: String, +} + +#[derive(Serialize)] +struct IssueUpdateVariables<'a> { + id: &'a str, + input: IssueUpdateInput, +} + +#[derive(Serialize)] +struct IssueUpdateInput { + #[serde(rename = "stateId", skip_serializing_if = "Option::is_none")] + state_id: Option, + #[serde(rename = "labelIds", skip_serializing_if = "Option::is_none")] + label_ids: Option>, + #[serde(rename = "addedLabelIds", skip_serializing_if = "Option::is_none")] + added_label_ids: Option>, + #[serde(rename = "removedLabelIds", skip_serializing_if = "Option::is_none")] + removed_label_ids: Option>, +} + +#[derive(Deserialize)] +struct IssueUpdateData { + #[serde(rename = "issueUpdate")] + issue_update: MutationSuccess, +} + +#[derive(Serialize)] +struct IssueArchiveVariables<'a> { + id: &'a str, + trash: bool, +} + +#[derive(Deserialize)] +struct IssueArchiveData { + #[serde(rename = "issueArchive")] + issue_archive: MutationSuccess, +} + +#[derive(Deserialize)] +struct MutationSuccess { + success: bool, +} + +#[derive(Serialize)] +struct TeamLabelByNameVariables { + #[serde(rename = "teamId")] + team_id: String, + #[serde(rename = "labelName")] + label_name: String, +} + +#[derive(Deserialize)] +struct TeamLabelByNameData { + #[serde(rename = "issueLabels")] + issue_labels: LabelConnection, +} + +#[derive(Serialize)] +struct CommentCreateVariables { + input: CommentCreateInput, +} + +#[derive(Serialize)] +struct CommentCreateInput { + body: String, + #[serde(rename = "issueId")] + issue_id: String, +} + +#[derive(Deserialize)] +struct CommentCreateData { + #[serde(rename = "commentCreate")] + comment_create: MutationSuccess, +} + +fn rate_limited_error_message(errors: &[GraphqlError]) -> Option { + errors.iter().find_map(|error| { + let extensions = error.extensions.as_ref()?; + let code = extensions.get("code").and_then(Value::as_str)?; + + if code != "RATELIMITED" { + return None; + } + + let user_message = extensions + .get("userPresentableMessage") + .and_then(Value::as_str) + .unwrap_or(error.message.as_str()); + let reset = extensions.get("reset").and_then(Value::as_i64); + + Some(match reset { + Some(reset) => + format!("Linear connector is rate limited until `{reset}`: {user_message}"), + None => format!("Linear connector is rate limited: {user_message}"), + }) + }) +} + +fn require_end_cursor(page_info: PageInfo, message: &str) -> Result { + page_info.end_cursor.ok_or_else(|| eyre::eyre!(message.to_owned())) +} + +fn map_blockers(relations: &[LinearIssueRelation]) -> Vec { + relations + .iter() + .filter(|relation| relation.relation_type == "blocks") + .map(|relation| TrackerIssueBlocker { + id: relation.issue.id.clone(), + identifier: relation.issue.identifier.clone(), + state: TrackerState { + id: relation.issue.state.id.clone(), + name: relation.issue.state.name.clone(), + }, + }) + .collect() +} + +fn map_issue(issue: LinearIssue, blockers: Vec) -> TrackerIssue { + TrackerIssue { + id: issue.id, + identifier: issue.identifier, + #[cfg(test)] + project_slug: None, + title: issue.title, + description: issue.description.unwrap_or_default(), + priority: issue.priority, + created_at: issue.created_at, + updated_at: issue.updated_at, + state: TrackerState { id: issue.state.id, name: issue.state.name }, + team: TrackerTeam { + id: issue.team.id, + name: issue.team.name, + states: issue + .team + .states + .nodes + .into_iter() + .map(|state| TrackerState { id: state.id, name: state.name }) + .collect(), + labels: issue + .team + .labels + .nodes + .into_iter() + .map(|label| TrackerLabel { id: label.id, name: label.name }) + .collect(), + }, + labels_complete: issue.labels.page_info.is_none_or(|page_info| !page_info.has_next_page), + labels: issue + .labels + .nodes + .into_iter() + .map(|label| TrackerLabel { id: label.id, name: label.name }) + .collect(), + blockers, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::tracker::linear::{ + GraphqlError, IssueRelationConnection, LabelConnection, LinearIssue, LinearIssueRelation, + LinearLabel, LinearRelatedIssue, LinearState, LinearTeam, PageInfo, StateConnection, + }; + + #[test] + fn map_issue_preserves_priority_and_created_at() { + let issue = LinearIssue { + id: String::from("issue-1"), + identifier: String::from("PUB-101"), + title: String::from("Implement ordering"), + description: Some(String::from("Body")), + priority: Some(2), + created_at: String::from("2026-03-13T04:16:17.133Z"), + updated_at: String::from("2026-03-14T04:16:17.133Z"), + state: LinearState { id: String::from("state-todo"), name: String::from("Todo") }, + team: LinearTeam { + id: String::from("team-1"), + name: String::from("Pubfi"), + states: StateConnection { + nodes: vec![LinearState { + id: String::from("state-todo"), + name: String::from("Todo"), + }], + }, + labels: LabelConnection { + nodes: vec![LinearLabel { + id: String::from("label-needs"), + name: String::from("decodex:needs-attention"), + }], + page_info: None, + }, + }, + labels: LabelConnection { + nodes: vec![LinearLabel { + id: String::from("label-manual"), + name: String::from("decodex:manual-only"), + }], + page_info: Some(PageInfo { has_next_page: false, end_cursor: None }), + }, + inverse_relations: IssueRelationConnection { + nodes: vec![LinearIssueRelation { + relation_type: String::from("blocks"), + issue: LinearRelatedIssue { + id: String::from("issue-2"), + identifier: String::from("PUB-102"), + state: LinearState { + id: String::from("state-progress"), + name: String::from("In Progress"), + }, + }, + }], + page_info: PageInfo { has_next_page: false, end_cursor: None }, + }, + }; + let blockers = super::map_blockers(&issue.inverse_relations.nodes); + let mapped = super::map_issue(issue, blockers); + + assert_eq!(mapped.priority, Some(2)); + assert_eq!(mapped.created_at, "2026-03-13T04:16:17.133Z"); + assert_eq!(mapped.updated_at, "2026-03-14T04:16:17.133Z"); + assert_eq!(mapped.blockers.len(), 1); + assert_eq!(mapped.blockers[0].identifier, "PUB-102"); + assert_eq!(mapped.blockers[0].state.name, "In Progress"); + } + + #[test] + fn map_blockers_filters_non_blocking_relations() { + let blockers = super::map_blockers(&[ + LinearIssueRelation { + relation_type: String::from("blocks"), + issue: LinearRelatedIssue { + id: String::from("issue-2"), + identifier: String::from("PUB-102"), + state: LinearState { + id: String::from("state-progress"), + name: String::from("In Progress"), + }, + }, + }, + LinearIssueRelation { + relation_type: String::from("related"), + issue: LinearRelatedIssue { + id: String::from("issue-3"), + identifier: String::from("PUB-103"), + state: LinearState { + id: String::from("state-done"), + name: String::from("Done"), + }, + }, + }, + ]); + + assert_eq!(blockers.len(), 1); + assert_eq!(blockers[0].identifier, "PUB-102"); + } + + #[test] + fn rate_limited_error_message_uses_typed_linear_extensions() { + let errors = vec![GraphqlError { + message: String::from("Too many requests"), + extensions: Some(json!({ + "code": "RATELIMITED", + "userPresentableMessage": "API rate limit exceeded", + "reset": 1_777_392_000 + })), + }]; + let message = + super::rate_limited_error_message(&errors).expect("rate limit should classify"); + + assert!(message.contains("rate limited")); + assert!(message.contains("1777392000")); + assert!(message.contains("API rate limit exceeded")); + } +} diff --git a/apps/decodex/src/tracker/records.rs b/apps/decodex/src/tracker/records.rs new file mode 100644 index 00000000..5bf4a384 --- /dev/null +++ b/apps/decodex/src/tracker/records.rs @@ -0,0 +1,421 @@ +use std::path::{Component, Path}; + +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::Error; + +use crate::tracker::TrackerComment; + +#[cfg(test)] +pub(crate) const REVIEW_HANDOFF_RECORD_TYPE: &str = "review-handoff-record/1"; +#[cfg(test)] +pub(crate) const CLOSEOUT_RECORD_TYPE: &str = "closeout-record/1"; +pub(crate) const LINEAR_EXECUTION_EVENT_RECORD_TYPE: &str = "decodex.linear_execution_event"; +pub(crate) const LINEAR_EXECUTION_EVENT_RECORD_VERSION: i64 = 1; + +#[cfg(test)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +pub(crate) struct ReviewHandoffRecord { + #[serde(rename = "type")] + pub(crate) record_type: String, + pub(crate) completed_at: String, + pub(crate) run_id: String, + pub(crate) attempt_number: i64, + pub(crate) branch_name: String, + pub(crate) pr_url: String, + pub(crate) target_base_ref_name: String, + pub(crate) pr_head_ref_name: String, + pub(crate) pr_head_oid: String, + pub(crate) summary: String, +} + +#[cfg(test)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +pub(crate) struct CloseoutRecord { + #[serde(rename = "type")] + pub(crate) record_type: String, + pub(crate) completed_at: String, + pub(crate) run_id: String, + pub(crate) attempt_number: i64, + pub(crate) branch_name: String, + pub(crate) pr_url: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LinearExecutionEventRecord { + pub(crate) record_type: String, + pub(crate) record_version: i64, + pub(crate) event_type: String, + pub(crate) event_timestamp: String, + pub(crate) idempotency_key: String, + pub(crate) service_id: String, + pub(crate) issue_id: String, + pub(crate) issue_identifier: String, + pub(crate) run_id: String, + pub(crate) attempt_number: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) branch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) worktree_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) commit_sha: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pr_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pr_head_sha: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pr_base_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) validation_result: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) phase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) focus: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) next_action: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) blockers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) evidence: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) verification: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) error_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) terminal_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) cleanup_status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) transport: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) target_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) failed_command: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) raw_error: Option, +} +impl LinearExecutionEventRecord { + pub(crate) fn new( + identity: LinearExecutionEventIdentity<'_>, + event_type: &str, + event_timestamp: String, + stable_anchor: &str, + ) -> Self { + Self { + record_type: String::from(LINEAR_EXECUTION_EVENT_RECORD_TYPE), + record_version: LINEAR_EXECUTION_EVENT_RECORD_VERSION, + event_type: event_type.to_owned(), + event_timestamp, + idempotency_key: linear_execution_idempotency_key( + identity.service_id, + identity.issue_identifier, + identity.run_id, + identity.attempt_number, + event_type, + stable_anchor, + ), + service_id: identity.service_id.to_owned(), + issue_id: identity.issue_id.to_owned(), + issue_identifier: identity.issue_identifier.to_owned(), + run_id: identity.run_id.to_owned(), + attempt_number: identity.attempt_number, + branch: None, + worktree_path: None, + commit_sha: None, + pr_url: None, + pr_head_sha: None, + pr_base_ref: None, + summary: None, + validation_result: None, + phase: None, + focus: None, + next_action: None, + blockers: None, + evidence: None, + verification: None, + error_class: None, + terminal_path: None, + cleanup_status: None, + transport: None, + target_state: None, + failed_command: None, + raw_error: None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct LinearExecutionEventIdentity<'a> { + pub(crate) service_id: &'a str, + pub(crate) issue_id: &'a str, + pub(crate) issue_identifier: &'a str, + pub(crate) run_id: &'a str, + pub(crate) attempt_number: i64, +} + +pub(crate) fn format_structured_comment(record: &T) -> std::result::Result +where + T: Serialize, +{ + let payload = serde_json::to_string_pretty(record)?; + + Ok(format!("```json\n{payload}\n```")) +} + +pub(crate) fn append_structured_comment_record( + body: &str, + record: &T, +) -> std::result::Result +where + T: Serialize, +{ + let payload = format_structured_comment(record)?; + + if body.trim().is_empty() { + return Ok(payload); + } + + Ok(format!("{body}\n\n{payload}")) +} + +pub(crate) fn stable_event_anchor(parts: &[&str]) -> String { + let mut hash = 0xcbf29ce484222325_u64; + + for part in parts { + for byte in part.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + + hash ^= 0xff; + hash = hash.wrapping_mul(0x100000001b3); + } + + format!("{hash:016x}") +} + +pub(crate) fn validate_linear_execution_event_record( + record: &LinearExecutionEventRecord, +) -> Result<(), String> { + validate_linear_execution_event_envelope(record)?; + validate_linear_execution_event_fields(record)?; + + if let Some(worktree_path) = record.worktree_path.as_deref() { + validate_repo_relative_path(worktree_path, "worktree_path")?; + } + + Ok(()) +} + +pub(crate) fn has_linear_execution_event_record( + comments: &[TrackerComment], + service_id: &str, + issue_id: &str, + idempotency_key: &str, +) -> bool { + comments.iter().filter_map(|comment| parse_linear_execution_event_record(&comment.body)).any( + |record| { + record.service_id == service_id + && record.issue_id == issue_id + && record.idempotency_key == idempotency_key + }, + ) +} + +pub(crate) fn parse_linear_execution_event_record( + body: &str, +) -> Option { + parse_structured_comment::(body) + .filter(|record| validate_linear_execution_event_record(record).is_ok()) +} + +fn parse_structured_comment(body: &str) -> Option +where + T: DeserializeOwned, +{ + extract_structured_json_blocks(body) + .into_iter() + .rev() + .find_map(|payload| serde_json::from_str::(payload).ok()) + .or_else(|| serde_json::from_str::(body.trim()).ok()) +} + +fn extract_structured_json_blocks(body: &str) -> Vec<&str> { + body.match_indices("```json") + .filter_map(|(start, _)| { + let fenced = &body[start + "```json".len()..]; + let fenced = fenced.strip_prefix("\r\n").or_else(|| fenced.strip_prefix('\n'))?; + let end = fenced.find("\n```").or_else(|| fenced.find("\r\n```"))?; + + Some(fenced[..end].trim()) + }) + .collect() +} + +fn linear_execution_idempotency_key( + service_id: &str, + issue_identifier: &str, + run_id: &str, + attempt_number: i64, + event_type: &str, + stable_anchor: &str, +) -> String { + format!( + "{service_id}:{issue_identifier}:{run_id}:{attempt_number}:{event_type}:{stable_anchor}" + ) +} + +fn validate_linear_execution_event_envelope( + record: &LinearExecutionEventRecord, +) -> Result<(), String> { + if record.record_type != LINEAR_EXECUTION_EVENT_RECORD_TYPE { + return Err(format!("`record_type` must be `{LINEAR_EXECUTION_EVENT_RECORD_TYPE}`.")); + } + if record.record_version != LINEAR_EXECUTION_EVENT_RECORD_VERSION { + return Err(format!("`record_version` must be `{LINEAR_EXECUTION_EVENT_RECORD_VERSION}`.")); + } + + for (field, value) in [ + ("event_type", record.event_type.as_str()), + ("event_timestamp", record.event_timestamp.as_str()), + ("idempotency_key", record.idempotency_key.as_str()), + ("service_id", record.service_id.as_str()), + ("issue_id", record.issue_id.as_str()), + ("issue_identifier", record.issue_identifier.as_str()), + ("run_id", record.run_id.as_str()), + ] { + if value.trim().is_empty() { + return Err(format!("`{field}` must not be empty.")); + } + } + + if record.attempt_number < 1 { + return Err(String::from("`attempt_number` must be at least 1.")); + } + + Ok(()) +} + +fn validate_linear_execution_event_fields( + record: &LinearExecutionEventRecord, +) -> Result<(), String> { + match record.event_type.as_str() { + "intake" => require_string(record.summary.as_deref(), "summary"), + "lease_acquired" => { + require_string(record.branch.as_deref(), "branch")?; + + Ok(()) + }, + "worktree_prepared" => { + require_string(record.branch.as_deref(), "branch")?; + require_string(record.worktree_path.as_deref(), "worktree_path")?; + + require_string(record.commit_sha.as_deref(), "commit_sha") + }, + "agent_started" => { + require_string(record.branch.as_deref(), "branch")?; + + require_string(record.worktree_path.as_deref(), "worktree_path") + }, + "progress_checkpoint" => { + require_string(record.phase.as_deref(), "phase")?; + require_string(record.focus.as_deref(), "focus")?; + require_string(record.next_action.as_deref(), "next_action")?; + require_vec(record.blockers.as_ref(), "blockers")?; + + require_vec(record.evidence.as_ref(), "evidence") + }, + "pr_opened" | "pr_updated" => validate_pr_event_fields(record), + "review_handoff" | "repair_handoff" => { + validate_pr_event_fields(record)?; + require_string(record.validation_result.as_deref(), "validation_result")?; + require_string(record.summary.as_deref(), "summary")?; + + require_string(record.terminal_path.as_deref(), "terminal_path") + }, + "landed" => { + validate_pr_event_fields(record)?; + + require_string(record.summary.as_deref(), "summary") + }, + "closeout" => { + require_string(record.pr_url.as_deref(), "pr_url")?; + require_string(record.commit_sha.as_deref(), "commit_sha")?; + + require_string(record.summary.as_deref(), "summary") + }, + "needs_attention" => { + require_string(record.error_class.as_deref(), "error_class")?; + require_string(record.next_action.as_deref(), "next_action")?; + require_vec(record.blockers.as_ref(), "blockers")?; + require_vec(record.evidence.as_ref(), "evidence")?; + + require_string(record.terminal_path.as_deref(), "terminal_path") + }, + "terminal_failure" => { + require_string(record.error_class.as_deref(), "error_class")?; + require_string(record.next_action.as_deref(), "next_action")?; + require_vec(record.blockers.as_ref(), "blockers")?; + + require_vec(record.evidence.as_ref(), "evidence") + }, + "cleanup_complete" => { + require_string(record.branch.as_deref(), "branch")?; + require_string(record.worktree_path.as_deref(), "worktree_path")?; + require_string(record.cleanup_status.as_deref(), "cleanup_status")?; + + require_string(record.summary.as_deref(), "summary") + }, + other => Err(format!("Unsupported Linear execution event type `{other}`.")), + } +} + +fn validate_pr_event_fields(record: &LinearExecutionEventRecord) -> Result<(), String> { + require_string(record.branch.as_deref(), "branch")?; + require_string(record.pr_url.as_deref(), "pr_url")?; + require_string(record.pr_head_sha.as_deref(), "pr_head_sha")?; + require_string(record.pr_base_ref.as_deref(), "pr_base_ref")?; + + require_string(record.commit_sha.as_deref(), "commit_sha") +} + +fn require_string(value: Option<&str>, field: &str) -> Result<(), String> { + if value.is_some_and(|value| !value.trim().is_empty()) { + return Ok(()); + } + + Err(format!("`{field}` is required for this Linear execution event.")) +} + +fn require_vec(value: Option<&Vec>, field: &str) -> Result<(), String> { + if value.is_some() { + return Ok(()); + } + + Err(format!("`{field}` is required for this Linear execution event.")) +} + +fn validate_repo_relative_path(path: &str, field_name: &str) -> Result<(), String> { + if path.is_empty() { + return Err(format!("`{field_name}` must not be empty.")); + } + if path.starts_with('/') || path.starts_with("~/") || has_drive_root_prefix(path) { + return Err(format!("`{field_name}` must be repository-relative, not `{path}`.")); + } + if Path::new(path).components().any(|component| matches!(component, Component::ParentDir)) { + return Err(format!("`{field_name}` must stay within the repository, not `{path}`.")); + } + + Ok(()) +} + +fn has_drive_root_prefix(path: &str) -> bool { + let bytes = path.as_bytes(); + + bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/') +} diff --git a/apps/decodex/src/workflow.rs b/apps/decodex/src/workflow.rs new file mode 100644 index 00000000..007df9fc --- /dev/null +++ b/apps/decodex/src/workflow.rs @@ -0,0 +1,1715 @@ +//! Downstream `WORKFLOW.md` parsing and validation. + +use std::{ + collections::{BTreeSet, HashMap}, + fs, + path::{Component, Path}, +}; + +use globset::{Glob, GlobSet, GlobSetBuilder}; +use serde::{Deserialize, Serialize}; + +use crate::prelude::{Result, eyre}; + +const FRONTMATTER_DELIMITER: &str = "+++"; + +/// Parsed downstream workflow document. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WorkflowDocument { + frontmatter: WorkflowFrontmatter, + body: String, +} +impl WorkflowDocument { + /// Parse a workflow document from Markdown text. + pub fn parse_markdown(input: &str) -> Result { + let (frontmatter_input, body) = split_frontmatter(input)?; + let frontmatter = toml::from_str::(&frontmatter_input)?; + + frontmatter.validate()?; + + Ok(Self { frontmatter, body }) + } + + /// Load a workflow document from the repository root. + pub fn from_path(path: impl AsRef) -> Result { + let input = fs::read_to_string(path)?; + + Self::parse_markdown(&input) + } + + /// Machine-readable frontmatter for orchestration behavior. + pub fn frontmatter(&self) -> &WorkflowFrontmatter { + &self.frontmatter + } + + /// Human-readable Markdown policy body. + pub fn body(&self) -> &str { + &self.body + } + + /// Render the workflow back to Markdown for process-to-process handoff. + pub fn to_markdown(&self) -> Result { + let frontmatter = toml::to_string(&self.frontmatter)?; + let mut markdown = format!("{FRONTMATTER_DELIMITER}\n{frontmatter}{FRONTMATTER_DELIMITER}"); + + if !self.body.is_empty() { + markdown.push_str("\n\n"); + markdown.push_str(&self.body); + } + + Ok(markdown) + } +} + +/// Typed TOML frontmatter for a downstream workflow document. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowFrontmatter { + version: u8, + tracker: WorkflowTracker, + agent: WorkflowAgent, + execution: WorkflowExecution, + context: WorkflowContext, +} +impl WorkflowFrontmatter { + /// Contract version. + pub fn version(&self) -> u8 { + self.version + } + + /// Tracker policy for this repository. + pub fn tracker(&self) -> &WorkflowTracker { + &self.tracker + } + + /// Agent defaults for this repository. + pub fn agent(&self) -> &WorkflowAgent { + &self.agent + } + + /// Execution policy for this repository. + pub fn execution(&self) -> &WorkflowExecution { + &self.execution + } + + /// Extra early-load context paths for this repository. + pub fn context(&self) -> &WorkflowContext { + &self.context + } + + fn validate(&self) -> Result<()> { + if self.version != 1 { + eyre::bail!("Unsupported WORKFLOW.md version: {}", self.version); + } + + validate_non_empty_string_list("tracker.startable_states", &self.tracker.startable_states)?; + validate_non_empty_string_list("tracker.terminal_states", &self.tracker.terminal_states)?; + validate_trimmed_non_empty("tracker.in_progress_state", &self.tracker.in_progress_state)?; + validate_trimmed_non_empty("tracker.success_state", &self.tracker.success_state)?; + validate_trimmed_non_empty("tracker.failure_state", &self.tracker.failure_state)?; + validate_trimmed_non_empty("tracker.opt_out_label", &self.tracker.opt_out_label)?; + validate_trimmed_non_empty( + "tracker.needs_attention_label", + &self.tracker.needs_attention_label, + )?; + validate_trimmed_non_empty("agent.transport", &self.agent.transport)?; + + if self.execution.max_attempts == 0 { + eyre::bail!("`execution.max_attempts` must be greater than zero."); + } + if self.execution.max_turns == 0 { + eyre::bail!("`execution.max_turns` must be greater than zero."); + } + if self.execution.max_retry_backoff_ms == 0 { + eyre::bail!("`execution.max_retry_backoff_ms` must be greater than zero."); + } + + validate_trimmed_non_empty("tracker.completed_state", &self.tracker.completed_state)?; + + if !self.tracker.terminal_states.iter().any(|state| state == &self.tracker.completed_state) + { + eyre::bail!("`tracker.completed_state` must be one of `tracker.terminal_states`."); + } + + self.execution.validate()?; + self.context.validate()?; + + Ok(()) + } +} + +/// Tracker-facing repository policy. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowTracker { + provider: TrackerProvider, + startable_states: Vec, + terminal_states: Vec, + in_progress_state: String, + success_state: String, + completed_state: String, + failure_state: String, + opt_out_label: String, + needs_attention_label: String, +} +impl WorkflowTracker { + /// Tracker provider for this repository. + pub fn provider(&self) -> TrackerProvider { + self.provider + } + + /// States that are eligible for automatic execution. + pub fn startable_states(&self) -> &[String] { + &self.startable_states + } + + /// States that are considered terminal for automatic execution. + pub fn terminal_states(&self) -> &[String] { + &self.terminal_states + } + + /// State used when `decodex` starts work on an issue. + pub fn in_progress_state(&self) -> &str { + &self.in_progress_state + } + + /// State used after a successful run and validation pass. + pub fn success_state(&self) -> &str { + &self.success_state + } + + /// Explicit state used after a successful post-merge closeout. + pub fn completed_state(&self) -> &str { + &self.completed_state + } + + /// State used after a successful post-merge closeout. + pub fn resolved_completed_state(&self) -> &str { + &self.completed_state + } + + /// State used when retries are exhausted. + pub fn failure_state(&self) -> &str { + &self.failure_state + } + + /// Label that disables automation for an issue. + pub fn opt_out_label(&self) -> &str { + &self.opt_out_label + } + + /// Label that marks failed runs needing human attention. + pub fn needs_attention_label(&self) -> &str { + &self.needs_attention_label + } +} + +/// Repo-local agent defaults. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowAgent { + transport: String, +} +impl WorkflowAgent { + /// App-server transport. + pub fn transport(&self) -> &str { + &self.transport + } +} + +/// Repo-local execution and repo-gate policy. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowExecution { + max_attempts: u32, + max_turns: u32, + max_retry_backoff_ms: u64, + max_concurrent_agents: u32, + canonicalize_commands: Vec, + verify_commands: Vec, + gate_profiles: HashMap, + workspace_hooks: WorkflowWorkspaceHooks, +} +impl WorkflowExecution { + /// Maximum automatic attempts before human attention is required. + pub fn max_attempts(&self) -> u32 { + self.max_attempts + } + + /// Maximum same-thread turns per bounded run before Decodex yields cleanly. + pub fn max_turns(&self) -> u32 { + self.max_turns + } + + /// Maximum failure-retry backoff in milliseconds. + pub fn max_retry_backoff_ms(&self) -> u64 { + self.max_retry_backoff_ms + } + + /// Repo canonicalize commands that may rewrite the worktree before verification. + pub fn canonicalize_commands(&self) -> &[String] { + &self.canonicalize_commands + } + + /// Repo verification commands that must pass after canonicalize commands complete. + pub fn verify_commands(&self) -> &[String] { + &self.verify_commands + } + + /// Repo-owned named gate profiles for narrow path-scoped validation. + pub fn gate_profiles(&self) -> &HashMap { + &self.gate_profiles + } + + /// Repo-owned workspace lifecycle hooks. + pub fn workspace_hooks(&self) -> &WorkflowWorkspaceHooks { + &self.workspace_hooks + } + + /// Maximum concurrent agents allowed for this repository. + pub fn max_concurrent_agents(&self) -> u32 { + self.max_concurrent_agents + } + + /// Full default repo gate declared directly on `[execution]`. + pub fn default_repo_gate(&self) -> ResolvedRepoGate<'_> { + ResolvedRepoGate { + profile_name: None, + canonicalize_commands: &self.canonicalize_commands, + verify_commands: &self.verify_commands, + } + } + + /// Resolve the repo gate for a concrete changed-file set. + pub fn select_repo_gate_for_changed_files( + &self, + changed_files: &BTreeSet, + ) -> ResolvedRepoGate<'_> { + if changed_files.is_empty() { + return self.default_repo_gate(); + } + + let mut matching_profiles = self + .gate_profiles + .iter() + .filter_map(|(profile_name, profile)| { + profile.matches_changed_files(changed_files).ok().and_then(|matches| { + matches.then_some(ResolvedRepoGate { + profile_name: Some(profile_name.as_str()), + canonicalize_commands: profile.canonicalize_commands(), + verify_commands: profile.verify_commands(), + }) + }) + }) + .collect::>(); + + if matching_profiles.len() == 1 { + return matching_profiles.remove(0); + } + + self.default_repo_gate() + } + + fn validate(&self) -> Result<()> { + if self.max_concurrent_agents == 0 { + eyre::bail!("`execution.max_concurrent_agents` must be greater than zero."); + } + + validate_string_entries("execution.canonicalize_commands", &self.canonicalize_commands)?; + validate_string_entries("execution.verify_commands", &self.verify_commands)?; + + for (profile_name, profile) in &self.gate_profiles { + let trimmed = profile_name.trim(); + + if trimmed.is_empty() { + eyre::bail!("`execution.gate_profiles` keys must not be empty."); + } + if trimmed != profile_name { + eyre::bail!( + "`execution.gate_profiles.{profile_name}` must not include surrounding whitespace." + ); + } + + profile.validate(profile_name)?; + } + + self.workspace_hooks.validate()?; + + Ok(()) + } +} + +/// Repo-owned workspace lifecycle hooks around linked worktree setup and cleanup. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowWorkspaceHooks { + after_create_commands: Vec, + before_remove_commands: Vec, + timeout_seconds: u64, +} +impl WorkflowWorkspaceHooks { + /// Commands that run after Decodex creates a new linked worktree for a lane. + pub fn after_create_commands(&self) -> &[String] { + &self.after_create_commands + } + + /// Commands that run before Decodex removes a linked worktree for a lane. + pub fn before_remove_commands(&self) -> &[String] { + &self.before_remove_commands + } + + /// Shared timeout budget, in seconds, for each workspace hook command. + pub fn timeout_seconds(&self) -> u64 { + self.timeout_seconds + } + + fn validate(&self) -> Result<()> { + if self.timeout_seconds == 0 { + eyre::bail!("`execution.workspace_hooks.timeout_seconds` must be greater than zero."); + } + + validate_string_entries( + "execution.workspace_hooks.after_create_commands", + &self.after_create_commands, + )?; + validate_string_entries( + "execution.workspace_hooks.before_remove_commands", + &self.before_remove_commands, + )?; + + Ok(()) + } +} + +/// Narrow, repo-owned gate profile selected from changed tracked files. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowGateProfile { + match_mode: WorkflowGateMatchMode, + paths: Vec, + canonicalize_commands: Vec, + verify_commands: Vec, +} +impl WorkflowGateProfile { + /// Match mode for the profile. + pub fn match_mode(&self) -> WorkflowGateMatchMode { + self.match_mode + } + + /// Repo-relative path patterns covered by this profile. + pub fn paths(&self) -> &[String] { + &self.paths + } + + /// Canonicalize commands for this profile. + pub fn canonicalize_commands(&self) -> &[String] { + &self.canonicalize_commands + } + + /// Verify commands for this profile. + pub fn verify_commands(&self) -> &[String] { + &self.verify_commands + } + + fn validate(&self, profile_name: &str) -> Result<()> { + if self.paths.is_empty() { + eyre::bail!("`execution.gate_profiles.{profile_name}.paths` must not be empty."); + } + if self.canonicalize_commands.is_empty() && self.verify_commands.is_empty() { + eyre::bail!( + "`execution.gate_profiles.{profile_name}` must declare at least one canonicalize or verify command." + ); + } + + validate_repo_relative_paths( + &format!("execution.gate_profiles.{profile_name}.paths"), + &self.paths, + )?; + + self.compile_path_set(profile_name)?; + + validate_string_entries( + &format!("execution.gate_profiles.{profile_name}.canonicalize_commands"), + &self.canonicalize_commands, + )?; + validate_string_entries( + &format!("execution.gate_profiles.{profile_name}.verify_commands"), + &self.verify_commands, + )?; + + Ok(()) + } + + fn matches_changed_files(&self, changed_files: &BTreeSet) -> Result { + let path_set = self.compile_path_set("runtime")?; + + match self.match_mode { + WorkflowGateMatchMode::Only => + Ok(changed_files.iter().all(|path| path_set.is_match(Path::new(path)))), + } + } + + fn compile_path_set(&self, profile_name: &str) -> Result { + let mut builder = GlobSetBuilder::new(); + + for path in &self.paths { + let glob = Glob::new(path).map_err(|error| { + eyre::eyre!( + "Invalid glob pattern in `execution.gate_profiles.{profile_name}.paths`: `{path}` ({error})" + ) + })?; + + builder.add(glob); + } + + builder.build().map_err(|error| { + eyre::eyre!("Failed to compile `execution.gate_profiles.{profile_name}.paths`: {error}") + }) + } +} + +/// A resolved repo gate ready to execute. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ResolvedRepoGate<'a> { + profile_name: Option<&'a str>, + canonicalize_commands: &'a [String], + verify_commands: &'a [String], +} +impl<'a> ResolvedRepoGate<'a> { + /// Optional selected profile name; `None` means the default full gate. + pub fn profile_name(&self) -> Option<&'a str> { + self.profile_name + } + + /// Canonicalize commands selected for this gate run. + pub fn canonicalize_commands(&self) -> &'a [String] { + self.canonicalize_commands + } + + /// Verification commands selected for this gate run. + pub fn verify_commands(&self) -> &'a [String] { + self.verify_commands + } +} + +/// Repo-local early-load context. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkflowContext { + read_first: Vec, +} +impl WorkflowContext { + /// Repository-relative files to load before the broader prompt body. + pub fn read_first(&self) -> &[String] { + &self.read_first + } + + fn validate(&self) -> Result<()> { + validate_repo_relative_paths("context.read_first", &self.read_first) + } +} + +/// Supported tracker providers. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum TrackerProvider { + /// Linear issue tracking. + Linear, +} + +/// Match semantics for a repo-owned gate profile. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkflowGateMatchMode { + /// The profile applies only when every changed tracked file is covered by its path set. + Only, +} + +fn validate_string_entries(field_name: &str, values: &[String]) -> Result<()> { + for value in values { + let trimmed = value.trim(); + + if trimmed.is_empty() { + eyre::bail!("`{field_name}` entries must not be empty."); + } + if trimmed != value { + eyre::bail!("`{field_name}` entries must not include surrounding whitespace."); + } + } + + Ok(()) +} + +fn validate_repo_relative_paths(field_name: &str, values: &[String]) -> Result<()> { + validate_string_entries(field_name, values)?; + + for value in values { + let path = Path::new(value); + + if path.is_absolute() { + eyre::bail!("`{field_name}` entries must be repository-relative paths."); + } + if !path.components().all(|component| matches!(component, Component::Normal(_))) { + eyre::bail!( + "`{field_name}` entries must not contain `.`, `..`, root, or prefix components." + ); + } + } + + Ok(()) +} + +fn split_frontmatter(input: &str) -> Result<(String, String)> { + let input = input.trim_start_matches(['\u{feff}', '\n', '\r']); + let mut lines = input.lines(); + + if lines.next() != Some(FRONTMATTER_DELIMITER) { + eyre::bail!("WORKFLOW.md must begin with TOML frontmatter delimited by `+++`."); + } + + let mut frontmatter_lines = Vec::new(); + let mut body_lines = Vec::new(); + let mut found_end = false; + + for line in lines { + if !found_end && line == FRONTMATTER_DELIMITER { + found_end = true; + + continue; + } + if found_end { + body_lines.push(line); + } else { + frontmatter_lines.push(line); + } + } + + if !found_end { + eyre::bail!("WORKFLOW.md frontmatter is missing the closing `+++` delimiter."); + } + + let body = body_lines.join("\n").trim().to_string(); + + Ok((frontmatter_lines.join("\n"), body)) +} + +fn validate_trimmed_non_empty(field_name: &str, value: &str) -> Result<()> { + if value.trim().is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + if value != value.trim() { + eyre::bail!("`{field_name}` must not include surrounding whitespace."); + } + + Ok(()) +} + +fn validate_non_empty_string_list(field_name: &str, values: &[String]) -> Result<()> { + if values.is_empty() { + eyre::bail!("`{field_name}` must not be empty."); + } + + for value in values { + validate_trimmed_non_empty(&format!("{field_name} entries"), value)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeSet, fs}; + + use Edit::{Remove, Replace}; + use tempfile::NamedTempFile; + + use crate::{ + prelude::Result, + workflow::{TrackerProvider, WorkflowDocument, WorkflowGateMatchMode}, + }; + + enum Edit<'a> { + Remove(&'a str), + Replace(&'a str, &'a str), + } + impl Edit<'_> { + fn apply(&self, markdown: &mut String) { + match self { + Self::Remove(needle) => *markdown = markdown.replace(needle, ""), + Self::Replace(needle, replacement) => { + *markdown = markdown.replace(needle, replacement); + }, + } + } + } + + #[test] + fn parses_workflow_document() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 4 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 2 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] +gate_profiles = {} + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Start with the repo's documented routing entrypoint when one exists. +Use `cargo make`. + "#, + ) + .expect("workflow document should parse"); + + assert_eq!(document.frontmatter().version(), 1); + assert_eq!(document.frontmatter().tracker().provider(), TrackerProvider::Linear); + assert_eq!(document.frontmatter().tracker().completed_state(), "Done"); + assert_eq!(document.frontmatter().execution().max_attempts(), 3); + assert_eq!(document.frontmatter().execution().max_turns(), 4); + assert_eq!(document.frontmatter().execution().max_retry_backoff_ms(), 300_000); + assert_eq!(document.frontmatter().execution().max_concurrent_agents(), 2); + assert_eq!(document.frontmatter().execution().canonicalize_commands(), ["cargo make fmt"]); + assert_eq!(document.frontmatter().execution().verify_commands(), ["cargo make test"]); + assert_eq!( + document.body(), + "Start with the repo's documented routing entrypoint when one exists.\nUse `cargo make`." + ); + } + + #[test] + fn parses_workspace_hooks() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = ["./scripts/bootstrap-worktree.sh"] +before_remove_commands = ["./scripts/cleanup-worktree.sh"] +timeout_seconds = 45 + +[context] +read_first = [] ++++ + "#, + ) + .expect("workflow with workspace hooks should parse"); + let hooks = document.frontmatter().execution().workspace_hooks(); + + assert_eq!(hooks.after_create_commands(), ["./scripts/bootstrap-worktree.sh"]); + assert_eq!(hooks.before_remove_commands(), ["./scripts/cleanup-worktree.sh"]); + assert_eq!(hooks.timeout_seconds(), 45); + } + + #[test] + fn rejects_invalid_workspace_hook_config() { + for (case_name, needle, replacement, expected) in [ + ( + "zero timeout", + "timeout_seconds = 60", + "timeout_seconds = 0", + "`execution.workspace_hooks.timeout_seconds` must be greater than zero", + ), + ( + "surrounding whitespace", + "after_create_commands = []", + r#"after_create_commands = [" ./scripts/bootstrap-worktree.sh "]"#, + "`execution.workspace_hooks.after_create_commands` entries must not include surrounding whitespace", + ), + ] { + let result = parse_valid_workflow_with(|markdown| { + *markdown = markdown.replace(needle, replacement); + }); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn parses_named_gate_profile() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + ) + .expect("workflow with gate profile should parse"); + let profile = document + .frontmatter() + .execution() + .gate_profiles() + .get("config_subset") + .expect("config_subset profile should exist"); + + assert_eq!(profile.match_mode(), WorkflowGateMatchMode::Only); + assert_eq!(profile.paths(), ["config/**"]); + assert_eq!(profile.verify_commands(), ["python3 -c 'print(\"ok\")'"]); + } + + #[test] + fn selects_matching_gate_profile_when_all_changed_files_match_profile_paths() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + ) + .expect("workflow should parse"); + let changed_files = ["config/base.toml", "config/service.toml"] + .into_iter() + .map(str::to_owned) + .collect::>(); + let selection = + document.frontmatter().execution().select_repo_gate_for_changed_files(&changed_files); + + assert_eq!(selection.profile_name(), Some("config_subset")); + assert!(selection.canonicalize_commands().is_empty()); + assert_eq!(selection.verify_commands(), ["python3 -c 'print(\"ok\")'"]); + } + + #[test] + fn falls_back_to_full_gate_for_mixed_docs_and_runtime_changes() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + ) + .expect("workflow should parse"); + let changed_files = ["config/base.toml", "src/orchestrator/git_ops.rs"] + .into_iter() + .map(str::to_owned) + .collect::>(); + let selection = + document.frontmatter().execution().select_repo_gate_for_changed_files(&changed_files); + + assert_eq!(selection.profile_name(), None); + assert_eq!(selection.canonicalize_commands(), ["cargo make fmt"]); + assert_eq!(selection.verify_commands(), ["cargo make test"]); + } + + #[test] + fn falls_back_to_full_gate_for_ambiguous_profile_matches() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.gate_profiles.config_prod] +match_mode = "only" +paths = ["config/prod.toml"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + ) + .expect("workflow should parse"); + let changed_files = + ["config/prod.toml"].into_iter().map(str::to_owned).collect::>(); + let selection = + document.frontmatter().execution().select_repo_gate_for_changed_files(&changed_files); + + assert_eq!(selection.profile_name(), None); + assert_eq!(selection.verify_commands(), ["cargo make test"]); + } + + #[test] + fn rejects_incomplete_gate_profiles() { + for (case_name, paths, commands, expected) in [ + ( + "missing paths", + "[]", + r#"verify_commands = ["python3 -c 'print(\"ok\")'"]"#, + "`execution.gate_profiles.config_subset.paths` must not be empty", + ), + ( + "missing commands", + r#"["config/**"]"#, + "verify_commands = []", + "`execution.gate_profiles.config_subset` must declare at least one canonicalize or verify command", + ), + ] { + let result = parse_valid_workflow_with(|markdown| { + *markdown = markdown.replace( + r#"gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] +"#, + &format!( + r#" +canonicalize_commands = [] +verify_commands = [] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = {paths} +canonicalize_commands = [] +{commands} + +"#, + ), + ); + }); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn rejects_gate_profile_paths_that_escape_repo() { + for (path, expected) in [ + ("../config/**", "must not contain `.`, `..`, root, or prefix components"), + ("/tmp/config/**", "must be repository-relative paths"), + ] { + let result = parse_valid_workflow_with(|markdown| { + *markdown = markdown.replace( + r#"gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] +"#, + &format!( + r#" +canonicalize_commands = [] +verify_commands = [] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["{path}"] +canonicalize_commands = [] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +"# + ), + ); + }); + let error = result.expect_err("escaping gate profile path should fail"); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{path}`: {error:?}" + ); + } + } + + #[test] + fn loads_workflow_document_from_path() { + let file = NamedTempFile::new().expect("temp file should exist"); + + fs::write( + file.path(), + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Read the repo policy first. + "#, + ) + .expect("workflow document should be written"); + + let document = + WorkflowDocument::from_path(file.path()).expect("workflow should load from path"); + + assert_eq!(document.frontmatter().tracker().completed_state(), "Done"); + } + + #[test] + fn parses_explicit_completed_state() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Released", "Canceled"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Released" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Read the repo policy first. + "#, + ) + .expect("workflow document should parse"); + + assert_eq!(document.frontmatter().tracker().completed_state(), "Released"); + assert_eq!(document.frontmatter().tracker().resolved_completed_state(), "Released"); + } + + #[test] + fn rejects_invalid_completed_state_contract() { + for (case_name, edit, expected) in [ + ("missing completed_state", Remove("completed_state = \"Done\"\n"), "completed_state"), + ( + "completed_state outside terminal_states", + Replace( + r#"terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done""#, + r#"terminal_states = ["Released", "Canceled"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done""#, + ), + "`tracker.completed_state` must be one of `tracker.terminal_states`", + ), + ] { + let result = parse_valid_workflow_with(|markdown| edit.apply(markdown)); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn rejects_unknown_workflow_fields() { + for (case_name, edit, field) in [ + ( + "nested tracker field", + Replace( + "needs_attention_label = \"decodex:needs-attention\"", + "needs_attention_label = \"decodex:needs-attention\"\nunexpected_tracker_key = \"pubfi\"", + ), + "unexpected_tracker_key", + ), + ( + "execution field", + Replace( + "max_concurrent_agents = 1", + "max_concurrent_agents = 1\nunexpected_execution_field = [\"cargo make test\"]", + ), + "unexpected_execution_field", + ), + ( + "top-level table", + Replace( + "[context]\nread_first = []", + "[context]\nread_first = []\n\n[unexpected]\nenabled = true", + ), + "unexpected", + ), + ] { + let result = parse_valid_workflow_with(|markdown| edit.apply(markdown)); + let error = result.expect_err(case_name); + + assert!(error.to_string().contains(&format!("unknown field `{field}`"))); + } + } + + #[test] + fn rejects_missing_frontmatter() { + let result = WorkflowDocument::parse_markdown("Read the repo policy first."); + + assert!(result.is_err()); + } + + #[test] + fn rejects_missing_or_empty_required_workflow_contract() { + for (case_name, edit, expected) in [ + ( + "missing agent block", + Remove( + r#"[agent] +transport = "stdio://" + +"#, + ), + "agent", + ), + ("missing max_attempts", Remove("max_attempts = 3\n"), "max_attempts"), + ( + "empty terminal states", + Replace( + r#"terminal_states = ["Done", "Canceled", "Duplicate"]"#, + "terminal_states = []", + ), + "`tracker.terminal_states` must not be empty", + ), + ( + "blank agent transport", + Replace(r#"transport = "stdio://""#, r#"transport = """#), + "`agent.transport` must not be empty", + ), + ] { + let result = parse_valid_workflow_with(|markdown| edit.apply(markdown)); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn rejects_blank_required_tracker_policy_values() { + for ( + field, + in_progress_state, + success_state, + failure_state, + opt_out_label, + needs_attention_label, + ) in [ + ( + "in_progress_state", + "\"\"", + "\"In Review\"", + "\"Todo\"", + "\"decodex:manual-only\"", + "\"decodex:needs-attention\"", + ), + ( + "success_state", + "\"In Progress\"", + "\"\"", + "\"Todo\"", + "\"decodex:manual-only\"", + "\"decodex:needs-attention\"", + ), + ( + "failure_state", + "\"In Progress\"", + "\"In Review\"", + "\"\"", + "\"decodex:manual-only\"", + "\"decodex:needs-attention\"", + ), + ( + "opt_out_label", + "\"In Progress\"", + "\"In Review\"", + "\"Todo\"", + "\"\"", + "\"decodex:needs-attention\"", + ), + ( + "needs_attention_label", + "\"In Progress\"", + "\"In Review\"", + "\"Todo\"", + "\"decodex:manual-only\"", + "\"\"", + ), + ] { + let result = WorkflowDocument::parse_markdown(&format!( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = {in_progress_state} +success_state = {success_state} +completed_state = "Done" +failure_state = {failure_state} +opt_out_label = {opt_out_label} +needs_attention_label = {needs_attention_label} + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {{}} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + )); + + assert!( + result + .expect_err("blank required tracker value should fail") + .to_string() + .contains(&format!("`tracker.{field}` must not be empty")) + ); + } + } + + #[test] + fn rejects_blank_required_policy_entries() { + for (field, startable_states, terminal_states) in [ + ("startable_states", "[\"\"]", "[\"Done\", \"Canceled\", \"Duplicate\"]"), + ("terminal_states", "[\"Todo\"]", "[\"Done\", \"\"]"), + ] { + let result = WorkflowDocument::parse_markdown(&format!( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = {startable_states} +terminal_states = {terminal_states} +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {{}} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + )); + + assert!( + result + .expect_err("blank required tracker entry should fail") + .to_string() + .contains(&format!("`tracker.{field} entries` must not be empty")) + ); + } + } + + #[test] + fn rejects_missing_required_workflow_sections_and_fields() { + for (needle, expected) in [ + ("gate_profiles = {}\n", "gate_profiles"), + ( + r#"[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +"#, + "workspace_hooks", + ), + ( + r#"[context] +read_first = [] +"#, + "context", + ), + ] { + let result = parse_valid_workflow_with(|markdown| { + *markdown = markdown.replace(needle, ""); + }); + let error = result.expect_err("missing required workflow sections should fail"); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{expected}`: {error:?}" + ); + } + } + + #[test] + fn rejects_invalid_gate_command_entries() { + for (case_name, edit, expected) in [ + ( + "blank canonicalize command", + Replace("canonicalize_commands = []", "canonicalize_commands = [\"\"]"), + "`execution.canonicalize_commands` entries", + ), + ( + "untrimmed verify command", + Replace("verify_commands = []", "verify_commands = [\" cargo make test \"]"), + "`execution.verify_commands` entries", + ), + ( + "blank profile canonicalize command", + Replace( + r#"gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] +"#, + r#" +canonicalize_commands = [] +verify_commands = [] + +[execution.gate_profiles.config_subset] +match_mode = "only" +paths = ["config/**"] +canonicalize_commands = [" "] +verify_commands = ["python3 -c 'print(\"ok\")'"] + +"#, + ), + "`execution.gate_profiles.config_subset.canonicalize_commands` entries", + ), + ] { + let result = parse_valid_workflow_with(|markdown| edit.apply(markdown)); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn rejects_invalid_context_read_first_entries() { + for (case_name, replacement, expected) in [ + ( + "blank read_first entry", + "read_first = [\"\"]", + "`context.read_first` entries must not be empty", + ), + ( + "parent traversal read_first path", + "read_first = [\"../secret.md\"]", + "must not contain `.`, `..`, root, or prefix components", + ), + ( + "absolute read_first path", + "read_first = [\"/tmp/secret.md\"]", + "must be repository-relative paths", + ), + ] { + let result = parse_valid_workflow_with(|markdown| { + *markdown = markdown.replace("read_first = []", replacement); + }); + let error = result.expect_err(case_name); + + assert!( + error.to_string().contains(expected), + "unexpected error for `{case_name}`: {error:?}" + ); + } + } + + #[test] + fn workflow_document_markdown_round_trips() { + let document = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 5 +max_turns = 6 +max_retry_backoff_ms = 120000 +max_concurrent_agents = 2 +canonicalize_commands = ["cargo make fmt"] +verify_commands = ["cargo make test"] +gate_profiles = {} + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = ["docs/index.md", "README.md"] ++++ + +Read the repo policy first. +Then validate the lane. + "#, + ) + .expect("workflow document should parse"); + let reparsed = WorkflowDocument::parse_markdown( + &document.to_markdown().expect("workflow markdown should render"), + ) + .expect("rendered workflow should parse"); + + assert_eq!(reparsed, document); + } + + #[test] + fn rejects_zero_global_concurrency_limit() { + let result = WorkflowDocument::parse_markdown( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 0 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + "#, + ); + + assert!(result.is_err(), "zero global concurrency should be invalid"); + } + + fn parse_valid_workflow_with(rewrite: impl FnOnce(&mut String)) -> Result { + let mut markdown = valid_workflow_markdown(); + + rewrite(&mut markdown); + + WorkflowDocument::parse_markdown(&markdown) + } + + fn valid_workflow_markdown() -> String { + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {} +canonicalize_commands = [] +verify_commands = [] + +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = [] +timeout_seconds = 60 + +[context] +read_first = [] ++++ + +Read the repo policy first. + "# + .to_string() + } +} diff --git a/apps/decodex/src/worktree.rs b/apps/decodex/src/worktree.rs new file mode 100644 index 00000000..a798a373 --- /dev/null +++ b/apps/decodex/src/worktree.rs @@ -0,0 +1,2287 @@ +#[cfg(unix)] use std::os::{fd::AsRawFd, unix::process::CommandExt as _}; +use std::{ + env, + ffi::OsStr, + fs, + io::{Error, ErrorKind, Read}, + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, + thread, + time::{Duration, Instant}, +}; + +use libc::{ESRCH, F_GETFL, F_SETFL, O_NONBLOCK, SIGKILL}; + +use crate::{ + prelude::{Result, eyre}, + state::RUN_ACTIVITY_MARKER_FILE, + workflow::WorkflowWorkspaceHooks, +}; + +const AFTER_CREATE_PENDING_MARKER: &str = ".decodex-after-create.pending"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WorktreeSpec { + pub(crate) branch_name: String, + pub(crate) issue_identifier: String, + pub(crate) path: PathBuf, + pub(crate) reused_existing: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct MergedWorktreeCleanupDebt { + pub(crate) branch_name: String, + pub(crate) cleanliness: MergedWorktreeCleanliness, + pub(crate) default_branch: String, + pub(crate) path: PathBuf, +} + +pub(crate) struct WorktreeManager { + repo_root: PathBuf, + worktree_root: PathBuf, + project_id: String, +} +impl WorktreeManager { + pub(crate) fn new( + project_id: impl Into, + repo_root: impl Into, + worktree_root: impl Into, + ) -> Self { + Self { + repo_root: repo_root.into(), + worktree_root: worktree_root.into(), + project_id: project_id.into(), + } + } + + pub(crate) fn plan_for_issue(&self, issue_identifier: &str) -> WorktreeSpec { + let branch_suffix = sanitize_branch_component(issue_identifier); + let branch_owner = + configured_branch_owner(&self.repo_root).unwrap_or_else(|| String::from("x")); + let branch_name = format!( + "{}/{}-{}", + sanitize_branch_component(&branch_owner), + sanitize_branch_component(&self.project_id), + branch_suffix + ); + let path = self.worktree_root.join(issue_identifier); + let reused_existing = path.join(".git").exists(); + + WorktreeSpec { + branch_name, + issue_identifier: issue_identifier.to_owned(), + path, + reused_existing, + } + } + + #[cfg(test)] + pub(crate) fn ensure_worktree( + &self, + issue_identifier: &str, + dry_run: bool, + ) -> Result { + self.ensure_worktree_internal(issue_identifier, dry_run, None) + } + + pub(crate) fn ensure_worktree_with_hooks( + &self, + issue_identifier: &str, + dry_run: bool, + hooks: &WorkflowWorkspaceHooks, + ) -> Result { + self.ensure_worktree_internal(issue_identifier, dry_run, Some(hooks)) + } + + fn ensure_worktree_internal( + &self, + issue_identifier: &str, + dry_run: bool, + hooks: Option<&WorkflowWorkspaceHooks>, + ) -> Result { + let spec = self.plan_for_issue(issue_identifier); + + if dry_run { + return Ok(spec); + } + if spec.reused_existing { + self.validate_worktree_boundary(&spec.path)?; + + normalize_origin_remote_for_worktrees(&self.repo_root)?; + + self.resume_after_create_hooks_if_pending(&spec, hooks)?; + + return Ok(spec); + } + + fs::create_dir_all(&self.worktree_root)?; + + self.create_linked_worktree(&spec, hooks)?; + self.validate_worktree_boundary(&spec.path)?; + self.run_after_create_hooks(&spec, hooks)?; + + Ok(spec) + } + + #[cfg(test)] + pub(crate) fn remove_worktree_path(&self, path: &Path) -> Result { + self.remove_worktree_path_internal(path, None) + } + + pub(crate) fn remove_worktree_path_with_hooks( + &self, + issue_identifier: &str, + branch_name: &str, + path: &Path, + hooks: &WorkflowWorkspaceHooks, + ) -> Result { + self.remove_worktree_path_internal(path, Some((issue_identifier, branch_name, hooks))) + } + + fn remove_worktree_path_internal( + &self, + path: &Path, + hooks: Option<(&str, &str, &WorkflowWorkspaceHooks)>, + ) -> Result { + if !path.exists() { + return Ok(false); + } + + let worktree_root = fs::canonicalize(&self.worktree_root)?; + let canonical_path = fs::canonicalize(path)?; + + if !canonical_path.starts_with(&worktree_root) || canonical_path == worktree_root { + eyre::bail!( + "Refusing to remove worktree `{}` outside worktree_root `{}`.", + path.display(), + self.worktree_root.display() + ); + } + if remove_orphan_marker_directory_if_safe(&canonical_path)? { + return Ok(true); + } + + self.validate_worktree_boundary(&canonical_path)?; + + if let Some((issue_identifier, branch_name, hooks)) = hooks { + self.run_workspace_hook_phase( + "before_remove", + issue_identifier, + branch_name, + &canonical_path, + hooks.before_remove_commands(), + hooks.timeout_seconds(), + )?; + } + + run_git( + &self.repo_root, + [ + "worktree", + "remove", + "--force", + canonical_path.as_os_str().to_str().ok_or_else(|| { + eyre::eyre!("Worktree path `{}` is not valid UTF-8.", canonical_path.display()) + })?, + ], + "remove the linked worktree", + )?; + + Ok(true) + } + + fn create_linked_worktree( + &self, + spec: &WorktreeSpec, + hooks: Option<&WorkflowWorkspaceHooks>, + ) -> Result<()> { + let source_head = + git_stdout(&self.repo_root, ["rev-parse", "HEAD"], "read the source repository HEAD")?; + + if spec.path.exists() { + eyre::bail!( + "Worktree path `{}` already exists but does not look reusable.", + spec.path.display() + ); + } + + let create_output = Command::new("git") + .arg("-C") + .arg(&self.repo_root) + .args(["worktree", "add", "--quiet", "--detach"]) + .arg(&spec.path) + .arg(&source_head) + .output()?; + + if !create_output.status.success() { + let stderr = String::from_utf8_lossy(&create_output.stderr); + + eyre::bail!( + "Failed to create linked worktree `{}` from `{}`: {}", + spec.path.display(), + self.repo_root.display(), + stderr.trim() + ); + } + + let setup_result = normalize_origin_remote_for_worktrees(&self.repo_root).and_then(|_| { + self.checkout_worktree_branch(&spec.path, spec.branch_name.as_str(), &source_head) + }); + + if let Err(error) = setup_result { + let _ = self.remove_worktree_path_internal(&spec.path, None); + + return Err(error); + } + + if workspace_requires_after_create_pending_marker(hooks) { + let pending_marker = after_create_pending_marker_path(&spec.path); + + fs::write(&pending_marker, b"pending\n").map_err(|error| { + let _ = self.remove_worktree_path_internal(&spec.path, None); + + eyre::eyre!( + "Failed to write pending after-create marker `{}`: {error}", + pending_marker.display() + ) + })?; + } + + Ok(()) + } + + fn checkout_worktree_branch( + &self, + worktree_path: &Path, + branch_name: &str, + source_head: &str, + ) -> Result<()> { + if fetch_remote_branch_if_present(&self.repo_root, branch_name)? { + let remote_tracking_ref = format!("refs/remotes/origin/{branch_name}"); + + run_git( + worktree_path, + ["checkout", "--quiet", "-B", branch_name, remote_tracking_ref.as_str()], + "checkout the worktree branch from the remote lane head", + )?; + } else { + run_git( + worktree_path, + ["checkout", "--quiet", "-B", branch_name, source_head], + "checkout the worktree branch", + )?; + } + + Ok(()) + } + + fn validate_worktree_boundary(&self, worktree_path: &Path) -> Result<()> { + let git_pointer = worktree_path.join(".git"); + + if !git_pointer.is_file() { + eyre::bail!( + "Worktree `{}` is not a linked git worktree: expected `.git` to be a pointer file.", + worktree_path.display() + ); + } + + let repo_git_dir = resolve_source_repo_git_common_dir(&self.repo_root)?; + let worktree_admin_root = repo_git_dir.join("worktrees"); + let canonical_worktree_path = fs::canonicalize(worktree_path)?; + let git_dir = fs::canonicalize(PathBuf::from(git_stdout( + worktree_path, + ["rev-parse", "--path-format=absolute", "--git-dir"], + "resolve worktree git dir", + )?))?; + let git_common_dir = fs::canonicalize(PathBuf::from(git_stdout( + worktree_path, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "resolve worktree git common dir", + )?))?; + + if !git_dir.starts_with(&worktree_admin_root) { + eyre::bail!( + "Worktree `{}` is not linked through `{}`: git dir resolved to `{}`.", + worktree_path.display(), + worktree_admin_root.display(), + git_dir.display() + ); + } + if git_common_dir != repo_git_dir { + eyre::bail!( + "Worktree `{}` must share git common dir `{}`, found `{}`.", + worktree_path.display(), + repo_git_dir.display(), + git_common_dir.display() + ); + } + if !worktree_is_registered(&self.repo_root, &canonical_worktree_path)? { + eyre::bail!( + "Worktree `{}` is not registered with the source repository worktree admin.", + worktree_path.display() + ); + } + + Ok(()) + } + + fn run_workspace_hook_phase( + &self, + phase_name: &str, + issue_identifier: &str, + branch_name: &str, + worktree_path: &Path, + commands: &[String], + timeout_seconds: u64, + ) -> Result<()> { + if commands.is_empty() { + return Ok(()); + } + + let envs = [ + ("DECODEX_REPO_ROOT", self.repo_root.display().to_string()), + ("DECODEX_WORKTREE_PATH", worktree_path.display().to_string()), + ("DECODEX_ISSUE_ID", issue_identifier.to_owned()), + ("DECODEX_BRANCH", branch_name.to_owned()), + ]; + + for command in commands { + let output = run_workspace_hook_shell_command( + command, + worktree_path, + &envs, + Duration::from_secs(timeout_seconds), + ) + .map_err(|error| { + eyre::eyre!( + "Failed to run workspace hook `{phase_name}` command `{command}` in `{}`: {error}", + worktree_path.display() + ) + })?; + + if !output.status.success() { + let mut details = String::new(); + + append_output_details(&mut details, &output); + + eyre::bail!( + "Workspace hook `{phase_name}` command `{command}` failed in `{}` with status `{}`.{details}", + worktree_path.display(), + output.status + ); + } + } + + Ok(()) + } + + fn run_after_create_hooks( + &self, + spec: &WorktreeSpec, + hooks: Option<&WorkflowWorkspaceHooks>, + ) -> Result<()> { + let Some(hooks) = hooks else { + return Ok(()); + }; + + if hooks.after_create_commands().is_empty() { + return Ok(()); + } + + let pending_marker = after_create_pending_marker_path(&spec.path); + + if let Err(error) = self.run_workspace_hook_phase( + "after_create", + spec.issue_identifier.as_str(), + spec.branch_name.as_str(), + &spec.path, + hooks.after_create_commands(), + hooks.timeout_seconds(), + ) { + fs::write(&pending_marker, b"pending\n").map_err(|marker_error| { + eyre::eyre!( + "Workspace after-create hook failed and pending marker `{}` could not be restored: {marker_error}. Original error: {error}", + pending_marker.display() + ) + })?; + + return Err(error); + } + if let Err(error) = fs::remove_file(&pending_marker) + && error.kind() != ErrorKind::NotFound + { + return Err(eyre::eyre!( + "Failed to clear pending after-create marker `{}` after successful bootstrap: {error}", + pending_marker.display() + )); + } + + Ok(()) + } + + fn resume_after_create_hooks_if_pending( + &self, + spec: &WorktreeSpec, + hooks: Option<&WorkflowWorkspaceHooks>, + ) -> Result<()> { + let Some(hooks) = hooks else { + return Ok(()); + }; + + if hooks.after_create_commands().is_empty() { + return Ok(()); + } + if !after_create_pending_marker_path(&spec.path).exists() { + return Ok(()); + } + + self.run_after_create_hooks(spec, Some(hooks)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct LinkedWorktree { + branch_name: String, + path: PathBuf, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MergedWorktreeCleanliness { + Clean, + Dirty, +} +impl MergedWorktreeCleanliness { + pub(crate) fn is_dirty(self) -> bool { + self == Self::Dirty + } +} + +pub(crate) fn infer_default_branch_name(repo_root: &Path) -> Result> { + if let Some(remote_head) = symbolic_ref(repo_root, "refs/remotes/origin/HEAD")? + && let Some(branch_name) = remote_head.strip_prefix("origin/") + && !branch_name.is_empty() + { + return Ok(Some(branch_name.to_owned())); + } + + current_branch_name(repo_root) +} + +pub(crate) fn merged_worktree_cleanup_debts( + repo_root: &Path, + worktree_root: &Path, + default_branch: &str, +) -> Result> { + if default_branch.is_empty() || !worktree_root.exists() { + return Ok(Vec::new()); + } + + let mut debts = Vec::new(); + + for worktree in linked_worktrees(repo_root)? { + if worktree.branch_name == default_branch + || linked_worktree_under_root(&worktree.path, worktree_root)?.is_none() + || branch_merged_into_default(repo_root, &worktree.branch_name, default_branch)? + .is_none() + { + continue; + } + + debts.push(MergedWorktreeCleanupDebt { + branch_name: worktree.branch_name, + cleanliness: worktree_cleanliness(&worktree.path)?, + default_branch: default_branch.to_owned(), + path: worktree.path, + }); + } + + debts.sort_by(|left, right| { + left.path.cmp(&right.path).then_with(|| left.branch_name.cmp(&right.branch_name)) + }); + + Ok(debts) +} + +fn linked_worktrees(repo_root: &Path) -> Result> { + Ok(parse_linked_worktrees(&git_stdout( + repo_root, + ["worktree", "list", "--porcelain"], + "list linked worktrees", + )?)) +} + +fn parse_linked_worktrees(output: &str) -> Vec { + let mut entries = Vec::new(); + let mut current_path: Option = None; + let mut current_branch: Option = None; + + for line in output.lines() { + if line.is_empty() { + push_linked_worktree_entry(&mut entries, &mut current_path, &mut current_branch); + + continue; + } + + if let Some(path) = line.strip_prefix("worktree ") { + push_linked_worktree_entry(&mut entries, &mut current_path, &mut current_branch); + + current_path = Some(PathBuf::from(path)); + + continue; + } + if let Some(branch_ref) = line.strip_prefix("branch refs/heads/") { + current_branch = Some(branch_ref.to_owned()); + } + } + + push_linked_worktree_entry(&mut entries, &mut current_path, &mut current_branch); + + entries +} + +fn push_linked_worktree_entry( + entries: &mut Vec, + path: &mut Option, + branch_name: &mut Option, +) { + if let (Some(path), Some(branch_name)) = (path.take(), branch_name.take()) { + entries.push(LinkedWorktree { branch_name, path }); + } + + *path = None; + *branch_name = None; +} + +fn linked_worktree_under_root(path: &Path, worktree_root: &Path) -> Result> { + if !path.exists() || !worktree_root.exists() { + return Ok(None); + } + + let canonical_path = fs::canonicalize(path)?; + let canonical_root = fs::canonicalize(worktree_root)?; + + if canonical_path.starts_with(&canonical_root) && canonical_path != canonical_root { + return Ok(Some(())); + } + + Ok(None) +} + +fn branch_merged_into_default( + repo_root: &Path, + branch_name: &str, + default_branch: &str, +) -> Result> { + let branch_ref = format!("refs/heads/{branch_name}"); + let default_ref = format!("refs/heads/{default_branch}"); + + if !git_ref_exists(repo_root, &branch_ref)? || !git_ref_exists(repo_root, &default_ref)? { + return Ok(None); + } + if git_refs_point_to_same_tip(repo_root, &branch_ref, &default_ref)? { + return Ok(None); + } + if branch_tip_is_on_default_first_parent(repo_root, &branch_ref, &default_ref)? { + return Ok(None); + } + + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["merge-base", "--is-ancestor", branch_ref.as_str(), default_ref.as_str()]) + .output()?; + + if output.status.success() { + return Ok(Some(())); + } + if output.status.code() == Some(1) { + return Ok(None); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to determine whether worktree branch `{branch_name}` is merged into `{default_branch}` in `{}`: {}", + repo_root.display(), + stderr.trim() + ); +} + +fn branch_tip_is_on_default_first_parent( + repo_root: &Path, + branch_ref: &str, + default_ref: &str, +) -> Result { + let branch_tip = git_stdout(repo_root, ["rev-parse", branch_ref], "resolve branch tip")?; + let first_parent_history = git_stdout( + repo_root, + ["rev-list", "--first-parent", default_ref], + "list default branch first-parent history", + )?; + + Ok(first_parent_history.lines().any(|commit| commit == branch_tip)) +} + +fn git_refs_point_to_same_tip(repo_root: &Path, left_ref: &str, right_ref: &str) -> Result { + let left_tip = git_stdout(repo_root, ["rev-parse", left_ref], "resolve git ref tip")?; + let right_tip = git_stdout(repo_root, ["rev-parse", right_ref], "resolve git ref tip")?; + + Ok(left_tip == right_tip) +} + +fn git_ref_exists(repo_root: &Path, ref_name: &str) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["show-ref", "--verify", "--quiet", ref_name]) + .output()?; + + if output.status.success() { + return Ok(true); + } + if output.status.code() == Some(1) { + return Ok(false); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect git ref `{ref_name}` in `{}`: {}", + repo_root.display(), + stderr.trim() + ); +} + +fn worktree_cleanliness(worktree_path: &Path) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(["status", "--porcelain"]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect worktree cleanliness in `{}`: {}", + worktree_path.display(), + stderr.trim() + ); + } + if String::from_utf8_lossy(&output.stdout).trim().is_empty() { + return Ok(MergedWorktreeCleanliness::Clean); + } + + Ok(MergedWorktreeCleanliness::Dirty) +} + +fn symbolic_ref(repo_root: &Path, ref_name: &str) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["symbolic-ref", "--quiet", "--short", ref_name]) + .output()?; + + if !output.status.success() { + return Ok(None); + } + + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + + Ok((!value.is_empty()).then_some(value)) +} + +fn current_branch_name(repo_root: &Path) -> Result> { + let output = + Command::new("git").arg("-C").arg(repo_root).args(["branch", "--show-current"]).output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to inspect current branch in `{}`: {}", + repo_root.display(), + stderr.trim() + ); + } + + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + + Ok((!value.is_empty()).then_some(value)) +} + +fn configured_branch_owner(repo_root: &Path) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["config", "--get", "codex.github-identity"]) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + + (!value.is_empty()).then_some(value) +} + +fn worktree_is_registered(repo_root: &Path, expected_path: &Path) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["worktree", "list", "--porcelain"]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to list linked worktrees in `{}`: {}", + repo_root.display(), + stderr.trim() + ); + } + + for line in String::from_utf8_lossy(&output.stdout).lines() { + let Some(path) = line.strip_prefix("worktree ") else { + continue; + }; + let candidate = fs::canonicalize(path).unwrap_or_else(|_| PathBuf::from(path)); + + if candidate == expected_path { + return Ok(true); + } + } + + Ok(false) +} + +fn resolve_source_repo_git_common_dir(repo_root: &Path) -> Result { + Ok(fs::canonicalize(PathBuf::from(git_stdout( + repo_root, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "resolve source repository git common dir", + )?))?) +} + +fn after_create_pending_marker_path(worktree_path: &Path) -> PathBuf { + worktree_path.join(AFTER_CREATE_PENDING_MARKER) +} + +fn workspace_requires_after_create_pending_marker(hooks: Option<&WorkflowWorkspaceHooks>) -> bool { + hooks.is_some_and(|hooks| !hooks.after_create_commands().is_empty()) +} + +fn remove_orphan_marker_directory_if_safe(path: &Path) -> Result { + if path.join(".git").exists() || !path.is_dir() { + return Ok(false); + } + + let mut has_marker = false; + + for entry in fs::read_dir(path)? { + let entry = entry?; + let file_type = entry.file_type()?; + let file_name = entry.file_name(); + let Some(file_name) = file_name.to_str() else { + return Ok(false); + }; + + if !file_type.is_file() { + return Ok(false); + } + + match file_name { + RUN_ACTIVITY_MARKER_FILE | AFTER_CREATE_PENDING_MARKER => has_marker = true, + _ => return Ok(false), + } + } + + if !has_marker { + return Ok(false); + } + + fs::remove_dir_all(path)?; + + Ok(true) +} + +fn workspace_hook_shell_from_env( + shell: Option, +) -> (std::ffi::OsString, &'static str) { + if let Some(shell) = shell + && !shell.is_empty() + { + let shell_path = Path::new(&shell); + let shell_name = shell_path.file_name().and_then(OsStr::to_str); + + if shell_name == Some("sh") { + return (std::ffi::OsString::from("/bin/sh"), "-c"); + } + if !shell_path.is_absolute() || shell_path.is_file() { + return (shell, "-lc"); + } + } + + (std::ffi::OsString::from("/bin/sh"), "-c") +} + +#[cfg(unix)] +fn workspace_hook_shell() -> (std::ffi::OsString, &'static str) { + workspace_hook_shell_from_env(env::var_os("SHELL")) +} + +#[cfg(unix)] +fn run_workspace_hook_shell_command( + command: &str, + cwd: &Path, + envs: &[(&str, String)], + timeout: Duration, +) -> Result { + let (shell, shell_flag) = workspace_hook_shell(); + let deadline = Instant::now() + timeout; + let mut shell_command = Command::new(&shell); + + shell_command + .arg(shell_flag) + .arg(command) + .current_dir(cwd) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .envs(envs.iter().map(|(key, value)| (*key, value.as_str()))); + unsafe { + shell_command.pre_exec(|| { + if libc::setpgid(0, 0) == -1 { + return Err(Error::last_os_error()); + } + + Ok(()) + }); + } + + let mut child = shell_command.spawn().map_err(|error| { + eyre::eyre!( + "Failed to spawn workspace hook shell command `{command}` in `{}` via `{}` `{}`: {error}", + cwd.display(), + shell.to_string_lossy(), + shell_flag + ) + })?; + let stdout_reader = child.stdout.take().ok_or_else(|| { + eyre::eyre!( + "Failed to capture stdout for workspace hook shell command `{command}` in `{}`.", + cwd.display() + ) + })?; + let stderr_reader = child.stderr.take().ok_or_else(|| { + eyre::eyre!( + "Failed to capture stderr for workspace hook shell command `{command}` in `{}`.", + cwd.display() + ) + })?; + let mut stdout_reader = stdout_reader; + let mut stderr_reader = stderr_reader; + + configure_nonblocking_pipe(&stdout_reader, "stdout")?; + configure_nonblocking_pipe(&stderr_reader, "stderr")?; + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + + loop { + drain_pipe_nonblocking(&mut stdout_reader, &mut stdout, "stdout")?; + drain_pipe_nonblocking(&mut stderr_reader, &mut stderr, "stderr")?; + + if let Some(status) = child.try_wait()? { + drain_pipe_nonblocking(&mut stdout_reader, &mut stdout, "stdout")?; + drain_pipe_nonblocking(&mut stderr_reader, &mut stderr, "stderr")?; + + return Ok(Output { status, stdout, stderr }); + } + + if Instant::now() >= deadline { + let process_group_cleanup = kill_workspace_hook_process_group(child.id()); + let _ = child.kill(); + let status = child.wait()?; + + drain_pipe_nonblocking(&mut stdout_reader, &mut stdout, "stdout")?; + drain_pipe_nonblocking(&mut stderr_reader, &mut stderr, "stderr")?; + + let output = Output { status, stdout, stderr }; + let mut details = String::new(); + + append_output_details(&mut details, &output); + append_process_group_cleanup_details(&mut details, process_group_cleanup); + + eyre::bail!( + "Workspace hook shell command `{command}` in `{}` exceeded the {}s timeout.{details}", + cwd.display(), + timeout.as_secs() + ); + } + + thread::sleep(Duration::from_millis(25)); + } +} + +#[cfg(unix)] +fn configure_nonblocking_pipe(reader: &R, stream_name: &str) -> Result<()> +where + R: AsRawFd, +{ + let fd = reader.as_raw_fd(); + let flags = unsafe { libc::fcntl(fd, F_GETFL) }; + + if flags == -1 { + return Err(eyre::eyre!( + "Failed to read workspace hook {stream_name} flags: {}", + std::io::Error::last_os_error() + )); + } + if flags & O_NONBLOCK != 0 { + return Ok(()); + } + + let result = unsafe { libc::fcntl(fd, F_SETFL, flags | O_NONBLOCK) }; + + if result == -1 { + return Err(eyre::eyre!( + "Failed to set workspace hook {stream_name} pipe nonblocking: {}", + std::io::Error::last_os_error() + )); + } + + Ok(()) +} + +#[cfg(unix)] +fn kill_workspace_hook_process_group(process_id: u32) -> Result<()> { + let process_group_id = i32::try_from(process_id).map_err(|error| { + eyre::eyre!("Workspace hook process id `{process_id}` is out of range: {error}") + })?; + let result = unsafe { libc::killpg(process_group_id, SIGKILL) }; + + if result == -1 { + let error = Error::last_os_error(); + + if error.raw_os_error() == Some(ESRCH) { + return Ok(()); + } + + return Err(eyre::eyre!( + "Failed to terminate workspace hook process group `{process_group_id}`: {error}" + )); + } + + Ok(()) +} + +#[cfg(unix)] +fn drain_pipe_nonblocking(reader: &mut R, buffer: &mut Vec, stream_name: &str) -> Result<()> +where + R: Read, +{ + loop { + let mut chunk = [0_u8; 8 * 1_024]; + + match reader.read(&mut chunk) { + Ok(0) => return Ok(()), + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + Err(error) if error.kind() == ErrorKind::WouldBlock => return Ok(()), + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => { + return Err(eyre::eyre!("Failed to read workspace hook {stream_name}: {error}")); + }, + } + } +} + +fn append_output_details(buffer: &mut String, output: &Output) { + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + + if !stdout.is_empty() { + buffer.push_str(&format!(" stdout: `{stdout}`.")); + } + if !stderr.is_empty() { + buffer.push_str(&format!(" stderr: `{stderr}`.")); + } +} + +fn append_process_group_cleanup_details(buffer: &mut String, cleanup_result: Result<()>) { + if let Err(error) = cleanup_result { + buffer.push_str(&format!(" process-group cleanup error: `{error}`.")); + } +} + +fn git_stdout(repo_root: &Path, args: I, action: &str) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let output = Command::new("git").arg("-C").arg(repo_root).args(args).output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to {action} in `{}`: {}", repo_root.display(), stderr.trim()); + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn try_git_stdout(repo_root: &Path, args: I, action: &str) -> Result> +where + I: IntoIterator, + S: AsRef, +{ + let output = Command::new("git").arg("-C").arg(repo_root).args(args).output()?; + + if output.status.success() { + return Ok(Some(String::from_utf8_lossy(&output.stdout).trim().to_owned())); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + if stderr.contains("No such remote") { + return Ok(None); + } + + eyre::bail!("Failed to {action} in `{}`: {}", repo_root.display(), stderr.trim()); +} + +fn run_git(repo_root: &Path, args: I, action: &str) -> Result<()> +where + I: IntoIterator, + S: AsRef, +{ + let output = Command::new("git").arg("-C").arg(repo_root).args(args).output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!("Failed to {action} in `{}`: {}", repo_root.display(), stderr.trim()); + } + + Ok(()) +} + +fn normalize_origin_remote_for_worktrees(repo_root: &Path) -> Result<()> { + let Some(origin_url) = try_git_stdout( + repo_root, + ["remote", "get-url", "origin"], + "read source repository origin remote", + )? + else { + return Ok(()); + }; + + if !is_relative_filesystem_remote(origin_url.as_str()) { + return Ok(()); + } + + let absolute_origin = fs::canonicalize(repo_root.join(&origin_url))?; + let absolute_origin = absolute_origin.to_str().ok_or_else(|| { + eyre::eyre!( + "Resolved absolute origin path `{}` is not valid UTF-8.", + absolute_origin.display() + ) + })?; + + run_git( + repo_root, + ["remote", "set-url", "origin", absolute_origin], + "normalize the source repository origin remote for linked worktrees", + ) +} + +fn is_relative_filesystem_remote(remote_url: &str) -> bool { + if remote_url.starts_with("./") || remote_url.starts_with("../") { + return true; + } + if remote_url == "~" || remote_url.starts_with("~/") { + return false; + } + + !remote_url.contains("://") && !remote_url.contains(':') && !Path::new(remote_url).is_absolute() +} + +fn configure_noninteractive_git(command: &mut Command) -> &mut Command { + command.env("GIT_TERMINAL_PROMPT", "0").env("GCM_INTERACTIVE", "never") +} + +fn fetch_remote_branch_if_present(repo_root: &Path, branch_name: &str) -> Result { + if try_git_stdout( + repo_root, + ["remote", "get-url", "origin"], + "read source repository origin remote", + )? + .is_none() + { + return Ok(false); + } + + let remote_ref = format!("refs/heads/{branch_name}"); + let mut branch_check = Command::new("git"); + + configure_noninteractive_git(&mut branch_check); + + let branch_check = branch_check + .arg("-C") + .arg(repo_root) + .args(["ls-remote", "--exit-code", "--heads", "origin", remote_ref.as_str()]) + .output()?; + + if !branch_check.status.success() { + if branch_check.status.code() == Some(2) { + return Ok(false); + } + + let stderr = String::from_utf8_lossy(&branch_check.stderr); + + eyre::bail!( + "Failed to inspect remote worktree branch `{branch_name}` in `{}`: {}", + repo_root.display(), + stderr.trim() + ); + } + + let remote_tracking_ref = format!("refs/remotes/origin/{branch_name}"); + let mut fetch = Command::new("git"); + + configure_noninteractive_git(&mut fetch); + + let output = fetch + .arg("-C") + .arg(repo_root) + .args([ + "fetch", + "--quiet", + "--no-tags", + "origin", + &format!("refs/heads/{branch_name}:{remote_tracking_ref}"), + ]) + .output()?; + + if output.status.success() { + return Ok(true); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + + eyre::bail!( + "Failed to fetch remote worktree branch `{branch_name}` in `{}`: {}", + repo_root.display(), + stderr.trim() + ); +} + +fn sanitize_branch_component(value: &str) -> String { + value + .chars() + .map(|ch| match ch { + 'A'..='Z' => ch.to_ascii_lowercase(), + 'a'..='z' | '0'..='9' => ch, + '-' | '_' => '-', + _ => '-', + }) + .collect::() + .trim_matches('-') + .to_owned() +} + +#[cfg(test)] +mod tests { + use std::{ + env, fs, + path::{Path, PathBuf}, + process::Command, + thread, + time::{Duration, Instant}, + }; + + use tempfile::TempDir; + + use crate::{workflow::WorkflowDocument, worktree::WorktreeManager}; + + fn workspace_hooks( + workspace_hooks_frontmatter: &str, + ) -> crate::workflow::WorkflowWorkspaceHooks { + let markdown = format!( + r#" ++++ +version = 1 + +[tracker] +provider = "linear" +startable_states = ["Todo"] +terminal_states = ["Done", "Canceled", "Duplicate"] +in_progress_state = "In Progress" +success_state = "In Review" +completed_state = "Done" +failure_state = "Todo" +opt_out_label = "decodex:manual-only" +needs_attention_label = "decodex:needs-attention" + +[agent] +transport = "stdio://" + +[execution] +max_attempts = 3 +max_turns = 1 +max_retry_backoff_ms = 300000 +max_concurrent_agents = 1 +gate_profiles = {{}} +canonicalize_commands = [] +verify_commands = [] + +{workspace_hooks_frontmatter} + +[context] +read_first = [] ++++ + "#, + ); + + WorkflowDocument::parse_markdown(&markdown) + .expect("workflow should parse") + .frontmatter() + .execution() + .workspace_hooks() + .clone() + } + + fn test_git_command() -> Command { + let mut command = Command::new("git"); + + clear_injected_git_config(&mut command); + + command + } + + fn clear_injected_git_config(command: &mut Command) { + let config_count = env::var("GIT_CONFIG_COUNT") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + + command.env_remove("GIT_CONFIG_COUNT"); + command.env_remove("GIT_CONFIG_PARAMETERS"); + + for index in 0..config_count { + command.env_remove(format!("GIT_CONFIG_KEY_{index}")); + command.env_remove(format!("GIT_CONFIG_VALUE_{index}")); + } + } + + fn run_git(repo_root: &Path, args: &[&str]) { + let output = test_git_command() + .args(["-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"]) + .arg("-C") + .arg(repo_root) + .args(args) + .output() + .expect("git command should run"); + + assert!( + output.status.success(), + "git {:?} failed in {}: {}", + args, + repo_root.display(), + String::from_utf8_lossy(&output.stderr) + ); + } + + fn git_stdout(repo_root: &Path, args: &[&str]) -> String { + let output = test_git_command() + .arg("-C") + .arg(repo_root) + .args(args) + .output() + .expect("git command should run"); + + assert!( + output.status.success(), + "git {:?} failed in {}: {}", + args, + repo_root.display(), + String::from_utf8_lossy(&output.stderr) + ); + + String::from_utf8_lossy(&output.stdout).trim().to_owned() + } + + fn init_repo() -> (TempDir, PathBuf) { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let repo_root = temp_dir.path().join("repo"); + let default_origin = repo_root.parent().unwrap().join("source-origin.git"); + + fs::create_dir_all(&repo_root).expect("repo root should exist"); + + run_git( + default_origin.parent().unwrap(), + &["init", "--bare", default_origin.to_str().unwrap()], + ); + run_git(&repo_root, &["init", "--initial-branch", "main"]); + run_git(&repo_root, &["config", "user.name", "Decodex Tests"]); + run_git(&repo_root, &["config", "user.email", "decodex-tests@example.com"]); + run_git(&repo_root, &["config", "commit.gpgsign", "false"]); + run_git(&repo_root, &["config", "tag.gpgsign", "false"]); + run_git(&repo_root, &["remote", "add", "origin", default_origin.to_str().unwrap()]); + + fs::write(repo_root.join("README.md"), "hello\n").expect("seed file should write"); + + run_git(&repo_root, &["add", "README.md"]); + run_git(&repo_root, &["commit", "-m", "seed"]); + + (temp_dir, repo_root) + } + + #[test] + fn merged_worktree_cleanup_debts_detects_dirty_merged_worktree() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let worktree_path = worktree_root.join("accounts-column-format"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + run_git( + &repo_root, + &[ + "worktree", + "add", + "-b", + "xy/accounts-column-format", + worktree_path.to_str().expect("worktree path should be UTF-8"), + "main", + ], + ); + + fs::write(worktree_path.join("README.md"), "feature work\n") + .expect("worktree file should write"); + + run_git(&worktree_path, &["add", "README.md"]); + run_git(&worktree_path, &["commit", "-m", "feature work"]); + run_git( + &repo_root, + &["merge", "--no-ff", "xy/accounts-column-format", "-m", "land feature"], + ); + + fs::write(worktree_path.join("README.md"), "dirty after land\n") + .expect("worktree file should become dirty"); + + let debts = super::merged_worktree_cleanup_debts(&repo_root, &worktree_root, "main") + .expect("cleanup debt scan should succeed"); + + assert_eq!(debts.len(), 1); + assert_eq!(debts[0].branch_name, "xy/accounts-column-format"); + assert_eq!( + fs::canonicalize(&debts[0].path).expect("debt path should canonicalize"), + fs::canonicalize(&worktree_path).expect("worktree path should canonicalize") + ); + assert_eq!(debts[0].cleanliness, super::MergedWorktreeCleanliness::Dirty); + } + + #[test] + fn merged_worktree_cleanup_debts_ignores_dirty_worktree_started_from_old_default() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let worktree_path = worktree_root.join("scroll-capture-motion"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + run_git( + &repo_root, + &[ + "worktree", + "add", + "-b", + "xy/scroll-capture-motion", + worktree_path.to_str().expect("worktree path should be UTF-8"), + "main", + ], + ); + + fs::write(repo_root.join("main.txt"), "main advanced\n") + .expect("main branch file should write"); + + run_git(&repo_root, &["add", "main.txt"]); + run_git(&repo_root, &["commit", "-m", "advance main"]); + + fs::write(worktree_path.join("README.md"), "manual dirty work\n") + .expect("worktree file should become dirty"); + + let debts = super::merged_worktree_cleanup_debts(&repo_root, &worktree_root, "main") + .expect("cleanup debt scan should succeed"); + + assert!( + debts.is_empty(), + "dirty worktrees started from an older default commit are manual work, not post-land debt" + ); + } + + #[test] + fn merged_worktree_cleanup_debts_ignores_unmerged_worktree() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let worktree_path = worktree_root.join("dashboard-ws-control-plane"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + run_git( + &repo_root, + &[ + "worktree", + "add", + "-b", + "xy/dashboard-ws-control-plane", + worktree_path.to_str().expect("worktree path should be UTF-8"), + "main", + ], + ); + + fs::write(worktree_path.join("README.md"), "feature work\n") + .expect("worktree file should write"); + + run_git(&worktree_path, &["add", "README.md"]); + run_git(&worktree_path, &["commit", "-m", "feature work"]); + + let debts = super::merged_worktree_cleanup_debts(&repo_root, &worktree_root, "main") + .expect("cleanup debt scan should succeed"); + + assert!(debts.is_empty(), "unmerged branch worktrees should remain usable"); + } + + #[test] + fn merged_worktree_cleanup_debts_ignores_dirty_worktree_at_default_tip() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let worktree_path = worktree_root.join("XY-454"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + run_git( + &repo_root, + &[ + "worktree", + "add", + "-b", + "y/decodex-xy-454", + worktree_path.to_str().expect("worktree path should be UTF-8"), + "main", + ], + ); + + fs::write(worktree_path.join(crate::state::RUN_ACTIVITY_MARKER_FILE), "started\n") + .expect("run activity marker should write"); + + let debts = super::merged_worktree_cleanup_debts(&repo_root, &worktree_root, "main") + .expect("cleanup debt scan should succeed"); + + assert!(debts.is_empty(), "default-tip run worktrees should remain usable"); + } + + #[test] + fn plans_worktree_paths_and_identity_scoped_branch_names() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let default_spec = manager.plan_for_issue("PUB-101"); + + assert_eq!(default_spec.branch_name, "x/pubfi-pub-101"); + assert_eq!(default_spec.path, worktree_root.join("PUB-101")); + assert!(!default_spec.reused_existing); + + run_git(&repo_root, &["config", "codex.github-identity", "y"]); + + let routed_spec = manager.plan_for_issue("PUB-101"); + + assert_eq!(routed_spec.branch_name, "y/pubfi-pub-101"); + } + + #[test] + fn workspace_hook_shell_uses_posix_sh_for_sh_or_missing_shell() { + for shell_env in [Some(std::ffi::OsString::from("/bin/sh")), None] { + let (shell, shell_flag) = super::workspace_hook_shell_from_env(shell_env); + + assert_eq!(shell, std::ffi::OsString::from("/bin/sh")); + assert_eq!(shell_flag, "-c"); + } + } + + #[test] + fn creates_linked_worktree() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + assert_eq!(spec.branch_name, "x/pubfi-pub-101"); + assert!(spec.path.join(".git").is_file()); + assert_eq!( + git_stdout(&spec.path, &["rev-parse", "--abbrev-ref", "HEAD"]), + "x/pubfi-pub-101" + ); + + let repo_git_dir = fs::canonicalize(repo_root.join(".git")).expect("repo git dir"); + let git_dir = fs::canonicalize(PathBuf::from(git_stdout( + &spec.path, + &["rev-parse", "--path-format=absolute", "--git-dir"], + ))) + .expect("git dir should canonicalize"); + let git_common_dir = fs::canonicalize(PathBuf::from(git_stdout( + &spec.path, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))) + .expect("git common dir should canonicalize"); + + assert!(git_dir.starts_with(repo_git_dir.join("worktrees"))); + assert_eq!(git_common_dir, repo_git_dir); + assert!( + super::worktree_is_registered( + &repo_root, + &fs::canonicalize(&spec.path).expect("canonical worktree path") + ) + .expect("worktree registration should inspect") + ); + } + + #[test] + fn after_create_hook_runs_only_for_new_worktree() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let hook_log = repo_root.join("after-create.log"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["printf '%s\n' \"$DECODEX_BRANCH\" >> \"$DECODEX_REPO_ROOT/after-create.log\""] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let created = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("worktree should be created"); + let reused = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("worktree should be reused"); + + assert!(!created.reused_existing); + assert!(reused.reused_existing); + assert_eq!( + fs::read_to_string(&hook_log).expect("hook log should exist"), + "x/pubfi-pub-101\n" + ); + } + + #[test] + fn after_create_hook_failure_keeps_created_worktree() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["exit 23"] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let planned = manager.plan_for_issue("PUB-101"); + let error = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect_err("after_create hook failure should stop setup"); + + assert!( + error.to_string().contains("Workspace hook `after_create` command `exit 23` failed") + ); + assert!(planned.path.exists(), "failed hook should keep the worktree for inspection"); + assert!(planned.path.join(".git").is_file(), "failed hook should keep the linked worktree"); + assert!( + super::after_create_pending_marker_path(&planned.path).exists(), + "failed after-create hook should leave a pending bootstrap marker" + ); + } + + #[test] + fn reused_lane_retries_bootstrap_after_interrupted_create_window() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let hook_log = repo_root.join("after-create.log"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["printf '%s\n' \"$DECODEX_BRANCH\" >> \"$DECODEX_REPO_ROOT/after-create.log\""] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let planned = manager.plan_for_issue("PUB-101"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + manager + .create_linked_worktree(&planned, Some(&hooks)) + .expect("linked worktree should be created"); + manager + .validate_worktree_boundary(&planned.path) + .expect("created worktree should validate"); + + assert!( + super::after_create_pending_marker_path(&planned.path).exists(), + "newly created lane should persist the pending bootstrap marker before first hook run" + ); + assert!(!hook_log.exists(), "simulated crash window should not have run hooks yet"); + + let reused = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("reused lane should resume interrupted bootstrap"); + + assert!(reused.reused_existing); + assert_eq!( + fs::read_to_string(&hook_log).expect("hook log should exist after resumed bootstrap"), + "x/pubfi-pub-101\n" + ); + assert!( + !super::after_create_pending_marker_path(&planned.path).exists(), + "successful resumed bootstrap should clear the pending marker" + ); + } + + #[test] + fn after_create_hook_retries_before_reused_lane_dispatch() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let hook_log = repo_root.join("after-create.log"); + let allow_file = repo_root.join("allow-bootstrap"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["printf '%s\n' \"$DECODEX_BRANCH\" >> \"$DECODEX_REPO_ROOT/after-create.log\" && test -f \"$DECODEX_REPO_ROOT/allow-bootstrap\""] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let planned = manager.plan_for_issue("PUB-101"); + let first_error = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect_err("missing bootstrap prerequisite should fail"); + + assert!(first_error.to_string().contains("Workspace hook `after_create` command")); + assert_eq!( + fs::read_to_string(&hook_log).expect("hook log should exist after first failure"), + "x/pubfi-pub-101\n" + ); + assert!( + super::after_create_pending_marker_path(&planned.path).exists(), + "failed bootstrap should leave the pending marker behind" + ); + + fs::write(&allow_file, "ready\n").expect("bootstrap prerequisite should write"); + + let reused = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("reused lane should rerun the pending bootstrap hook"); + + assert!(reused.reused_existing); + assert_eq!( + fs::read_to_string(&hook_log).expect("hook log should include retried bootstrap"), + "x/pubfi-pub-101\nx/pubfi-pub-101\n" + ); + assert!( + !super::after_create_pending_marker_path(&planned.path).exists(), + "successful retry should clear the pending bootstrap marker" + ); + } + + #[test] + fn after_create_hook_handles_hook_managed_pending_marker_removal() { + { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["rm -f \"$DECODEX_WORKTREE_PATH/.decodex-after-create.pending\""] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let spec = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("successful hook that removes the marker should still pass"); + + assert!(spec.path.exists(), "worktree should remain usable after bootstrap"); + assert!( + !super::after_create_pending_marker_path(&spec.path).exists(), + "successful hook should not leave a stale pending marker behind" + ); + } + { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = ["rm -f \"$DECODEX_WORKTREE_PATH/.decodex-after-create.pending\"", "exit 23"] +before_remove_commands = [] +timeout_seconds = 60 + "#, + ); + let planned = manager.plan_for_issue("PUB-101"); + let error = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect_err("failed hook should still leave the lane pending for retry"); + + assert!( + error + .to_string() + .contains("Workspace hook `after_create` command `exit 23` failed") + ); + assert!( + super::after_create_pending_marker_path(&planned.path).exists(), + "failed bootstrap should restore the pending marker even if an earlier command removed it" + ); + } + } + + #[test] + fn workspace_hook_command_returns_without_waiting_for_background_child_pipe_close() { + let (_temp_dir, repo_root) = init_repo(); + let start = Instant::now(); + let output = super::run_workspace_hook_shell_command( + "sleep 5 & printf 'done\\n'", + &repo_root, + &[], + Duration::from_secs(1), + ) + .expect("shell exit should not block on inherited stdout/stderr pipe handles"); + + assert!(output.status.success(), "backgrounded child should not fail the shell command"); + assert_eq!(String::from_utf8_lossy(&output.stdout), "done\n"); + assert!( + start.elapsed() < Duration::from_secs(3), + "hook output collection should not wait for background child pipe closure after shell exit" + ); + } + + #[cfg(unix)] + #[test] + fn workspace_hook_timeout_kills_background_descendants() { + let (_temp_dir, repo_root) = init_repo(); + let child_pid_file = repo_root.join("hook-child.pid"); + let error = super::run_workspace_hook_shell_command( + "sleep 300 & bg=$!; printf '%s\n' \"$bg\" > \"$DECODEX_REPO_ROOT/hook-child.pid\"; wait", + &repo_root, + &[("DECODEX_REPO_ROOT", repo_root.display().to_string())], + Duration::from_secs(1), + ) + .expect_err("timed out hook should fail"); + + assert!(error.to_string().contains("exceeded the 1s timeout")); + + let child_pid = fs::read_to_string(&child_pid_file) + .expect("background child pid should be recorded before timeout") + .trim() + .parse::() + .expect("background child pid should parse"); + let kill_deadline = Instant::now() + Duration::from_secs(2); + + while process_is_alive(child_pid) && Instant::now() < kill_deadline { + thread::sleep(Duration::from_millis(25)); + } + + assert!( + !process_is_alive(child_pid), + "timed out workspace hook should terminate background descendants" + ); + } + + #[cfg(unix)] + fn process_is_alive(process_id: i32) -> bool { + let result = unsafe { libc::kill(process_id, 0) }; + + if result == 0 { + return true; + } + + std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) + } + + #[test] + fn after_create_hook_tolerates_verbose_success_output() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +timeout_seconds = 1 +after_create_commands = ["yes hook-output | head -c 131072 >/dev/stdout"] +before_remove_commands = [] + "#, + ); + let spec = manager + .ensure_worktree_with_hooks("PUB-101", false, &hooks) + .expect("verbose successful hook should not deadlock on captured output"); + + assert!(spec.path.exists(), "worktree should remain usable after verbose bootstrap"); + assert!( + !super::after_create_pending_marker_path(&spec.path).exists(), + "successful verbose hook should clear the pending marker" + ); + } + + #[test] + fn creates_linked_worktree_when_repo_root_is_also_a_linked_worktree() { + let (_temp_dir, primary_repo_root) = init_repo(); + let linked_repo_root = primary_repo_root.parent().unwrap().join("linked-root"); + + run_git( + &primary_repo_root, + &["worktree", "add", "--quiet", "--detach", linked_repo_root.to_str().unwrap(), "HEAD"], + ); + run_git(&linked_repo_root, &["checkout", "--quiet", "-B", "x/pubfi-linked-root", "HEAD"]); + + let worktree_root = linked_repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &linked_repo_root, &worktree_root); + let spec = manager + .ensure_worktree("PUB-101", false) + .expect("worktree should be created from linked repo root"); + + assert_eq!(spec.branch_name, "x/pubfi-pub-101"); + assert!(spec.path.join(".git").is_file()); + + let repo_git_dir = fs::canonicalize(PathBuf::from(git_stdout( + &linked_repo_root, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))) + .expect("linked repo common dir should canonicalize"); + let git_dir = fs::canonicalize(PathBuf::from(git_stdout( + &spec.path, + &["rev-parse", "--path-format=absolute", "--git-dir"], + ))) + .expect("git dir should canonicalize"); + let git_common_dir = fs::canonicalize(PathBuf::from(git_stdout( + &spec.path, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))) + .expect("git common dir should canonicalize"); + + assert!(git_dir.starts_with(repo_git_dir.join("worktrees"))); + assert_eq!(git_common_dir, repo_git_dir); + } + + #[test] + fn linked_worktree_inherits_repo_local_identity_config() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + + run_git(&repo_root, &["config", "user.signingkey", "worktree-tests"]); + run_git(&repo_root, &["config", "codex.github-identity", "y"]); + run_git(&repo_root, &["config", "codex.linear-workspace", "hackink"]); + + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + assert_eq!(git_stdout(&spec.path, &["config", "--get", "user.name"]), "Decodex Tests"); + assert_eq!( + git_stdout(&spec.path, &["config", "--get", "user.email"]), + "decodex-tests@example.com" + ); + assert_eq!(git_stdout(&spec.path, &["config", "--get", "commit.gpgsign"]), "false"); + assert_eq!( + git_stdout(&spec.path, &["config", "--get", "user.signingkey"]), + "worktree-tests" + ); + assert_eq!(git_stdout(&spec.path, &["config", "--get", "codex.github-identity"]), "y"); + assert_eq!( + git_stdout(&spec.path, &["config", "--get", "codex.linear-workspace"]), + "hackink" + ); + } + + #[test] + fn linked_worktree_inherits_repo_local_identity_from_included_config() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let included_config = repo_root.parent().unwrap().join("identity.inc"); + + run_git(&repo_root, &["config", "--unset-all", "user.name"]); + run_git(&repo_root, &["config", "--unset-all", "user.email"]); + + fs::write( + &included_config, + "[user]\n\tname = Included Tests\n\temail = included@example.com\n[codex]\n\tgithub-identity = y\n\tlinear-workspace = hackink\n", + ) + .expect("included config should write"); + + run_git( + &repo_root, + &["config", "--local", "include.path", included_config.to_str().unwrap()], + ); + + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + assert_eq!(git_stdout(&spec.path, &["config", "--get", "user.name"]), "Included Tests"); + assert_eq!( + git_stdout(&spec.path, &["config", "--get", "user.email"]), + "included@example.com" + ); + assert_eq!(git_stdout(&spec.path, &["config", "--get", "codex.github-identity"]), "y"); + assert_eq!( + git_stdout(&spec.path, &["config", "--get", "codex.linear-workspace"]), + "hackink" + ); + } + + #[test] + fn linked_worktree_uses_existing_remote_lane_branch_when_present() { + let (_temp_dir, repo_root) = init_repo(); + let bare_remote = repo_root.parent().unwrap().join("lane-remote.git"); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let lane_branch = "x/pubfi-pub-101"; + + run_git(bare_remote.parent().unwrap(), &["init", "--bare", bare_remote.to_str().unwrap()]); + run_git(&repo_root, &["remote", "set-url", "origin", "../lane-remote.git"]); + run_git(&repo_root, &["push", "-u", "origin", "main"]); + run_git(&repo_root, &["checkout", "-b", lane_branch]); + + fs::write(repo_root.join("LANE.md"), "lane branch\n").expect("lane file should write"); + + run_git(&repo_root, &["add", "LANE.md"]); + run_git(&repo_root, &["commit", "-m", "lane branch"]); + run_git(&repo_root, &["push", "-u", "origin", lane_branch]); + run_git(&repo_root, &["checkout", "main"]); + + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + assert_eq!(git_stdout(&spec.path, &["rev-parse", "--abbrev-ref", "HEAD"]), lane_branch); + assert_eq!( + fs::read_to_string(spec.path.join("LANE.md")).expect("lane file should exist"), + "lane branch\n" + ); + assert_eq!( + git_stdout(&spec.path, &["remote", "get-url", "origin"]), + fs::canonicalize(&bare_remote) + .expect("bare remote should canonicalize") + .to_str() + .expect("bare remote should be valid UTF-8") + ); + } + + #[test] + fn linked_worktree_push_uses_normalized_absolute_origin_when_source_remote_is_relative() { + let (_temp_dir, repo_root) = init_repo(); + let bare_remote = repo_root.parent().unwrap().join("lane-remote.git"); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + + run_git(bare_remote.parent().unwrap(), &["init", "--bare", bare_remote.to_str().unwrap()]); + run_git(&repo_root, &["remote", "set-url", "origin", "../lane-remote.git"]); + run_git(&repo_root, &["push", "-u", "origin", "main"]); + + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + fs::write(spec.path.join("WORKTREE.md"), "linked worktree lane\n") + .expect("worktree file should write"); + + run_git(&spec.path, &["add", "WORKTREE.md"]); + run_git(&spec.path, &["commit", "-m", "worktree change"]); + run_git(&spec.path, &["push", "-u", "origin", "x/pubfi-pub-101"]); + + assert_eq!( + git_stdout(&spec.path, &["remote", "get-url", "origin"]), + fs::canonicalize(&bare_remote) + .expect("bare remote should canonicalize") + .to_str() + .expect("bare remote should be valid UTF-8") + ); + } + + #[test] + fn reused_linked_worktree_normalizes_relative_origin_on_reentry() { + let (_temp_dir, repo_root) = init_repo(); + let bare_remote = repo_root.parent().unwrap().join("lane-remote.git"); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + + run_git(bare_remote.parent().unwrap(), &["init", "--bare", bare_remote.to_str().unwrap()]); + run_git(&repo_root, &["remote", "set-url", "origin", "../lane-remote.git"]); + run_git(&repo_root, &["push", "-u", "origin", "main"]); + + let created = + manager.ensure_worktree("PUB-101", false).expect("worktree should be created"); + + run_git(&repo_root, &["remote", "set-url", "origin", "../lane-remote.git"]); + + let reused = manager.ensure_worktree("PUB-101", false).expect("worktree should be reused"); + + assert!(reused.reused_existing); + assert_eq!(reused.path, created.path); + assert_eq!( + git_stdout(&reused.path, &["remote", "get-url", "origin"]), + fs::canonicalize(&bare_remote) + .expect("bare remote should canonicalize") + .to_str() + .expect("bare remote should be valid UTF-8") + ); + } + + #[test] + fn linked_worktree_leaves_home_relative_origin_unchanged() { + let (_temp_dir, repo_root) = init_repo(); + + run_git(&repo_root, &["remote", "set-url", "origin", "~/lane-remote.git"]); + + super::normalize_origin_remote_for_worktrees(&repo_root) + .expect("home-relative remotes should bypass normalization"); + + assert_eq!(git_stdout(&repo_root, &["remote", "get-url", "origin"]), "~/lane-remote.git"); + assert!(!super::is_relative_filesystem_remote("~/lane-remote.git")); + assert!(!super::is_relative_filesystem_remote("~")); + } + + #[test] + fn linked_worktree_rolls_back_when_origin_normalization_fails() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let spec = manager.plan_for_issue("PUB-101"); + + run_git(&repo_root, &["remote", "set-url", "origin", "../missing-remote.git"]); + + let error = manager + .ensure_worktree("PUB-101", false) + .expect_err("worktree creation should fail when origin normalization fails"); + + assert!( + error.to_string().contains("No such file or directory") + || error.to_string().contains("does not exist"), + "unexpected error: {error:?}" + ); + assert!(!spec.path.exists(), "failed setup should remove the new worktree path"); + assert!( + !super::worktree_is_registered(&repo_root, &spec.path) + .expect("worktree registration should inspect"), + "failed setup should unregister the new worktree" + ); + } + + #[test] + fn linked_worktree_fails_when_remote_branch_probe_errors() { + let (temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let missing_remote = temp_dir.path().join("missing-origin.git"); + + run_git(&repo_root, &["remote", "set-url", "origin", missing_remote.to_str().unwrap()]); + + let error = manager + .ensure_worktree("PUB-101", false) + .expect_err("worktree create should fail when remote probe errors"); + + assert!(error.to_string().contains("Failed to inspect remote worktree branch")); + } + + #[test] + fn rejects_reused_non_worktree_checkout_with_embedded_git_dir() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let worktree_path = worktree_root.join("PUB-101"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + + run_git( + &repo_root, + &["clone", "--quiet", "--no-checkout", ".", worktree_path.to_str().unwrap()], + ); + + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let error = manager + .ensure_worktree("PUB-101", false) + .expect_err("embedded git checkout should be rejected"); + + assert!( + error + .to_string() + .contains("is not a linked git worktree: expected `.git` to be a pointer file") + ); + } + + #[test] + fn removes_linked_worktree_path() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should exist"); + + assert!(manager.remove_worktree_path(&spec.path).expect("worktree should remove")); + assert!(!spec.path.exists()); + assert!( + !git_stdout(&repo_root, &["worktree", "list", "--porcelain"]) + .contains(&format!("worktree {}", spec.path.display())) + ); + } + + #[test] + fn removes_orphaned_marker_directory_without_linked_git_metadata() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let orphan_path = worktree_root.join("PUB-101"); + let hook_log = repo_root.join("before-remove.log"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = ["printf 'hook-ran\n' > \"$DECODEX_REPO_ROOT/before-remove.log\""] +timeout_seconds = 60 + "#, + ); + + fs::create_dir_all(&orphan_path).expect("orphan path should exist"); + fs::write(orphan_path.join(crate::state::RUN_ACTIVITY_MARKER_FILE), "run_id=run-orphan\n") + .expect("runtime marker should write"); + + assert!( + manager + .remove_worktree_path_with_hooks("PUB-101", "x/pubfi-pub-101", &orphan_path, &hooks,) + .expect("orphan marker directory should remove") + ); + assert!(!orphan_path.exists(), "orphan marker directory should be deleted"); + assert!( + !hook_log.exists(), + "before_remove hook should not run for a non-worktree marker directory" + ); + } + + #[test] + fn before_remove_hook_runs_before_cleanup() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let hook_log = repo_root.join("before-remove.log"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = ["printf '%s:%s\n' \"$DECODEX_ISSUE_ID\" \"$DECODEX_BRANCH\" > \"$DECODEX_REPO_ROOT/before-remove.log\""] +timeout_seconds = 60 + "#, + ); + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should exist"); + + assert!( + manager + .remove_worktree_path_with_hooks( + &spec.issue_identifier, + &spec.branch_name, + &spec.path, + &hooks + ) + .expect("worktree should remove") + ); + assert_eq!( + fs::read_to_string(&hook_log).expect("hook log should exist"), + "PUB-101:x/pubfi-pub-101\n" + ); + assert!(!spec.path.exists(), "successful cleanup should still remove the worktree"); + } + + #[test] + fn before_remove_hook_failure_blocks_cleanup() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = ["exit 19"] +timeout_seconds = 60 + "#, + ); + let spec = manager.ensure_worktree("PUB-101", false).expect("worktree should exist"); + let error = manager + .remove_worktree_path_with_hooks( + &spec.issue_identifier, + &spec.branch_name, + &spec.path, + &hooks, + ) + .expect_err("before_remove hook failure should block cleanup"); + + assert!( + error.to_string().contains("Workspace hook `before_remove` command `exit 19` failed") + ); + assert!(spec.path.exists(), "blocked cleanup should keep the worktree"); + } + + #[test] + fn before_remove_hook_does_not_run_for_unregistered_directory() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let rogue_path = worktree_root.join("PUB-rogue"); + let hook_log = repo_root.join("before-remove.log"); + + fs::create_dir_all(&rogue_path).expect("rogue path should exist"); + fs::write(rogue_path.join(".git"), b"not-a-worktree\n") + .expect("rogue path should contain a fake git pointer"); + + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let hooks = workspace_hooks( + r#" +[execution.workspace_hooks] +after_create_commands = [] +before_remove_commands = ["printf 'hook-ran\n' > \"$DECODEX_REPO_ROOT/before-remove.log\""] +timeout_seconds = 60 + "#, + ); + let error = manager + .remove_worktree_path_with_hooks("PUB-rogue", "x/pubfi-pub-rogue", &rogue_path, &hooks) + .expect_err("unregistered directory should fail validation before before_remove hooks"); + + assert!( + !error.to_string().trim().is_empty(), + "validation failure should still surface an actionable error" + ); + assert!( + !hook_log.exists(), + "before_remove hook should not run before linked worktree validation succeeds" + ); + assert!( + rogue_path.exists(), + "failed validation should leave the unregistered directory untouched" + ); + } + + #[test] + fn rejects_worktree_removal_when_path_escapes_root_via_parent_components() { + let (_temp_dir, repo_root) = init_repo(); + let worktree_root = repo_root.join(".worktrees"); + let escaped_target = repo_root.join("outside").join("PUB-101"); + + fs::create_dir_all(&worktree_root).expect("worktree root should exist"); + fs::create_dir_all(&escaped_target).expect("escaped target should exist"); + + let manager = WorktreeManager::new("pubfi", &repo_root, &worktree_root); + let escaped_path = worktree_root.join("../outside/PUB-101"); + let error = manager + .remove_worktree_path(&escaped_path) + .expect_err("escaped worktree path should be rejected"); + + assert!(error.to_string().contains("outside worktree_root")); + assert!(escaped_target.exists()); + } +} diff --git a/build.rs b/build.rs deleted file mode 100644 index 6cbe3eb7..00000000 --- a/build.rs +++ /dev/null @@ -1,20 +0,0 @@ -#![allow(missing_docs)] - -use std::error::Error; - -use vergen_gitcl::{CargoBuilder, Emitter, GitclBuilder}; - -fn main() -> Result<(), Box> { - let mut emitter = Emitter::default(); - - emitter.add_instructions(&CargoBuilder::default().target_triple(true).build()?)?; - - // Disable the git version if installed from . - if emitter.add_instructions(&GitclBuilder::default().sha(true).build()?).is_err() { - println!("cargo:rustc-env=VERGEN_GIT_SHA=crates.io"); - } - - emitter.emit()?; - - Ok(()) -} diff --git a/decodex.example.toml b/decodex.example.toml new file mode 100644 index 00000000..87f6a9c6 --- /dev/null +++ b/decodex.example.toml @@ -0,0 +1,20 @@ +service_id = "your-service-id" + +[tracker] +api_key_env_var = "LINEAR_API_KEY" + +[github] +token_env_var = "GITHUB_TOKEN" + +# Optional Codex-specific runtime policy. +# Omit this block to use `internal_review_mode = "loop"` and `external_review_enabled = true`. +[codex] +external_review_enabled = true +internal_review_mode = "loop" + +# Required project paths. Project configs are managed outside checkouts, for example under +# `~/.codex/decodex/projects//project.toml`. +# Omit `worktree_root` to use `/.worktrees`. +[paths] +repo_root = "/absolute/path/to/target/repo" +worktree_root = ".worktrees" diff --git a/dev/operator-dashboard-mock.mjs b/dev/operator-dashboard-mock.mjs new file mode 100755 index 00000000..4086f5da --- /dev/null +++ b/dev/operator-dashboard-mock.mjs @@ -0,0 +1,743 @@ +#!/usr/bin/env node + +import http from "node:http"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_LISTEN_ADDRESS = "127.0.0.1:57399"; +const DEFAULT_READY_STALE_SECONDS = 120; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +function parseArgs(argv) { + const options = { + authDir: null, + dashboardHtml: path.join(repoRoot, "src/orchestrator/operator_dashboard.html"), + listenAddress: DEFAULT_LISTEN_ADDRESS, + readyStaleSeconds: DEFAULT_READY_STALE_SECONDS, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + if (arg === "--listen-address") { + options.listenAddress = requiredValue(argv, (index += 1), arg); + continue; + } + if (arg === "--dashboard-html") { + options.dashboardHtml = path.resolve(requiredValue(argv, (index += 1), arg)); + continue; + } + if (arg === "--codex-auth-dir") { + options.authDir = path.resolve(requiredValue(argv, (index += 1), arg)); + continue; + } + if (arg === "--use-codex-auth") { + options.authDir = path.join(process.env.HOME || ".", ".codex"); + continue; + } + if (arg === "--ready-stale-seconds") { + options.readyStaleSeconds = Number(requiredValue(argv, (index += 1), arg)); + if (!Number.isFinite(options.readyStaleSeconds) || options.readyStaleSeconds <= 0) { + throw new Error("--ready-stale-seconds must be a positive number"); + } + continue; + } + + throw new Error(`Unknown argument: ${arg}`); + } + + return options; +} + +function requiredValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith("--")) { + throw new Error(`${flag} requires a value`); + } + + return value; +} + +function printHelp() { + console.log(`Usage: node dev/operator-dashboard-mock.mjs [options] + +Serves the real operator dashboard HTML with a comprehensive mock /state payload. + +Options: + --listen-address HOST:PORT Bind address (default ${DEFAULT_LISTEN_ADDRESS}) + --dashboard-html PATH Dashboard HTML path + --use-codex-auth Load auth*.json accounts from ~/.codex + --codex-auth-dir DIR Load auth*.json accounts from DIR + --ready-stale-seconds N /readyz freshness window (default ${DEFAULT_READY_STALE_SECONDS}) + -h, --help Show this help +`); +} + +function splitListenAddress(value) { + const [host, portText] = value.split(":"); + const port = Number(portText); + if (!host || !Number.isInteger(port) || port <= 0 || port > 65_535) { + throw new Error(`Invalid listen address: ${value}`); + } + + return { host, port }; +} + +function nowUnix() { + return Math.floor(Date.now() / 1000); +} + +function unixToIso(seconds) { + return new Date(seconds * 1000).toISOString(); +} + +function ago(seconds) { + return unixToIso(nowUnix() - seconds); +} + +function later(seconds) { + return unixToIso(nowUnix() + seconds); +} + +function unixLater(seconds) { + return nowUnix() + seconds; +} + +function account({ + email, + fingerprint, + plan = "pro", + status = "available", + primary = 72, + primaryReset = 14_400, + secondary = 91, + secondaryReset = 518_400, + creditsBalance = "9.99", + creditsHasCredits = true, + creditsUnlimited = false, + note = "usage probe ok", + selected = false, +}) { + return { + account_email: email, + account_fingerprint: fingerprint, + plan_type: plan, + status: selected ? "selected" : status, + refresh_status: "not_needed", + checked_at_unix_epoch: nowUnix() - 30, + selected_at_unix_epoch: selected ? nowUnix() - 20 : null, + primary_window_seconds: 18_000, + primary_remaining_percent: primary, + primary_resets_at_unix_epoch: unixLater(primaryReset), + secondary_window_seconds: 604_800, + secondary_remaining_percent: secondary, + secondary_resets_at_unix_epoch: unixLater(secondaryReset), + credits_has_credits: creditsHasCredits, + credits_unlimited: creditsUnlimited, + credits_balance: creditsBalance, + rate_limit_reached_type: null, + cooldown_until_unix_epoch: null, + note, + }; +} + +function mockAccounts() { + return [ + account({ + email: "primary@example.test", + fingerprint: "...acct01", + primary: 96, + secondary: 92, + selected: true, + }), + account({ + email: "weekly-depleted@example.test", + fingerprint: "...acct02", + status: "usage_limited", + primary: 100, + secondary: 0, + creditsBalance: "0", + creditsHasCredits: false, + }), + account({ + email: "nightly@example.test", + fingerprint: "...acct03", + primary: 44, + secondary: 78, + creditsBalance: "4.20", + }), + ]; +} + +function childAgentActivity() { + return { + buckets: [ + { + name: "Model", + wall_seconds: 693, + event_count: 12, + tool_call_count: 0, + input_tokens: 4_270_000, + output_tokens: 12_000, + output_bytes: 0, + }, + { + name: "Shell", + wall_seconds: 96, + event_count: 10, + tool_call_count: 6, + input_tokens: 0, + output_tokens: 0, + output_bytes: 24_000, + }, + { + name: "Browser/Image", + wall_seconds: 41, + event_count: 6, + tool_call_count: 3, + input_tokens: 0, + output_tokens: 0, + output_bytes: 180_000, + }, + { + name: "Tracker", + wall_seconds: 0, + event_count: 2, + tool_call_count: 2, + input_tokens: 0, + output_tokens: 0, + output_bytes: 2_100, + }, + ], + current_bucket: "Model", + current_detail: "waiting after tool output", + current_started_unix_epoch: null, + current_elapsed_seconds: 652, + wall_seconds: 830, + event_count: 30, + tool_call_count: 11, + input_tokens_current: 105_000, + input_tokens_max: 128_000, + input_tokens_cumulative: 4_270_000, + output_tokens_cumulative: 12_000, + largest_tool_output_bytes: 180_000, + largest_tool_output_tool: "view_image", + large_output_warnings: ["view_image repeated 3 large outputs; largest 180000 bytes"], + }; +} + +function activeRun({ + accounts, + attempt = 1, + issue = "XY-445", + operation = "agent_run", + status = "running", + title = "Account pool dashboard polish", + processAlive = true, + activeLease = true, + childActivity = childAgentActivity(), +}) { + const selectedAccount = accounts.find((item) => item.status === "selected") || accounts[0] || null; + + return { + project_id: "decodex-preview", + run_id: `${issue.toLowerCase()}-attempt-${attempt}-mock`, + issue_id: issue, + issue_identifier: issue, + title, + attempt_number: attempt, + status, + attempt_status: status, + phase: status === "stalled" ? "reconciling" : "executing", + wait_reason: null, + current_operation: operation, + thread_id: `thread-${issue.toLowerCase()}`, + turn_id: `turn-${attempt}`, + thread_status: processAlive ? "active" : "systemError", + thread_active_flags: processAlive ? [] : ["waitingOnApproval"], + interactive_requested: !processAlive, + continuation_pending: false, + active_lease: activeLease, + queue_lease_state: activeLease ? "held" : "not_held", + execution_liveness: processAlive ? "process_alive" : "protocol_observed", + updated_at: ago(12), + last_run_activity_at: ago(processAlive ? 8 : 190), + last_protocol_activity_at: ago(processAlive ? 16 : 185), + last_progress_at: ago(processAlive ? 18 : 240), + idle_for_seconds: processAlive ? 8 : 190, + protocol_idle_for_seconds: processAlive ? 16 : 185, + suspected_stall: !processAlive, + last_event_type: "turn/completed", + last_event_at: ago(processAlive ? 16 : 185), + event_count: childActivity?.event_count || 4, + process_id: processAlive ? process.pid : 44_444, + process_alive: processAlive, + retry_kind: processAlive ? null : "failure_retry", + next_retry_at: processAlive ? null : later(600), + effective_model: "gpt-5.4", + effective_model_provider: "openai", + effective_cwd: `/Users/x/code/y/hack-ink/decodex/.worktrees/${issue}`, + effective_approval_policy: "never", + effective_approvals_reviewer: null, + effective_sandbox_mode: "danger-full-access", + protocol_event: `turn/completed @ ${ago(processAlive ? 16 : 185)}`, + codex_account: selectedAccount, + codex_accounts: accounts, + child_agent_activity: childActivity, + branch_name: `xy/${issue.toLowerCase()}-mock`, + worktree_path: `/Users/x/code/y/hack-ink/decodex/.worktrees/${issue}`, + }; +} + +function queuedCandidates() { + return [ + { + issue_id: "issue-xy-445", + issue_identifier: "XY-445", + title: "Running lane still owns this queue claim", + state: "In Progress", + priority: 1, + created_at: ago(2_400), + classification: "claimed", + reason: "shared_claim_present", + attention: null, + blocker_identifiers: [], + }, + { + issue_id: "issue-xy-450", + issue_identifier: "XY-450", + title: "Ready implementation lane", + state: "Todo", + priority: 2, + created_at: ago(4_800), + classification: "ready", + reason: "normal_dispatch", + attention: null, + blocker_identifiers: [], + }, + { + issue_id: "issue-xy-451", + issue_identifier: "XY-451", + title: "Capacity waits without becoming blocked", + state: "Todo", + priority: 3, + created_at: ago(6_200), + classification: "waiting", + reason: "global_concurrency_exhausted", + attention: null, + blocker_identifiers: [], + }, + { + issue_id: "issue-xy-452", + issue_identifier: "XY-452", + title: "Needs attention from previous stalled run", + state: "In Progress", + priority: 1, + created_at: ago(8_600), + classification: "blocked", + reason: "issue_needs_attention", + attention: { + summary: "App-server thread ended with systemError.", + run_id: "xy-452-attempt-2-mock", + attempt_number: 2, + current_operation: "agent_run", + thread_status: "systemError", + retry_budget_attempt_count: 2, + last_activity_at: ago(900), + last_progress_at: ago(1_200), + last_event_type: "thread/error", + event_count: 23, + process_alive: false, + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-452", + worktree_has_tracked_changes: true, + }, + blocker_identifiers: ["XY-399"], + }, + ]; +} + +function postReviewLanes() { + return [ + { + issue_id: "issue-xy-460", + issue_identifier: "XY-460", + issue_state: "In Review", + branch_name: "xy/xy-460-ready", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-460", + classification: "ready_to_land", + reason: "Approvals and required checks are complete.", + pr_url: "https://github.com/hack-ink/decodex/pull/460", + pr_state: "OPEN", + review_decision: "APPROVED", + mergeable: "MERGEABLE", + check_state: "SUCCESS", + unresolved_review_threads: 0, + }, + { + issue_id: "issue-xy-461", + issue_identifier: "XY-461", + issue_state: "In Review", + branch_name: "xy/xy-461-review-wait", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-461", + classification: "wait_for_review", + reason: "External review request is pending.", + pr_url: "https://github.com/hack-ink/decodex/pull/461", + pr_state: "OPEN", + review_decision: "REVIEW_REQUIRED", + mergeable: "UNKNOWN", + check_state: "PENDING", + unresolved_review_threads: 1, + }, + { + issue_id: "issue-xy-462", + issue_identifier: "XY-462", + issue_state: "Done", + branch_name: "xy/xy-462-closeout", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-462", + classification: "closeout_blocked", + reason: "Merged PR is visible but tracker closeout needs operator attention.", + pr_url: "https://github.com/hack-ink/decodex/pull/462", + pr_state: "MERGED", + review_decision: "APPROVED", + mergeable: "MERGEABLE", + check_state: "SUCCESS", + unresolved_review_threads: 0, + }, + ]; +} + +function worktrees() { + return [ + { + issue_id: "issue-xy-445", + issue_identifier: "XY-445", + issue_state: "In Progress", + branch_name: "xy/xy-445-mock", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-445", + ownership: "active_lane", + ownership_reason: "Currently leased by a running lane.", + }, + { + issue_id: "issue-xy-460", + issue_identifier: "XY-460", + issue_state: "In Review", + branch_name: "xy/xy-460-ready", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-460", + ownership: "post_review_lane", + ownership_reason: "Retained for review, landing, or closeout follow-up.", + }, + { + issue_id: "issue-xy-499", + issue_identifier: "XY-499", + issue_state: "Canceled", + branch_name: "xy/xy-499-orphan", + worktree_path: "/Users/x/code/y/hack-ink/decodex/.worktrees/XY-499", + ownership: "local_cleanup", + ownership_reason: + "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", + }, + ]; +} + +function historyLane(accounts) { + const run = activeRun({ + accounts, + attempt: 1, + issue: "XY-430", + operation: "completed", + status: "succeeded", + title: "Completed dashboard lane", + processAlive: false, + activeLease: false, + childActivity: null, + }); + run.updated_at = ago(7_200); + run.last_run_activity_at = ago(7_200); + run.last_protocol_activity_at = ago(7_260); + run.last_progress_at = ago(7_260); + run.process_alive = false; + run.thread_status = "completed"; + + return { + project_id: "decodex-preview", + issue_id: "issue-xy-430", + issue_identifier: "XY-430", + title: "Completed dashboard lane", + issue_key: "XY-430", + attempt_count: 2, + ledger_outcome: { + ledger_status: "present", + final_outcome: "succeeded", + final_event_type: "issue_closeout_complete", + final_event_at: ago(7_200), + summary: "Merged, closed out, and cleaned up.", + pr_url: "https://github.com/hack-ink/decodex/pull/430", + commit_sha: "abc123def456", + branch: "xy/xy-430-dashboard", + closeout_status: "completed", + needs_attention_reason: null, + lifecycle_started_at: ago(12_000), + lifecycle_finished_at: ago(7_200), + lifecycle_elapsed_seconds: 4_800, + record_count: 8, + }, + latest_run: run, + attempts: [ + { ...run, run_id: "xy-430-attempt-1-mock", attempt_number: 1, status: "failed" }, + { ...run, run_id: "xy-430-attempt-2-mock", attempt_number: 2, status: "succeeded" }, + ], + }; +} + +function buildSnapshot(accounts) { + const activeRuns = [ + activeRun({ accounts }), + activeRun({ + accounts, + attempt: 2, + issue: "XY-452", + title: "Stalled lane requiring attention", + processAlive: false, + activeLease: false, + }), + ]; + const reviewLanes = postReviewLanes(); + const retainedWorktrees = worktrees().filter((item) => item.ownership !== "active_lane"); + + return { + project_id: "decodex-preview", + run_limit: 25, + warnings: ["external_observer_status_skipped"], + connector_backoffs: [ + { + project_id: "decodex-preview", + connector: "linear", + sync_phase: "queued_candidates", + quota_class: "rate_limit", + reset_at: later(600), + reset_unix_epoch: unixLater(600), + reset_source: "retry_after", + retry_after_seconds: 600, + next_action: "serve cached local runtime state until reset", + warning: "tracker_rate_limited", + }, + ], + projects: [ + { + project_id: "decodex-preview", + config_path: "~/.codex/decodex/projects/decodex/project.toml", + repo_root: "/Users/x/code/y/hack-ink/decodex", + enabled: true, + active_run_count: activeRuns.length, + queued_candidate_count: queuedCandidates().length, + post_review_lane_count: reviewLanes.length, + retained_worktree_count: retainedWorktrees.length, + waiting_lane_count: 2, + attention_count: 2, + connector_state: "degraded", + last_activity_at: ago(8), + warning_count: 1, + }, + ], + active_runs: activeRuns, + queued_candidates: queuedCandidates(), + recent_runs: [], + history_lanes: [historyLane(accounts)], + worktrees: worktrees(), + post_review_lanes: reviewLanes, + }; +} + +function parseJwtPayload(token) { + if (!token || typeof token !== "string") { + return {}; + } + const payload = token.split(".")[1]; + if (!payload) { + return {}; + } + + try { + return JSON.parse(Buffer.from(payload, "base64url").toString("utf8")); + } catch (_error) { + return {}; + } +} + +function accountFingerprint(accountId) { + const value = String(accountId || ""); + + return value ? `...${value.slice(-6)}` : "unknown"; +} + +function scoreAccount(account) { + const primary = account.primary_remaining_percent ?? 0; + const secondary = account.secondary_remaining_percent ?? primary; + let score = primary * 1_000 + secondary * 10; + + if (account.rate_limit_reached_type) { + score -= 50_000; + } + if (account.credits_has_credits === false && account.credits_unlimited !== true) { + score -= 100_000; + } + + return score; +} + +async function codexAuthAccounts(authDir) { + const entries = await fs.readdir(authDir, { withFileTypes: true }); + const files = entries + .filter((entry) => entry.isFile() && /^auth.*\.json$/u.test(entry.name)) + .map((entry) => entry.name) + .sort((left, right) => { + if (left === "auth.json") { + return -1; + } + if (right === "auth.json") { + return 1; + } + + return left.localeCompare(right); + }); + + if (!files.length) { + throw new Error(`No auth*.json files found in ${authDir}`); + } + + const accounts = []; + + for (const file of files) { + const authPath = path.join(authDir, file); + const raw = JSON.parse(await fs.readFile(authPath, "utf8")); + const tokens = raw.tokens || {}; + const claims = parseJwtPayload(tokens.id_token); + const base = { + account_email: raw.email || tokens.email || claims.email || null, + account_fingerprint: accountFingerprint(tokens.account_id), + plan_type: null, + status: "available", + refresh_status: "not_needed", + checked_at_unix_epoch: nowUnix(), + selected_at_unix_epoch: null, + primary_window_seconds: 18_000, + primary_remaining_percent: null, + primary_resets_at_unix_epoch: null, + secondary_window_seconds: 604_800, + secondary_remaining_percent: null, + secondary_resets_at_unix_epoch: null, + credits_has_credits: null, + credits_unlimited: null, + credits_balance: null, + rate_limit_reached_type: null, + cooldown_until_unix_epoch: null, + note: "real auth loaded; usage not queried", + }; + + accounts.push(base); + } + + const selected = accounts.reduce((best, current) => + scoreAccount(current) > scoreAccount(best) ? current : best, + ); + for (const account of accounts) { + account.status = account === selected ? "selected" : accountUsageStatus(account); + account.selected_at_unix_epoch = account === selected ? nowUnix() : null; + } + + return accounts; +} + +function accountUsageStatus(account) { + if ( + account.rate_limit_reached_type || + account.primary_remaining_percent === 0 || + account.secondary_remaining_percent === 0 || + (account.credits_has_credits === false && account.credits_unlimited !== true) + ) { + return "usage_limited"; + } + if (account.status === "usage_probe_failed") { + return "usage_probe_failed"; + } + + return "available"; +} + +function send(response, statusCode, contentType, body, headers = {}) { + response.writeHead(statusCode, { + "content-type": contentType, + "content-length": Buffer.byteLength(body), + "cache-control": "no-store", + ...headers, + }); + response.end(body); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const staticAccounts = options.authDir + ? await codexAuthAccounts(options.authDir) + : mockAccounts(); + let lastPublishedAt = nowUnix(); + const { host, port } = splitListenAddress(options.listenAddress); + const server = http.createServer(async (request, response) => { + try { + if (request.method !== "GET") { + send(response, 405, "text/plain; charset=utf-8", "method not allowed"); + return; + } + const url = new URL(request.url || "/", `http://${options.listenAddress}`); + if (url.pathname === "/" || url.pathname === "/dashboard") { + const html = await fs.readFile(options.dashboardHtml, "utf8"); + send(response, 200, "text/html; charset=utf-8", html); + return; + } + if (url.pathname === "/livez") { + send(response, 200, "text/plain; charset=utf-8", "ok"); + return; + } + if (url.pathname === "/readyz") { + const stale = nowUnix() - lastPublishedAt > options.readyStaleSeconds; + send( + response, + stale ? 503 : 200, + "text/plain; charset=utf-8", + stale ? "snapshot_stale" : "ready", + ); + return; + } + if (url.pathname === "/state") { + lastPublishedAt = nowUnix(); + const snapshot = buildSnapshot(staticAccounts); + send(response, 200, "application/json", JSON.stringify(snapshot), { + "X-Decodex-Snapshot-Unix-Epoch": String(lastPublishedAt), + }); + return; + } + + send(response, 404, "text/plain; charset=utf-8", "not found"); + } catch (error) { + send(response, 500, "text/plain; charset=utf-8", error?.message || "mock server error"); + } + }); + + server.listen(port, host, () => { + console.log(`operator dashboard mock: http://${host}:${port}/dashboard`); + console.log( + options.authDir + ? `accounts: ${options.authDir} (${staticAccounts.length} loaded)` + : `accounts: ${staticAccounts.length} synthetic fixture accounts`, + ); + }); +} + +main().catch((error) => { + console.error(error?.message || error); + process.exit(1); +}); diff --git a/docs/decisions/decodex-plugin-source.md b/docs/decisions/decodex-plugin-source.md new file mode 100644 index 00000000..a92f389c --- /dev/null +++ b/docs/decisions/decodex-plugin-source.md @@ -0,0 +1,54 @@ +# Decodex Plugin Source + +Status: accepted +Date: 2026-05-09 +Question: Where should reusable agent-facing Decodex usage instructions live? +Decision: Maintain the canonical Decodex plugin in this repository under +`plugins/decodex/`. Generic Codex or Playbook repositories may keep portable routing +rules, but they should not own Decodex-specific CLI, +automation, tracker, label, review, landing, closeout, or project-contract details. +Consequences: Decodex runtime and usage guidance can now change in the same repository +lane. Playbook guidance that still mentions Decodex should point here or stay generic. + +## Context + +Decodex has two supported use modes: + +- manual CLI use for human-driven development, status inspection, commit creation, + PR landing, dry runs, project registration, and local operator checks +- runtime-owned automation for registered projects, retained lanes, service-scoped + Linear labels, issue-scoped tracker tools, review handoff, landing, closeout, and + cleanup + +Earlier Decodex instructions lived in generic Playbook skills while the CLI and +lifecycle were still settling. The Decodex-specific authority now lives in this +repository because the generic Playbook repo does not own Decodex runtime code, +registered project contracts, or operator docs. + +## Decision + +`plugins/decodex/` is the canonical installable plugin source for Decodex usage +instructions. + +The plugin should own reusable agent-facing procedures and mode routing: + +- `decodex` for choosing manual CLI mode versus automation mode +- `manual-cli` for operator CLI use +- `automation` for retained-lane control-plane use +- `commit` for human-driven `decodex commit` +- `land` for explicit human-driven `decodex land` +- `labels` for Decodex Linear labels + +The plugin must route to `apps/decodex/src/`, `docs/spec/`, `docs/runbook/`, `docs/reference/`, +registered project `WORKFLOW.md`, and registered project `project.toml` instead of +copying their full contracts. + +## Consequences + +- Decodex-specific skill updates can land with matching runtime, spec, and runbook + updates. +- Generic Playbook skills can shrink to generic repo discipline and explicit routing. +- `~/.codex/AGENTS.md` remains a portable bootstrap surface, not a Decodex runtime or + operator contract. +- Semantic drift audits for Decodex behavior changes should include `plugins/decodex/` + when the behavior affects agent-facing usage instructions. diff --git a/docs/decisions/index.md b/docs/decisions/index.md new file mode 100644 index 00000000..d7fc2723 --- /dev/null +++ b/docs/decisions/index.md @@ -0,0 +1,28 @@ +# Decisions Index + +Purpose: Route agents to durable design choices that explain why the repository is shaped +this way. + +Question this index answers: "why was it designed this way?" + +## Use this index when + +- You need the rationale behind a stable repository or packaging choice. +- You need to understand tradeoffs that should survive implementation churn. +- You are considering changing an existing design boundary and need the prior reasoning + first. + +## Do not use this index when + +- You need the current operator sequence. +- You need the current implementation map only. +- You need the normative contract without the rationale layer. + +## Current decisions + +- [`decodex-plugin-source.md`](./decodex-plugin-source.md) records why this repository + owns the canonical Decodex plugin and why generic Playbook guidance should only keep + portable routing. +- [`static-public-site.md`](./static-public-site.md) records why the public Decodex site + remains static while runtime/operator behavior stays in the CLI and local control + plane. diff --git a/docs/decisions/static-public-site.md b/docs/decisions/static-public-site.md new file mode 100644 index 00000000..1c17be29 --- /dev/null +++ b/docs/decisions/static-public-site.md @@ -0,0 +1,25 @@ +# Static Public Site + +Status: accepted + +Date: 2026-05-09 + +Question: Should the public Decodex site become a dynamic app now that the runtime and +site live in one mono repo? + +Decision: Keep the public site static by default. `site/` remains an Astro static site +that renders checked-in content and generated JSON from the GitHub signal pipeline. +Runtime orchestration, local operator state, tracker writes, app-server integration, and +the operator dashboard stay in `apps/decodex/` and the local `decodex serve` control +plane. + +Consequences: + +- Public content remains diffable, reviewable, cacheable, and deployable through GitHub + Pages without a live Decodex daemon. +- `tools/github/` remains the content-generation boundary for public signals and + release deltas. +- `apps/decodex/` can evolve the runtime without turning the public website into an + operational dependency. +- Dynamic public capabilities such as login, personalized feeds, live queries, or + paid/private access require a later decision before the site depends on a backend. diff --git a/docs/governance.md b/docs/governance.md deleted file mode 100644 index ced9007a..00000000 --- a/docs/governance.md +++ /dev/null @@ -1,94 +0,0 @@ -# Documentation Governance - -Purpose: Define how agent-facing documentation is organized, updated, and kept consistent -across this repository. - -Audience: All documentation under `docs/` is written for AI agents and LLM workflows. -The split between `spec` and `guide` is by task shape, not by reader type. - -## Principles - -- Optimize for retrieval, routing, and execution. -- Keep one authoritative document per topic. -- Separate normative truth from procedural steps. -- Prefer explicit section labels and stable links over prose-heavy narrative. -- Let structure emerge from real topics. Avoid premature folder taxonomies. - -## Document classes - -| Class | Location | Answers | Source of truth for | Update trigger | -| --- | --- | --- | --- | --- | -| Spec | `docs/spec/` | What must be true? | Contracts, schemas, invariants, required behavior | Any behavior or schema change | -| Guide | `docs/guide/` | What should I do? | Runbooks, migrations, validation, troubleshooting | Any procedure or operational change | -| Plan artifacts | `docs/plans/` | Which saved plan artifact should a planning tool or execution workflow use? | Tool-managed planning outputs | As emitted or updated by the relevant tool | - -## Placement rules - -- If a document defines correctness, it belongs in `docs/spec/`. -- If a document defines actions, it belongs in `docs/guide/`. -- Do not treat `docs/plans/` as a general-purpose docs bucket. -- Use `docs/plans/` only for artifacts produced or consumed by planning tools or - workflows that explicitly depend on saved plan files. -- Do not duplicate the same authoritative content across documents. Link to the source - of truth instead. -- A guide may summarize why a step exists, but normative statements still live in the - governing spec. - -## Document contracts - -Every document should start with a short routing header. - -Spec header: - -- `Purpose` -- `Status: normative` -- `Read this when` -- `Not this document` -- `Defines` - -Guide header: - -- `Goal` -- `Read this when` -- `Inputs` or `Preconditions` -- `Depends on` -- `Outputs` or `Verification` - -## Structure rules - -- Prefer shallow paths by default. -- Add subfolders only when they mirror stable system boundaries or improve retrieval. -- Use descriptive `snake_case` file names. -- Do not require fixed filename prefixes unless a real ambiguity appears. -- Do not create empty folders, empty indexes, or placeholder documents to satisfy a - taxonomy. - -## Canonical entry points - -- Unified documentation router: `docs/index.md` -- Normative router: `docs/spec/index.md` -- Procedural router: `docs/guide/index.md` -- Repo task and automation entrypoints: `Makefile.toml` - -## LLM reading guidance - -When answering a repository question: - -1. Read `docs/index.md` for routing. -2. Route by question type: - - "What must be true?" -> `docs/spec/index.md` - - "What should I do?" -> `docs/guide/index.md` -3. Read `Makefile.toml` when the task depends on repository automation or named tasks. -4. Use `docs/plans/` only when the task explicitly concerns a saved plan artifact used by - a planning tool or execution workflow. - -## Update workflow - -- Behavior or schema change: update the relevant spec. -- Procedure change: update the relevant guide. -- If a change touches both truth and procedure, update both documents and keep their - boundary explicit. -- When a guide starts carrying normative content, move that content into spec and link - to it. -- Do not impose local document-header requirements on files under `docs/plans/`; those - files are owned by the planning tool or workflow that created them. diff --git a/docs/guide/index.md b/docs/guide/index.md deleted file mode 100644 index 9cb1835f..00000000 --- a/docs/guide/index.md +++ /dev/null @@ -1,62 +0,0 @@ -# Guide Index - -Purpose: Route agents to procedural documents that tell them how to execute work safely -and repeatably. - -Question this index answers: "what should I do?" - -## Use this index when - -- You need a runbook, how-to, migration sequence, validation flow, troubleshooting - path, or maintenance procedure. -- You already know the relevant spec and need the operational steps. -- You need a bounded sequence with prerequisites and verification. - -## Do not use this index when - -- You need the authoritative contract, schema, or invariant. -- You need a planning-tool artifact or a saved execution plan under `docs/plans/`. -- You need broad documentation policy or repo task-entrypoint rules; read - `docs/governance.md` or `Makefile.toml` instead. - -## What belongs in `docs/guide/` - -- Task-oriented runbooks. -- Validation and test procedures. -- Migration, rollout, rollback, and recovery sequences. -- Troubleshooting flows and operator checklists. -- Short implementation recipes that depend on a governing spec. - -## Guide document contract - -Start each guide with a compact routing header: - -- `Goal` -- `Read this when` -- `Inputs` or `Preconditions` -- `Depends on` -- `Outputs` or `Verification` - -Then structure the body for execution: - -- Write steps in the order an agent should perform them. -- Keep commands, checks, and rollback points explicit. -- Link to specs for normative truth instead of restating contracts. -- Include failure branches only when they change the next action. -- End with verification so an agent can tell whether the guide succeeded. - -## Structure policy - -- Group guides by workflow or subsystem only when multiple guides exist and the grouping - improves retrieval. -- Do not create empty category folders or placeholder section headings. -- Prefer titles that encode the task or outcome, such as `validate_release.md` or - `rerun_ingest_job.md`. -- Keep the guide index as a router, not a dumping ground for long explanations. - -## Current guides - -- `local_github_signal_workflow.md` defines the MVP local loop for GitHub bundle - collection, Codex analysis, validation, and publish-to-Pages handoff. -- `github_pages_deploy.md` defines the GitHub Pages Actions deployment path and the - manual `decodex.space` custom-domain setup. diff --git a/docs/index.md b/docs/index.md index 57b12141..ab50daa3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,36 +1,67 @@ # Documentation Index -Purpose: Route agents to the smallest correct document set for the current task. +Purpose: Route agents to the smallest correct repository surface for the current task. Audience: All documentation in this repository is written for AI agents and LLM workflows. The split below is by question type, not by human-versus-agent audience. ## Read order -- Read `docs/governance.md` for document contracts and placement rules. -- Read `Makefile.toml` when the task depends on repo task names or execution entrypoints. +- Read `README.md` first when you need the repository scope, top-level layout, or + current source-of-truth boundaries. +- Use `cargo make` whenever an equivalent repo task exists. When task details matter, + inspect `Makefile.toml` directly or run `cargo make --list-all-steps`. +- Read `docs/policy.md` for document contracts, placement rules, and naming rules. +- Read the registered project `WORKFLOW.md` under + `~/.codex/decodex/projects//` when the question is about validation, + tracker routing, or execution policy. +- Read `plugins/decodex/skills/decodex/SKILL.md` when the question is how an agent + should use Decodex in manual CLI mode or runtime-owned automation mode. - Then choose one primary lane: - `docs/spec/index.md` when the question is "what must be true?" - - `docs/guide/index.md` when the question is "what should I do?" -- Use `docs/plans/` only when a planning tool or execution workflow explicitly points to - a saved plan artifact there. + - `docs/runbook/index.md` when the question is "which sequence should I execute?" + - `docs/reference/index.md` when the question is "how is it currently organized or + implemented?" + - `docs/decisions/index.md` when the question is "why was it designed this way?" +- Use `docs/plans/` only when a planning tool or historical execution workflow + explicitly points to a saved plan artifact there. ## Routing matrix -- Need contracts, invariants, schemas, enums, state machines, or required behavior -> - `docs/spec/` -- Need runbooks, migrations, validation steps, troubleshooting, or operational sequences -> - `docs/guide/` +- Need runtime contracts, invariants, schemas, enums, state machines, or required + behavior -> `docs/spec/` +- Need public static-site contracts, GitHub bundle schemas, signal-entry schemas, or + release-delta schemas -> `docs/spec/` +- Need runbooks, migrations, validation steps, troubleshooting, or operational + sequences -> `docs/runbook/` +- Need current repository layout, ownership boundaries, static-site/runtime split, or + implementation surface maps -> `docs/reference/` +- Need durable design rationale, packaging choices, or static-site tradeoffs -> + `docs/decisions/` +- Need the raw machine-authored research run artifacts used by shipped research tooling + -> `docs/research/` +- Need reusable agent-facing Decodex usage instructions -> `plugins/decodex/` +- Need the GitHub signal editorial workflow -> `plugins/decodex/skills/github-signal/` + plus `docs/runbook/local-github-signal-workflow.md` +- Need repository execution defaults or tracker-state policy -> registered project + `WORKFLOW.md` - Need repo task names or automation entrypoints -> `Makefile.toml` -- Need documentation placement or authoring rules -> `docs/governance.md` -- Need a planning-tool artifact or saved execution plan -> `docs/plans/` +- Need historical saved execution plans from the original static-site bootstrap -> + `docs/plans/` ## Retrieval rules - Optimize for agent routing and execution, not narrative flow. +- Read `docs/policy.md` for lane ownership and authoring rules. - Keep one authoritative document per topic. Link instead of copying. +- Keep runtime authority explicit: `apps/decodex/src/`, registered project contracts + under `~/.codex/decodex/projects//`, and `docs/spec/` outrank runbook, + reference, and decision material. +- Keep the public site static by default. `site/` consumes checked-in content and + generated JSON; it must not depend on a live Decodex daemon unless a later decision + changes that boundary. - Start each document with a short routing header that says what the document is for, when to read it, and what it does not cover. - Keep links explicit and stable. -- Let structure emerge from real topics. Do not create empty folders, empty indexes, or - naming schemes that are stricter than the current corpus needs. +- Treat `docs/research/` and `docs/plans/` as supporting or historical evidence, not as + primary authority lanes. diff --git a/docs/policy.md b/docs/policy.md new file mode 100644 index 00000000..08ea1195 --- /dev/null +++ b/docs/policy.md @@ -0,0 +1,134 @@ +# Documentation Policy + +Purpose: Define the repository-wide documentation taxonomy, naming rules, and placement +rules for durable agent-facing content. + +Audience: All documentation under `docs/` is written for AI agents and LLM workflows. +The split below is by question type, not by reader type. + +## Primary taxonomy + +This repository standardizes on four primary documentation lanes: + +| Lane | Location | Answers | Holds | +| --- | --- | --- | --- | +| Spec | `docs/spec/` | What must be true? | Contracts, schemas, invariants, required behavior | +| Runbook | `docs/runbook/` | Which sequence should I execute? | Operational procedures, rollout steps, validation flows, recovery steps | +| Reference | `docs/reference/` | How is it currently organized or implemented? | Repository layout, surface maps, current implementation boundaries | +| Decisions | `docs/decisions/` | Why is it shaped this way? | Durable design choices, tradeoffs, and consequences | + +## Lane ownership + +- Each documentation lane owns exactly one question type. +- A lane may link to another lane's authority, but it must not restate that lane's + authoritative content. +- `spec` defines truth, not procedure, current state, or rationale. +- `runbook` defines procedure, not truth, current state, or rationale. +- `reference` defines current state, not truth, procedure, or rationale. +- `decisions` defines rationale, not truth, procedure, or current state. +- `research` is supporting evidence only. It does not own repository truth, procedure, + current state, or rationale. +- If a document starts answering a second question type, split it and link to the + authoritative lane instead of stretching one document across lanes. + +## Artifact lanes + +- `docs/research/` is allowed for machine-authored research run artifacts produced by the + shipped research tooling. +- `docs/research/` is not a primary documentation lane and is not authoritative for + runtime behavior, repository policy, or operator procedures. +- `docs/plans/` is allowed for historical saved plan artifacts from the original + static-site bootstrap. It is not a primary documentation lane and is not authoritative + for runtime behavior, repository policy, or operator procedures. +- If a research result becomes durable repository guidance, promote it into `spec`, + `runbook`, `reference`, or `decisions` and link back to the originating artifact only + as supporting evidence. + +## Placement rules + +- If a document defines correctness, it belongs in `docs/spec/`. +- If a document defines operator actions, it belongs in `docs/runbook/`. +- If a document describes current structure, ownership, or implementation boundaries, it + belongs in `docs/reference/`. +- If a document records durable rationale or tradeoffs, it belongs in `docs/decisions/`. +- Do not duplicate authoritative content across lanes. Link to the source of truth. + +## Authoring rules + +- Write for AI agents and LLM execution, not for narrative human reading flow. +- Optimize for retrieval, routing, and exact execution over prose style or rhetorical + smoothness. +- Use stable terms for the same concept. Do not drift between synonyms for important + state names, commands, files, roles, or surfaces. +- Prefer short declarative bullets, tables, and headers over long mixed-purpose prose. +- Put authority, scope, inputs, outputs, and non-goals near the top of the document when + they matter. +- Keep commands, paths, state names, labels, and config keys explicit and literal. +- Keep one authoritative document per topic. Other documents should link to it rather + than paraphrasing it. +- If human readability and machine routing conflict, prefer the machine-readable form. + +## Naming rules + +- Directory names express document type. +- File names express stable topic. +- Use lowercase kebab-case for document file names. +- Do not encode temporary versions such as `v0`, `v1`, or `draft2` into stable file + names. +- Do not repeat the directory class in the file name when the topic is already clear. + Prefer `runtime.md` under `docs/spec/` over `runtime-spec.md`. + +## Document headers + +Every document should start with a short routing header. + +Spec header: + +- `Purpose` +- `Status: normative` +- `Read this when` +- `Not this document` +- `Defines` + +Runbook header: + +- `Goal` +- `Read this when` +- `Preconditions` or `Inputs` +- `Depends on` +- `Verification` or `Outputs` + +Reference header: + +- `Purpose` +- `Read this when` +- `Not this document` +- `Covers` + +Decision header: + +- `Status` +- `Date` +- `Question` +- `Decision` +- `Consequences` + +## Canonical entry points + +- Unified router: `docs/index.md` +- Normative router: `docs/spec/index.md` +- Procedural router: `docs/runbook/index.md` +- Current-state router: `docs/reference/index.md` +- Rationale router: `docs/decisions/index.md` +- Repo task and automation entrypoints: `Makefile.toml` + +## Update workflow + +- Behavior or schema change: update the relevant spec. +- Procedure change: update the relevant runbook. +- Structural or ownership change: update the relevant reference doc. +- Durable design or packaging change: update the relevant decision doc. +- If a document drifts across lanes, split it instead of stretching one document to do + several jobs. +- If a document repeats another lane's authority, remove the duplicate text and replace + it with a link to the source of truth. diff --git a/docs/reference/github-operations.md b/docs/reference/github-operations.md new file mode 100644 index 00000000..4fe6682c --- /dev/null +++ b/docs/reference/github-operations.md @@ -0,0 +1,49 @@ +# GitHub Operations + +Purpose: Map Decodex's current GitHub-facing execution surface and the decision for +each area to use `gh`, keep a custom model, or avoid GitHub ownership. + +Read this when: You are changing landing, PR inspection, review orchestration, +review handoff validation, or retained-lane cleanup. + +Not this document: The runtime state machine, tracker tool contract, operator pilot +procedure, or merge policy. + +Covers: Current GitHub operation ownership, keep-vs-replace decisions, and follow-up +criteria for future simplification. + +## Decision Table + +| Area | Current authority | Decision | Reason | +| --- | --- | --- | --- | +| Manual admin merge and retained clean-path admin merge | `gh pr merge --admin --merge --match-head-commit` in `apps/decodex/src/github.rs` | Already `gh`-backed | `gh` owns admin merge semantics and head matching without Decodex hand-assembling a merge mutation. Manual landing keeps its existing manual gate policy; retained runtime landing calls admin merge only when the PR is already on the deterministic clean path. | +| Repository context inspection | `gh repo view --json name,owner,defaultBranchRef,mergeCommitAllowed` in `apps/decodex/src/github.rs` | Already `gh`-backed | The CLI exposes the required repository fields directly. | +| Review handoff PR validation | `gh pr view --json url,baseRefName,headRefName,headRefOid,state,isDraft,headRepository,headRepositoryOwner` in `apps/decodex/src/agent/tracker_tool_bridge.rs` | Already `gh`-backed | The CLI covers the branch, head, base, repository, draft, and state checks needed before accepting `issue_review_handoff`. | +| Post-merge result inspection | `gh pr view --json state,headRefOid,mergeCommit` in `apps/decodex/src/github.rs` | Already `gh`-backed | Decodex needs only merged state, reviewed head identity, and merge commit OID after merge. | +| Merge commit subject inspection | `gh api repos///commits/` in `apps/decodex/src/github.rs` | Keep `gh api` | Decodex needs the authoritative landed `decodex/commit/1` subject from the merge commit. `gh pr view` does not provide that subject in the current model. | +| External review request comments | `gh api repos///issues//comments` in `apps/decodex/src/github.rs` | Keep `gh api` | Decodex persists the created issue-comment database id and timestamp so later review orchestration can match acknowledgements and results precisely. | +| Landing gate state | `gh api graphql` in `apps/decodex/src/github.rs` | Keep custom GraphQL query through `gh` | Decodex needs mergeability, merge-state, review decision, pending review requests, status rollup, and paginated unresolved review-thread counts before merge. | +| Retained review state | `gh api graphql` in `apps/decodex/src/orchestrator/pull_request_review.rs` | Keep custom GraphQL query through `gh` | Runtime review orchestration needs paginated issue comments, reactions by actor, reviews, review threads, merge evidence, and head repository metadata in one stable state model. | +| Remote lane branch cleanup | `gh api --method DELETE repos///git/refs/heads/` in `apps/decodex/src/github.rs` | Replaced custom git plumbing | Decodex only needs idempotent GitHub ref deletion. `gh` removes the prior `git ls-remote` plus `git push --delete` path and the extra askpass helper just for cleanup. | +| Default branch sync and local branch/worktree cleanup | Git commands in `apps/decodex/src/default_branch_sync.rs` and `apps/decodex/src/orchestrator/git_ops.rs` | Keep local Git | These steps mutate or inspect the local repository/worktree state. `gh` does not replace the required local checkout synchronization and linked-worktree cleanup. | + +## Replacement Criteria + +Prefer `gh` for GitHub operations when all are true: + +- `gh` exposes the exact required semantics without weakening Decodex policy. +- Decodex does not need to persist API-only identifiers that the plain CLI command hides. +- The operation is scoped to GitHub state rather than local worktree or shared Git + administrative state. +- Failure handling can stay fail-closed and noninteractive with the configured + `github.token_env_var`. + +Keep a custom `gh api` or GraphQL model when Decodex needs stable structured fields, +pagination, actor-scoped reactions, review-thread state, or an idempotency record that +the higher-level `gh` command does not expose. + +## Implemented Follow-Up + +Remote branch cleanup now uses `gh api --method DELETE` against the PR repository ref. +Missing refs are treated as idempotent cleanup success; other `gh` failures preserve the +retained worktree and runtime mapping so cleanup can retry or surface operator attention. diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..d8cf25ff --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,33 @@ +# Reference Index + +Purpose: Route agents to descriptive documents that explain the repository's current +structure and implementation surfaces. + +Question this index answers: "how is it currently organized or implemented?" + +## Use this index when + +- You need the current repository layout, ownership boundaries, or where a topic lives. +- You need to know which directory or file surface is authoritative for a class of work. +- You need to understand the role of machine-authored artifacts such as `docs/research/`. + +## Do not use this index when + +- You need a normative contract. +- You need an execution sequence or operator runbook. +- You need durable rationale for why a design choice exists. + +## Current reference docs + +- [`operator-control-plane.md`](./operator-control-plane.md) for the current + single-machine control-plane shape, operator dashboard sections, local-vs-external + state boundary, and deferred operator directions. +- [`github-operations.md`](./github-operations.md) for the current keep-vs-replace map + for `gh`-backed GitHub operations, custom `gh api`/GraphQL reads, and local Git + cleanup boundaries. +- [`test-suite.md`](./test-suite.md) for the current test inventory, behavior grouping, + and keep/merge/delete standards. +- [`workspace-layout.md`](./workspace-layout.md) for the repository surface map and + directory ownership boundaries, including the canonical Decodex plugin source. +- [`research-runs.md`](./research-runs.md) for the role and limits of `docs/research/` + artifacts. diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md new file mode 100644 index 00000000..b938dc01 --- /dev/null +++ b/docs/reference/operator-control-plane.md @@ -0,0 +1,243 @@ +# Operator Control Plane + +Purpose: Describe the current single-machine Decodex control plane, operator +dashboard sections, and state ownership boundaries. + +Read this when: You need to understand what the operator UI is showing, where +runtime truth lives, or which planned operator features are intentionally not part of +the current implementation. + +Not this document: The normative runtime state machine, Linear event schema, pilot +procedure, UI styling rules, or durable design rationale. + +Covers: The local control-plane surfaces, project registry, dashboard information +architecture, Linear/GitHub boundary, and deferred operator directions. + +## Current Shape + +Decodex currently runs as a local, single-machine control plane: + +- `decodex serve` is the long-running operator entrypoint. +- One global runtime database lives at `~/.codex/decodex/runtime.sqlite3`. +- Project contracts live under `~/.codex/decodex/projects//`, not inside + target repositories. +- Each project directory uses fixed filenames: `project.toml` for service paths and + credentials, plus `WORKFLOW.md` for execution policy. +- Projects are registered explicitly with `decodex project add `. +- `decodex serve` does not scan `.codex` history, repo-local config files, or + open worktrees to infer projects. +- Each project row is scoped by `project_id` and canonical `repo_root`. +- The project-owned `WORKFLOW.md` remains the execution-policy contract for that + registered repo. + +Project registration is not service intake. `Projects` may show multiple enabled +projects at once, but a service is only eligible to intake Linear issues labeled with +its matching `decodex:queued:` label. For example, a Decodex-only run +intakes issues labeled `decodex:queued:decodex`; `rsnap` can stay enabled in +`Projects`, and issues labeled `decodex:queued:rsnap` remain rsnap intake rather than +Decodex intake. The pilot runbook owns enqueue and run steps. + +When `decodex run --dry-run` or the status output has no eligible intake candidate, +the operator hint points to the short checklist: `Todo`, the service-scoped +`decodex:queued:` label, no opt-out or needs-attention labels, a +non-terminal state, no open dependency blockers, and available local capacity. + +The runtime database is the local source of truth for active execution. Linear and +GitHub remain external collaboration mirrors and validation surfaces. + +## State Ownership + +| Surface | Owns | Does Not Own | +| --- | --- | --- | +| Runtime SQLite DB | active leases, attempts, protocol events, worktree mappings, retry state, retained PR state, phase timing, connector backoff, project registry | human backlog grooming or durable team-visible issue history | +| Central project config | `service_id`, repo root, worktree root, tracker/GitHub credential env-var names, enabled project registration | per-run state or issue ownership | +| Project `WORKFLOW.md` | repo policy, validation gate, state names, retry/review policy | runtime ownership, queue labels, credentials, model overrides | +| Linear | team-visible issue state, queue/active/manual-attention labels, coarse execution ledger comments, progress/failure/handoff/closeout summaries | high-frequency runtime truth, heartbeat, token pressure, raw attempts, connector retry budgets | +| GitHub | PR, checks, review comments, merge evidence, signed commit verification | queue selection or local lane ownership | +| `.decodex-run-activity` | short-lived child activity heartbeat for the active attempt | durable ownership, review handoff identity, cleanup authority | + +## Operator Dashboard Sections + +The browser dashboard is a read-only view over the same operator snapshot served at +`GET /state`. + +The dashboard header also shows the browser origin being viewed and, when the +served state response carries a publish timestamp, the relative age of that +snapshot so an operator can catch stale tabs or an old listener port quickly. + +Because the runtime SQLite DB is authoritative, dashboard sections describe current +runtime ownership before local directory presence. An existing `.worktrees/XY-*` +directory does not, by itself, mean an active lane is still running; the owning +section says whether the path belongs to an active lease, retained review/landing +lane, queued attention state, or cleanup/recovery inbox. + +`Projects` is the registered-project overview for this local installation. Its rows +come from project registrations and per-project runtime snapshot state stored in +`~/.codex/decodex/runtime.sqlite3`; it is not a repository discovery scan, Codex +conversation-history scan, or repo-local config search. If the view is empty, +Decodex has no registered project rows to summarize yet. If it is idle or +capacity-waiting, the registered projects exist but currently have no local lane ready +to run or are waiting on local capacity. Neither state is, by itself, evidence that +the Linear tracker or GitHub connector failed; confirm the central project registry +and service queue label before treating it as a connector problem. + +The browser dashboard reads the complete published state from `GET /state` and may +also keep a local WebSocket open at `GET /dashboard/control`. `/state` remains the +authoritative reconciliation snapshot; the WebSocket pushes Decodex-owned snapshot +and active-lane activity updates sooner than the polling interval, and accepts a +small local dashboard control protocol. Browser-originated `subscribe` / `focus` +messages scope live active-lane updates for that socket. `pauseProject` and +`resumeProject` use the existing local project registry enabled flag, and `retryRun` +starts the existing local `decodex run ` path for an explicit operator retry. +`ack` is dashboard-local acknowledgement only. The socket is not a browser connection +to Codex app-server, GitHub, or Linear, and it does not make high-frequency protocol +activity durable outside the local operator surface. + +| Section | Meaning | +| --- | --- | +| `Projects` | Fleet-level project rows. This is the multi-project overview: enabled state, health, connector state, capacity, attention count, retained local worktree count, and last activity. It should not duplicate per-lane details already shown below. | +| `Running Lanes` | Active leased or live-executing issue lanes. A lane here is currently owned by this local control plane, or a live process/thread/protocol marker still explains active execution even when the queue lease is not held. It shows issue identity, phase, operation, attempt, queue lease state, execution liveness, thread/protocol status, child-agent activity when captured, timing, branch, and worktree. | +| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after retries, the candidate is partial retained progress and not just a generic retry-budget hold. Running lanes are not repeated as normal intake work. | +| `Review & Landing` | Retained PR lanes after review handoff. This section owns post-review repair, wait-for-review, ready-to-land, closeout, cleanup, and blocked retained-lane visibility. | +| `Recovery Worktrees` | Retained local worktrees that are not currently owned by `Running Lanes`, `Review & Landing`, or queued attention in `Intake Queue`. This is the cleanup or recovery inbox for recovered paths, retained PR leftovers, and cleanup-only local worktrees. Empty is the normal healthy state. | +| `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Raw local attempts and heartbeat details stay in debug expansion. | + +Worktree visibility follows the owning dashboard section: + +- `Running Lanes` means the runtime DB still has an active lease, active attempt, or + child process/thread/protocol relationship for the path. `active_lease` is queue + lease ownership only; `execution_liveness` explains why the lane is still visible + when the queue lease is not held. +- Running lanes derive CLI and dashboard text from the same `OperatorRunStatus` + object. `protocol_activity`, when present, summarizes app-server structured + notifications for turn status, waiting reason, rate-limit status, and recent + protocol events. The dashboard uses that shared summary to explain whether active + time is going to model execution, tools, approval/user input, or protocol idleness. + These high-frequency details remain local/operator-only and are not written to + Linear except through existing lifecycle summaries. +- Dynamic tool failures appear in local protocol activity as + `item/tool/call/failure` with a normalized failure class and next action. Invalid + or undeclared app-server tool requests are protocol failures; declared Decodex + tools that return `success = false` remain tool failures the model can correct + within the same turn. +- `Review & Landing` means a retained PR lane still owns the path for review repair, + landing, closeout, or retained-lane cleanup. +- `Intake Queue` means queued attention still owns the path, including partial retained + progress after retries. +- `Recovery Worktrees` means the path is retained local state after the authoritative + runtime owner is gone or cannot explain it as active, review/landing, or queued + work. + +Every `/state` worktree row includes an `ownership` and `ownership_reason` that +distinguishes active-lane ownership, post-review ownership, queued attention, and +cleanup-only local retention. A `Recovery Worktrees` row tells the operator to inspect +the local path and either clean it up or recover local-only changes; it is not, by +itself, evidence that the SQLite runtime store lost an active lane. When the tracker +issue is already `Done` and no retained lane owns the worktree, the row is neutral +cleanup-only state, not a blocking recovery error. + +When a retained worktree reports `role: cleanup_only`, treat it as local cleanup +hygiene rather than an active lane. It does not imply that an agent, child +process, post-review repair, closeout, or queued recovery run is still executing, +and it is not queue pressure or a hidden capacity claim. The row only says local +disk still has a retained checkout after the runtime owner is gone; once the +operator verifies the issue or PR is terminal, `main` contains the intended work, +and the checkout has no local-only changes that need recovery, the safe action is +to remove that local worktree. + +The expected operator path for a cleanup-only row is short: + +1. Verify the tracker issue and any associated PR are merged, done, or otherwise + terminal, and confirm the same worktree is absent from `Running Lanes`, + `Intake Queue`, and `Review & Landing`. +2. Inspect the local checkout before deletion, such as with + `git -C status --short`. Tracked edits (`M`, `A`, `D`, `R`, and + similar status output) mean the row is not safe to auto-delete until those + changes are intentionally preserved, recovered, or discarded. +3. Clean the worktree only after the terminal state and local changes are + understood; otherwise preserve it as local retention for manual recovery. + +If the same worktree is owned by `Review & Landing`, follow the retained +post-review lane state instead; if it is attached to a queued candidate in +`Intake Queue`, treat it as queued attention or partial retained progress rather +than cleanup-only local retention. + +Closeout has a short tracker/local ordering window. A `Closeout` child may observe +the tracker issue as `Done` while it is still finishing local cleanup; while the +child, retained lane, or activity heartbeat still owns that closeout, the control +plane treats it as in-flight closeout/cleanup, not a terminal stale lane. + +The UI should answer three operator questions first: + +- What is running right now? +- What needs operator attention? +- What finished, landed, or needs cleanup? + +It should not expose internal object lists as primary navigation when those lists do not +map to an operator decision. + +## Liveness And Timing + +`Running Lanes` and `Run Ledger` expose timing at two different levels: + +- Lane/run timing comes from runtime attempt rows, process status, and persisted + snapshot fields. +- Queue ownership and execution liveness are separate. `queue_lease_state` reports + whether the local queue lease is held, while `execution_liveness` reports observed + process, app-server thread, or protocol activity. +- `status` is the operator-facing run status. If the raw attempt is still `starting` + after app-server thread, model, or protocol evidence exists, `status` is shown as + `running` and `attempt_status` preserves the raw persisted attempt value. +- Child-agent activity comes from `.decodex-run-activity` when the app-server recorder + captured model/tool/tracker/browser/image buckets. +- The child-agent breakdown is diagnostic. It explains where observed wall time went; + it is not a scheduler contract. +- Missing child-agent activity means no breakdown was captured for that run, not that + the lane is invalid. + +The dashboard should avoid pretending that every bucket has a fixed total budget. +When a row is event-only or sub-second, the UI should present it as diagnostic event +activity rather than a misleading progress bar. + +## Linear And Connector Behavior + +Decodex should keep publishing a local operator snapshot when Linear or GitHub is slow, +rate-limited, or unavailable. + +- Connector failures should appear as typed health/backoff state, not raw API error + blobs in the main layout. +- When a tracker connector enters backoff, `/state` includes a `connector_backoffs` + entry with the affected `project_id`, `connector`, `sync_phase`, `quota_class`, + `reset_at`, `reset_unix_epoch`, `retry_after_seconds`, and operator `next_action`. + Running lanes should still render from local runtime DB state while external sync + is paused. +- Linear writes should stay coarse: start, progress checkpoint, PR-ready/handoff, + blocked/failed, landed, done, and cleanup summaries. +- Fine-grained retry budgets, raw attempts, heartbeat, child buckets, token pressure, + and recovery details stay local. +- Completed lanes without Decodex Linear execution ledger records are reported as + `missing` / `execution_ledger_missing`. Tracker terminal state, local attempt + success, and non-ledger comments never satisfy the Run Ledger outcome contract. + +## Current Non-Goals + +These directions were discussed but are not part of the current implemented contract: + +- Conflict-domain scheduling for `ui-preview`, `docs`, `tests`, `runtime`, or similar + lane classes. +- Demo batch planning that automatically selects two or three small visible issues and + generates operator observation points. +- Editing project configuration from the operator UI. +- Inferring registered projects by scanning `.codex` history or repository-local config + files. +- Treating Linear comments as the real-time runtime backend. + +If any of these become implementation work, promote the chosen behavior into the +governing spec first, then update the operator runbook and this reference. + +## Authority Links + +- Runtime contract: [`../spec/runtime.md`](../spec/runtime.md) +- Linear execution ledger schema: [`../spec/linear-execution-ledger.md`](../spec/linear-execution-ledger.md) +- Pilot procedure: [`../runbook/self-dogfood-pilot.md`](../runbook/self-dogfood-pilot.md) +- Workspace layout: [`./workspace-layout.md`](./workspace-layout.md) diff --git a/docs/reference/research-runs.md b/docs/reference/research-runs.md new file mode 100644 index 00000000..dd120a78 --- /dev/null +++ b/docs/reference/research-runs.md @@ -0,0 +1,39 @@ +# Research Runs + +Purpose: Explain the role of `docs/research/` artifacts in this repository and how they +relate to the primary documentation taxonomy. + +Read this when: You encounter `docs/research/.json` files and need to know +whether they are authoritative documentation, generated artifacts, or supporting +evidence. + +Not this document: The research method itself, the runtime contract, or a design +decision record. + +Covers: Artifact placement, authority boundaries, and promotion rules for research +results. + +## Status of `docs/research/` + +- `docs/research/` is the persistence root for the shipped research tooling. +- Files in `docs/research/` are machine-authored run artifacts, not primary + documentation lanes. +- A research run may contain useful evidence, alternatives, and objections, but it does + not by itself define repository truth. + +## Promotion rules + +- If a research result defines required behavior, promote the conclusion into + `docs/spec/`. +- If a research result defines an operator sequence, promote the conclusion into + `docs/runbook/`. +- If a research result explains current structure, promote the conclusion into + `docs/reference/`. +- If a research result records a durable tradeoff or design choice, promote the + conclusion into `docs/decisions/`. + +## Practical reading rule + +- Read `docs/research/` when you need the original evidence trail. +- Read one of the four primary documentation lanes when you need current repository + guidance. diff --git a/docs/reference/test-suite.md b/docs/reference/test-suite.md new file mode 100644 index 00000000..9fdbb6af --- /dev/null +++ b/docs/reference/test-suite.md @@ -0,0 +1,162 @@ +# Test Suite + +Purpose: Map the current test suite by behavior surface so pruning, additions, and +debugging start from a shared inventory. + +Read this when: You need to decide where a Decodex test belongs, whether two tests can +be merged, or which dense test matrix is intentionally retained. + +Not this document: Runtime truth, operator procedure, or the command authority for +repository checks. + +Covers: Current test inventory, grouping rules, high-density test surfaces, and the +standards for keeping, merging, or deleting tests. + +## Current Snapshot + +This cleanup keeps 685 `nextest` tests plus one ignored live app-server test. Regenerate +the runnable inventory with: + +```sh +cargo nextest list --workspace --all-targets --all-features +``` + +Regenerate the top-level grouping with: + +```sh +cargo nextest list --workspace --all-targets --all-features 2>/dev/null \ + | awk '{print $2}' \ + | sed 's/::[^:]*$//' \ + | sort \ + | uniq -c \ + | sort -nr +``` + +## Primary Groups + +| Group | Count | Primary surfaces | Owns | +| --- | ---: | --- | --- | +| Orchestrator | 359 | `apps/decodex/src/orchestrator/tests.rs`, `apps/decodex/src/orchestrator/tests/**/*.rs` | Intake, retry, review/landing, runtime cleanup, operator status, repo gates | +| Tracker tool bridge | 85 | `apps/decodex/src/agent/tracker_tool_bridge/tests.rs`, `apps/decodex/src/agent/tracker_tool_bridge/tests/**/*.rs` | Dynamic tracker tools, continuation guards, review handoff writes, closeout writes | +| App-server protocol/runtime | 38 | `apps/decodex/src/agent/app_server/tests.rs`, `apps/decodex/src/agent/json_rpc.rs`, app-server protocol tests | JSON-RPC parsing, turn execution, dynamic tools, thread config, transport failures | +| Runtime state and locks | 39 | `state::tests`, `runtime::tests` | Persistent local state, lock ownership, runtime database contracts | +| Workflow and config parsing | 53 | `workflow::tests`, `config::tests` | `WORKFLOW.md`, project config, removed-field rejection, default policy | +| Git, worktree, and landing helpers | 93 | `worktree::tests`, `manual::tests`, `commit_message::tests`, `github::tests`, `default_branch_sync::tests`, `pull_request::tests` | Git/worktree behavior, manual landing, GitHub/PR helpers, commit-message policy | +| CLI, archive, and tracker integration | 18 | `cli::tests`, `archive_hygiene::tests`, `tracker::linear::tests` | User-facing commands, archive hygiene, direct Linear adapter behavior | + +## Orchestrator Inventory + +The orchestrator suite is intentionally split by lifecycle stage. Do not add another +large catch-all test file unless the behavior crosses several of these stages. + +| File | Count | Group | +| --- | ---: | --- | +| `apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs` | 4 | Workflow reload and cached policy snapshots | +| `apps/decodex/src/orchestrator/tests/intake/eligibility.rs` | 7 | Intake eligibility and queue label safety | +| `apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs` | 38 | Prompt construction, machine-only redaction, run setup | +| `apps/decodex/src/orchestrator/tests/intake/prepare_issue_run.rs` | 10 | Worktree preparation and pre-run guards | +| `apps/decodex/src/orchestrator/tests/intake/candidate_selection.rs` | 24 | Candidate ordering, retained lane preference, closeout dispatch policy | +| `apps/decodex/src/orchestrator/tests/retry/scheduling.rs` | 28 | Retry timing, dry-run behavior, retry marker semantics | +| `apps/decodex/src/orchestrator/tests/retry/selection.rs` | 16 | Retry queue selection and blocked retry candidates | +| `apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs` | 8 | Repo gate command selection, cleanliness, shell fallback, and failure classification | +| `apps/decodex/src/orchestrator/tests/runtime/failure.rs` | 35 | Failure comments, runtime credentials, cleanup, lease release | +| `apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs` | 18 | Stale lease, recovery worktree, and reconciliation behavior | +| `apps/decodex/src/orchestrator/tests/recovery/terminal_support.rs` | 0 | Shared retained recovery and closeout fixtures | +| `apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs` | 4 | Direct closeout dispatch and PR validation | +| `apps/decodex/src/orchestrator/tests/recovery/closeout/identity.rs` | 6 | Closeout identity reuse after retained runs | +| `apps/decodex/src/orchestrator/tests/recovery/closeout/cleanup.rs` | 6 | Retained closeout cleanup and cleanup blockers | +| `apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs` | 8 | Terminal failure labeling and nonretryable attention | +| `apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs` | 25 | Runtime reentry, recovered worktrees, liveness, and live-run recovery | +| `apps/decodex/src/orchestrator/tests/operator/status_support.rs` | 0 | Shared operator status fixtures | +| `apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs` | 3 | Registered project control-plane rows | +| `apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs` | 22 | Running lanes, stalled lanes, active-run hydration, and local worktrees | +| `apps/decodex/src/orchestrator/tests/operator/status/history.rs` | 4 | Run ledger and Linear history hydration | +| `apps/decodex/src/orchestrator/tests/operator/status/text.rs` | 4 | Human-readable operator status text | +| `apps/decodex/src/orchestrator/tests/operator/status/publishing.rs` | 6 | Snapshot publishing, degraded observers, and tracker backoff | +| `apps/decodex/src/orchestrator/tests/operator/status/queue.rs` | 8 | Intake queue classifications and shared-claim visibility | +| `apps/decodex/src/orchestrator/tests/operator/status/http.rs` | 10 | Operator `/state`, `/livez`, readiness, and dashboard route responses | +| `apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs` | 3 | Dashboard client rendering contracts | +| `apps/decodex/src/orchestrator/tests/review_landing/status_support.rs` | 0 | Shared Review & Landing status fixtures | +| `apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs` | 18 | Review & Landing status rows and handoff lineage | +| `apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs` | 12 | Review orchestration, admin merge, and repair routing | +| `apps/decodex/src/orchestrator/tests/review_landing/status_markers.rs` | 2 | Review orchestration marker handling and recovered targeted visibility | +| `apps/decodex/src/orchestrator/tests/review_landing/classification_review.rs` | 13 | Review repair, request-pending, stale handoff, merged PR classification | +| `apps/decodex/src/orchestrator/tests/review_landing/classification_checks.rs` | 15 | Required checks, GitHub token gates, GraphQL pagination/query shape | +| `apps/decodex/src/orchestrator/tests/review_landing/review_state.rs` | 2 | Pull-request review-state conversion from GitHub GraphQL nodes | + +## Tracker Bridge Inventory + +| File | Count | Group | +| --- | ---: | --- | +| `apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/dispatch.rs` | 22 | Tool argument validation, state transitions, label mutations, closeout dispatch | +| `apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/continuation.rs` | 13 | Continuation-blocking writes and reactivation safety | +| `apps/decodex/src/agent/tracker_tool_bridge/tests/mutation/progress.rs` | 5 | Progress checkpoint comments and worktree path handling | +| `apps/decodex/src/agent/tracker_tool_bridge/tests/review/policy.rs` | 22 | Internal-review stop policy, repair/writeback behavior, checkpoint handling | +| `apps/decodex/src/agent/tracker_tool_bridge/tests/review/handoff.rs` | 23 | Review handoff, repair complete, terminal finalize, closeout complete | + +## Keep Standards + +Keep separate tests when the case protects a different observable contract: + +- Different public surface, such as CLI output, operator status JSON, tracker comments, + Git commands, runtime database state, or app-server protocol payloads. +- Different state-machine outcome, especially blocked versus ineligible, retryable versus + terminal, repair versus closeout, or queued versus retained. +- Different persisted marker semantics, such as review handoff lineage, retry schedule + marker, cleanup handoff marker, or closeout identity reuse. +- Different authority boundary, such as GitHub token routing, Linear tracker writes, + repo-local Git config, or runtime-only state. +- Different process or concurrency boundary, such as active child reconciliation, lock + contention, stale lease cleanup, or app-server transport failure. + +## Merge Standards + +Prefer table-driven tests when all cases exercise the same branch and assert the same +observable contract: + +- Same tool rejects several invalid argument spellings. +- Same policy accepts or rejects multiple equivalent labels, transitions, or field names. +- Same redaction or prompt rule varies only fixture text. +- Same pagination guard varies only which stable metadata field changed. +- Same missing-configuration rule varies only absent versus blank environment values. + +The merged test name should describe the behavior contract, not the fixture shape. + +## Delete Standards + +Delete a test only when another remaining test is a strict behavioral superset: + +- Same entrypoint. +- Same state setup except irrelevant spelling. +- Same branch or failure class. +- Same externally visible assertion. + +Do not delete a test only because its fixture looks similar. Similar fixtures often +protect different contracts in retained review lanes, status rows, and cleanup flows. + +## Intentionally Dense Areas + +These areas should stay dense unless the implementation contract changes: + +- `operator/status/` covers operator-facing JSON, text, dashboard, `/livez`, and + readiness behavior. These tests are noisy but protect the local control-plane surface. +- `review_landing/status_rows.rs` keeps both descendant handoff and lineage-rewrite cases + because the production branch distinguishes accepted ancestry from rejected rewrites. +- `intake/candidate_selection.rs` keeps planner, dispatch policy, and block-reason cases + separate when they expose different operator or API outcomes. +- `retry/scheduling.rs` keeps retry and review-repair failure modes separate when the + lane identity changes the marker or operator-facing reason. +- `recovery/closeout/` and `recovery/runtime_reentry.rs` keep closeout identity, + cleanup handoff, and retained worktree + recovery matrices separate because they preserve different recovery authority. + +## Placement Rules + +- Add orchestrator tests to the lifecycle subdirectory that owns the behavior surface + above. +- Add tracker bridge tests to `mutation/` or `review/`, not the shared harness. +- Add a new file only when a behavior family has several tests and no existing group owns + its lifecycle stage. +- Convert same-contract variants into table rows before adding sibling tests. +- Update this document when a new test family is created or when a dense matrix is + intentionally collapsed. diff --git a/docs/reference/workspace-layout.md b/docs/reference/workspace-layout.md new file mode 100644 index 00000000..e8a9902c --- /dev/null +++ b/docs/reference/workspace-layout.md @@ -0,0 +1,130 @@ +# Workspace Layout + +Purpose: Describe the current top-level repository surfaces and which concerns each one +owns. + +Read this when: You need to know where runtime code, static-site code, workflow policy, +GitHub signal tooling, and documentation topics currently live. + +Not this document: The normative behavior contract, operator procedures, or durable +design rationale. + +Covers: The repository surface map, ownership boundaries, and local directories that +should not be treated as repository source. + +## Top-level surfaces + +| Path | Role | +| --- | --- | +| `apps/decodex/` | Rust package that builds the `decodex` CLI and runtime. Runtime, orchestration, tracker integration, app-server integration, operator HTTP, and local control-plane behavior live under `apps/decodex/src/`. | +| `site/` | Astro static site for the public Decodex signal surface. It renders checked-in content and generated JSON from `site/src/content/`; it is not backed by a live Decodex daemon. | +| `tools/github/` | Deterministic GitHub collection, normalization, render, and validation scripts for public signal content. | +| `plugins/decodex/` | Canonical installable Decodex plugin source and reusable agent-facing skills, including manual CLI, automation, commit, land, labels, and GitHub signal drafting. | +| `docs/spec/` | Normative runtime, workflow, site, and content contracts. | +| `docs/runbook/` | Operator procedures, validation sequences, deployment steps, and content workflows. | +| `docs/reference/` | Current repository and artifact surface maps. | +| `docs/decisions/` | Durable rationale for repository-level design choices. | +| `docs/research/` | Machine-authored research run artifacts used by shipped research tooling. | +| `docs/plans/` | Historical saved plan artifacts from the static-site bootstrap. These are not primary authority. | +| `scripts/` | Repository-level helper scripts that are not part of the Rust runtime binary. | +| `dev/` | Local development helpers such as the operator dashboard mock server. | +| `assets/` | Shared static assets that are not owned by the Astro app's generated output. | +| `.github/` | CI, release, Pages deployment, and content-refresh workflows. | +| `Makefile.toml` | Repo-native task names and automation entrypoints. | +| `decodex.example.toml` | Redacted template for a project `project.toml`; live project contracts live under `~/.codex/decodex/projects//`. | + +## Rust workspace + +The root `Cargo.toml` is a workspace manifest. It does not define a root package. + +`apps/decodex/Cargo.toml` is the only checked-in Rust package in this first integrated +layout. Use package-qualified commands when invoking the runtime from the workspace root: + +```sh +cargo run -p decodex -- --help +cargo build -p decodex +cargo install --path apps/decodex --force +``` + +Do not add new runtime behavior to a root `src/` directory. If Decodex later needs +shared crates, add them under `packages/` and make the boundary explicit in this +reference document and the root workspace manifest. + +## Static public site + +`site/` remains the public, static Decodex surface. It owns: + +- homepage and feed rendering +- signal cards and release-delta presentation +- checked-in content collections under `site/src/content/` +- Astro build and type-check behavior + +The site does not own: + +- Decodex runtime scheduling +- local operator state +- tracker writes +- app-server orchestration +- live operator dashboard behavior + +Those runtime and operator surfaces stay in `apps/decodex/` and `docs/spec/`. + +## GitHub signal tooling + +`tools/github/` owns deterministic content tooling. It may call Codex for the editorial +drafting step through the plugin skill at `plugins/decodex/skills/github-signal/`, but +the scripts must keep generated artifacts explicit and checked into the repository. + +## Installable Codex surface + +The installable Codex home surface, including `~/.codex/AGENTS.md`, is not a Decodex +runtime contract and is not tracked in this repository. Do not reintroduce checked-in +`.codex/AGENTS.md` content here to carry Decodex-specific policy. + +Global agent guidance should stay portable. Decodex-specific runtime, workflow, +identity, review, landing, closeout, and cleanup policy belongs in `apps/decodex/src/`, +`docs/spec/`, the registered project `WORKFLOW.md`, project `project.toml`, runbooks, +or the Decodex plugin skill that owns a reusable method. The normative split is defined +by [`../spec/installable-agent-policy.md`](../spec/installable-agent-policy.md). + +## Local Decodex home + +Runtime state that belongs to the local operator, not to this repository, lives under +`~/.codex/decodex/`: + +- `runtime.sqlite3` is the single-machine control-plane database for all registered + projects. +- `logs/` stores Decodex process logs. +- `projects//project.toml` stores the central service config for one + registered project. +- `projects//WORKFLOW.md` stores that project's execution policy. +- Project discovery comes from explicit registration, not from scanning Codex history + or repo-local config files. + +This local control-plane state chooses registered projects. Once a checkout is selected, +the matching project directory's `WORKFLOW.md` remains the execution contract for gates, +tracker routing, and policy. + +## Boundary notes + +- Runtime authority stays in `apps/decodex/src/`, the registered project contract under + `~/.codex/decodex/projects//`, and the governing specs under + `docs/spec/`. +- Public site authority stays in `site/`, `tools/github/`, and the site/content specs. +- Reusable agent-facing Decodex usage instructions live under `plugins/decodex/`. +- `docs/runbook/`, `docs/reference/`, and `docs/decisions/` must not override runtime or + workflow authority. +- `docs/research/` and `docs/plans/` are supporting evidence only. They do not become + policy until their conclusions are promoted into governing docs. + +## Local-only and generated directories + +These paths are intentionally ignored and should not be treated as tracked repository +structure: + +- `target/`: Rust build products and local analysis artifacts +- `site/dist/`: Astro build output +- `site/.astro/`: Astro local cache +- `.worktrees/`: local Git worktree lanes +- `.workspaces/`: local clone-backed workspace lanes from older workflows +- `.codex/`: local agent/runtime state diff --git a/docs/research/2026-03-31_decodex-plan-boundary-v2.json b/docs/research/2026-03-31_decodex-plan-boundary-v2.json new file mode 100644 index 00000000..f312ce1e --- /dev/null +++ b/docs/research/2026-03-31_decodex-plan-boundary-v2.json @@ -0,0 +1,99 @@ +{ + "schema": "research-run/2", + "run_id": "2026-03-31_decodex-plan-boundary-v2", + "question": "How should decodex handle plan-backed Linear issues while staying decoupled from plugin-specific contracts?", + "success_criteria": [ + "Clarify whether decodex should directly understand the plan plugin surface.", + "Identify the minimal stable surfaces decodex should depend on.", + "Identify the concrete conflict between the new single-issue plan authority pointer and decodex's current execution prompt input." + ], + "constraints": [ + "Keep decodex dependent only on Linear, GitHub, and Codex runtime surfaces.", + "Do not turn decodex into a consumer of plugin-private schemas by default.", + "Use repository-local evidence only for this bounded pass." + ], + "stop_rule": "Stop once the bounded pass either identifies one stable boundary shape or proves that a specific surface contract is still missing.", + "primary_hypothesis": "Decodex should remain plan-agnostic, and the current conflict comes from overloading the routed Linear issue body as both human task description and machine authority pointer.", + "rival_hypotheses": [ + "Decodex should parse and consume plan-authority or tracked-plan directly.", + "The current plan issue-body contract can coexist with decodex unchanged." + ], + "falsifiers": [ + "If decodex already consumes a generic execution surface that safely replaces issue.description, then the current conflict is overstated.", + "If the plan plugin can make the issue body machine-only without degrading decodex dispatch or operator comprehension, then the issue-body overload claim is false." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "stop", + "attempt": 1, + "max_attempts": 1 + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [] + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "source_family": "repo_code", + "summary": "Decodex currently builds its execution prompt by copying issue.description into the Description section for normal, review-repair, and delivery-closeout dispatch." + }, + { + "id": "E2", + "kind": "observation", + "source_family": "repo_docs", + "summary": "The updated plan plugin writing contract requires the routed Linear issue body to be exactly one fenced plan-authority/1 JSON block and explicitly forbids prose in that snapshot." + }, + { + "id": "E3", + "kind": "observation", + "source_family": "repo_docs", + "summary": "Decodex's documented plan-related boundary is currently negative only: saved plan phase=done never substitutes for explicit terminal finalization, but decodex does not otherwise consume plan-authority, tracked-plan, or plan-event contracts." + } + ], + "seq": 2 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Making decodex parse plugin-private plan contracts would couple runtime orchestration to plugin method contracts and turn decodex into a second control plane.", + "supporting_evidence_ids": [ + "E3" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Keeping decodex plan-agnostic while also replacing the issue body with a machine-only authority pointer leaves decodex without a stable human-readable task briefing surface.", + "supporting_evidence_ids": [ + "E1", + "E2" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 3 + }, + { + "type": "finalized_not_decision_ready", + "reason": "The bounded pass confirms that decodex should stay plan-agnostic and identifies a real surface conflict, but it still lacks an explicit replacement contract for the stable human-readable execution briefing surface when a routed issue body becomes a machine-only plan pointer.", + "missing_evidence": [ + "An explicit repository contract for where decodex should read the human-readable execution briefing when the routed issue body is reserved for plan-authority/1.", + "A repository decision on whether plan-backed issues are eligible for direct decodex dispatch before Codex-side plan execution has materialized the current task." + ], + "seq": 4 + } + ] +} diff --git a/docs/research/2026-03-31_decodex-plan-boundary-v3.json b/docs/research/2026-03-31_decodex-plan-boundary-v3.json new file mode 100644 index 00000000..f45e30f4 --- /dev/null +++ b/docs/research/2026-03-31_decodex-plan-boundary-v3.json @@ -0,0 +1,320 @@ +{ + "schema": "research-run/2", + "run_id": "2026-03-31_decodex-plan-boundary-v3", + "question": "How should decodex handle plan-backed Linear issues while staying decoupled from plugin-specific contracts?", + "success_criteria": [ + "Choose a stable execution-briefing surface that lets decodex stay plan-agnostic.", + "Compare at least two realistic design options for where the human-readable task briefing should live.", + "Produce a bounded recommendation that preserves the Linear + GitHub + Codex dependency boundary." + ], + "constraints": [ + "Keep decodex dependent only on Linear, GitHub, and Codex runtime surfaces.", + "Do not require decodex to parse plugin-private plan contracts by default.", + "Use repository-local evidence for the bounded pass." + ], + "stop_rule": "Stop once one option is recommendation-worthy under the current evidence or the bounded pass proves that no safe choice is ready.", + "primary_hypothesis": "The right design is to keep decodex plan-agnostic and move the machine plan pointer away from the only surface that decodex currently uses as a human-readable execution briefing.", + "rival_hypotheses": [ + "Decodex should read the linked tracked-plan surface directly when a routed issue is plan-backed.", + "Plan-backed issues should remain ineligible for direct decodex dispatch until a Codex-side step materializes a generic current-task surface." + ], + "falsifiers": [ + "If a decodex-safe human-readable briefing can be derived from the existing plan-linked document without making decodex understand plan-private schemas, the current conflict may be solvable without moving the pointer.", + "If moving the pointer off the issue body creates worse coordination failure than teaching decodex a generic linked-document briefing read, the plan-agnostic recommendation is too strict." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "stop", + "attempt": 1, + "max_attempts": 1 + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "Which surface should carry the stable human-readable execution briefing when a routed issue also carries machine plan authority?", + "Which option preserves decodex's runtime boundary best while minimizing workflow churn for Linear-backed planning?" + ], + "external_slices": [] + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "source_family": "repo_code", + "summary": "Decodex currently builds its execution prompt by copying issue.description into the Description section for normal, review-repair, and delivery-closeout dispatch." + }, + { + "id": "E2", + "kind": "observation", + "source_family": "repo_docs", + "summary": "The updated plan plugin writing contract requires the routed Linear issue body to be exactly one fenced plan-authority/1 JSON block and explicitly forbids prose in that snapshot." + }, + { + "id": "E3", + "kind": "observation", + "source_family": "repo_docs", + "summary": "Decodex's current documented plan boundary only says that saved plan phase=done cannot replace explicit terminal finalization; it does not otherwise consume plan-authority, tracked-plan, or plan-event contracts." + } + ], + "seq": 2 + }, + { + "type": "worker_completed", + "worker": "analyst", + "summary": "Preferred option A. It keeps decodex plan-agnostic and preserves the existing human-readable issue briefing path, while B risks turning generic linked-document reads into de facto plan parsing and C adds dispatch gating and materialization friction.", + "seq": 3, + "target_inventory_seq": 1 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Option A preserves the cleanest boundary because decodex can continue to rely on a human-readable issue brief and never consume plugin-private plan schemas.", + "supporting_evidence_ids": [ + "E1", + "E3" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Option A has the highest migration cost because the current plan plugin contract hard-codes the issue body as the authority pointer surface.", + "supporting_evidence_ids": [ + "E2" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Option B appears cheaper, but it weakens the boundary because a generic linked-document briefing read risks becoming de facto parsing of plan-owned structure.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E3" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T4", + "summary": "Option C keeps decodex decoupled but adds dispatch gating and a second materialization surface before normal decodex execution can begin.", + "supporting_evidence_ids": [ + "E1", + "E2" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 4 + }, + { + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "A_move_machine_pointer_off_issue_body", + "rejected_options": [ + "B_generic_linked_document_briefing_read_in_decodex", + "C_make_plan_backed_issues_ineligible_until_materialized" + ], + "decision_claim": "Keep decodex plan-agnostic and move the machine plan pointer off the routed issue body, preserving a stable human-readable issue description or equivalent generic issue briefing surface for decodex dispatch.", + "key_evidence_ids": [ + "E1", + "E2", + "E3" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4" + ] + }, + "seq": 5, + "judgment_hash": "sha256:e413dcbbac1ed9f12c4cf3d6449a1ec895c71de715092e8f6d723711e2e288e7" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The judgment survives only in narrowed form: decodex should stay plan-agnostic, but moving the machine pointer off the routed issue body does not survive challenge as an immediate recommendation because it breaks the current plan authority contract unless that contract is redesigned in lockstep.", + "objections": [ + { + "id": "O1", + "summary": "Moving the pointer off the routed issue body immediately breaks the current plan writing, execution, and validation contract." + }, + { + "id": "O2", + "summary": "Any replacement surface must remain singular and machine-addressable inside Linear or the plan authority model becomes ambiguous." + }, + { + "id": "O3", + "summary": "Decodex currently dispatches from issue.description, so any preserved human-readable briefing must either remain there or be accompanied by an explicit synchronized dispatch-surface change." + } + ], + "seq": 6, + "target_judgment_hash": "sha256:e413dcbbac1ed9f12c4cf3d6449a1ec895c71de715092e8f6d723711e2e288e7" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E4", + "kind": "observation", + "source_family": "repo_docs", + "summary": "The plan execution contract hard-stops unless the routed issue body contains a valid plan-authority/1 envelope, and it forbids execution from mutating those issue-body pointer fields." + } + ], + "seq": 7 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T5", + "summary": "The immediate safe recommendation is to keep decodex plan-agnostic and reject machine-only issue bodies for normal decodex dispatch, because moving the pointer off issue.description without a coordinated contract migration breaks the current plan authority model.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E4" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T6", + "summary": "A future contract redesign may still move machine authority away from the issue body, but only if it preserves one singular machine-addressable Linear authority surface and simultaneously gives decodex an explicit stable dispatch briefing surface.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E4" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 8 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "O1", + "O2", + "O3" + ], + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "A_prime_keep_decodex_plan_agnostic_but_disallow_machine_only_issue_bodies_for_direct_dispatch", + "rejected_options": [ + "B_generic_linked_document_briefing_read_in_decodex", + "C_make_plan_backed_issues_ineligible_until_materialized" + ], + "decision_claim": "Keep decodex plan-agnostic. Under the current contracts, decodex should not accept routed issues whose issue body is only a machine plan pointer, because decodex dispatch still depends on a stable human-readable issue.description surface. Immediate recommendation: do not teach decodex to parse plan-private contracts; instead require any decodex-dispatched issue to preserve a stable generic briefing surface in issue.description, and treat a machine-only issue body as inaligned with normal decodex dispatch until a coordinated contract redesign lands.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4" + ], + "key_tradeoff_ids": [ + "T1", + "T5", + "T6" + ] + }, + "seq": 9, + "judgment_hash": "sha256:ffe1ba1e527bcb61d5cb1455c461a421798f210cc958c7698160fb5d88e7fb57" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The narrowed judgment still overstates the current contract. The source set proves a prompt-surface mismatch, but not an existing decodex eligibility rule. It also needed one explicit field-mapping check to confirm that the plan plugin's issue-body pointer is the same Linear description surface that decodex dispatches.", + "objections": [ + { + "id": "O4", + "summary": "The recommendation should be framed as a design recommendation or contract rule, not as an existing decodex acceptance contract." + }, + { + "id": "O5", + "summary": "The issue-body conflict needed one explicit field-mapping check from Linear description to TrackerIssue.description before it could support a firm integration claim." + } + ], + "seq": 10, + "target_judgment_hash": "sha256:ffe1ba1e527bcb61d5cb1455c461a421798f210cc958c7698160fb5d88e7fb57" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E5", + "kind": "observation", + "source_family": "repo_code", + "summary": "Decodex's Linear tracker maps the GraphQL issue.description field directly into TrackerIssue.description, and orchestrator prompt construction injects that same TrackerIssue.description into the dispatch prompt's Description section." + } + ], + "seq": 11 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T7", + "summary": "The strongest supported output is a design recommendation and contract rule for future integration, not a claim that current decodex runtime already enforces dispatch eligibility based on issue-body shape.", + "supporting_evidence_ids": [ + "E1", + "E3", + "E5" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 12 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "O4", + "O5" + ], + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "A_double_prime_keep_decodex_plan_agnostic_and_define_a_contract_rule_for_dispatch_surface", + "rejected_options": [ + "B_generic_linked_document_briefing_read_in_decodex", + "C_make_plan_backed_issues_ineligible_until_materialized" + ], + "decision_claim": "Design recommendation: keep decodex plan-agnostic and do not teach it to parse plan-private contracts. Because decodex dispatch currently reads Linear description verbatim and the current plan plugin requires that same issue body to be machine-only plan-authority/1 JSON, a machine-only routed issue body is inaligned with normal decodex dispatch under the present contracts. The immediate contract rule should therefore be: any issue that decodex dispatches must preserve a stable generic human-readable briefing in the Linear description surface, and plan-contract changes that remove that surface require a coordinated redesign of both plan authority and decodex dispatch inputs.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E5" + ], + "key_tradeoff_ids": [ + "T1", + "T6", + "T7" + ] + }, + "seq": 13, + "judgment_hash": "sha256:273e70dd4050cc4a7bb756f5a75397eb8883af9065bfb715619d483cace91eab" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The judgment survives. The strongest caveat is that the present evidence proves prompt-content mismatch rather than a guaranteed hard runtime failure, and a future coordinated redesign could still intentionally expose a public planning surface to decodex without violating plan-agnosticism.", + "seq": 14, + "target_judgment_hash": "sha256:273e70dd4050cc4a7bb756f5a75397eb8883af9065bfb715619d483cace91eab" + }, + { + "type": "finalized_decision_ready", + "judgment_hash": "sha256:273e70dd4050cc4a7bb756f5a75397eb8883af9065bfb715619d483cace91eab", + "confidence": "high", + "missing_evidence": [], + "seq": 15 + } + ] +} diff --git a/docs/research/2026-03-31_decodex-plan-boundary/run.json b/docs/research/2026-03-31_decodex-plan-boundary/run.json new file mode 100644 index 00000000..956a7ef5 --- /dev/null +++ b/docs/research/2026-03-31_decodex-plan-boundary/run.json @@ -0,0 +1,87 @@ +{ + "schema": "research-run/2", + "run_id": "2026-03-31_decodex-plan-boundary", + "question": "How should decodex handle plan-backed Linear issues while staying decoupled from plugin-specific contracts?", + "success_criteria": [ + "Clarify whether decodex needs any direct awareness of the plan plugin.", + "Identify the minimal stable surfaces decodex should depend on.", + "Identify the concrete conflict between the new plan issue-body contract and the current decodex dispatch path." + ], + "constraints": [ + "Keep decodex dependent only on Linear, GitHub, and Codex runtime surfaces.", + "Do not turn decodex into a consumer of plugin-private schemas by default.", + "Use repository-local evidence only for this bounded pass." + ], + "stop_rule": "Stop once the bounded pass either identifies one stable boundary shape or proves that more explicit surface design work is still required.", + "primary_hypothesis": "Decodex should remain plan-agnostic, and the current conflict comes from overloading the Linear issue body as both human task description and machine authority pointer.", + "rival_hypotheses": [ + "Decodex should parse and consume plan-authority or tracked-plan directly.", + "The current plan issue-body contract can coexist with decodex unchanged." + ], + "falsifiers": [ + "If decodex already consumes a generic execution surface that can replace issue.description safely, then the current conflict is overstated.", + "If the plan plugin can keep the issue body machine-only without degrading decodex task dispatch, then the issue-body overload claim is false." + ], + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The plan plugin writing contract requires the routed Linear issue body to be exactly one fenced plan-authority/1 JSON block and explicitly forbids prose in that snapshot." + }, + { + "id": "E2", + "kind": "observation", + "summary": "Decodex currently builds its dispatch prompt by copying issue.description into the Description section for normal, review-repair, and delivery-closeout runs." + }, + { + "id": "E3", + "kind": "observation", + "summary": "Decodex's current plan-related logic is limited to preventing a saved plan phase=done from replacing explicit terminal finalization; it does not otherwise consume plan-authority, tracked-plan, or plan-event contracts." + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Making decodex parse plugin-private plan contracts would couple runtime orchestration to plugin method contracts and create a second control plane boundary.", + "supporting_evidence_ids": [ + "E3" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Keeping decodex plan-agnostic while also replacing the issue body with a machine-only pointer leaves decodex without a stable human-readable task briefing surface.", + "supporting_evidence_ids": [ + "E1", + "E2" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "finalized_not_decision_ready", + "reason": "The bounded pass confirms a real surface conflict but does not yet establish the replacement contract for where a decodex-safe human task briefing should live when the issue body becomes a machine-only authority pointer.", + "missing_evidence": [ + "An explicit repository-level contract for the human-readable execution briefing surface when a routed issue also carries plan-authority/1.", + "A decision on whether planned issues are ever eligible for direct decodex dispatch before Codex-side plan execution has materialized the current task." + ] + } + ] +} diff --git a/docs/research/2026-04-01_plan-linear-decoupled-design/run.json b/docs/research/2026-04-01_plan-linear-decoupled-design/run.json new file mode 100644 index 00000000..eb1259fa --- /dev/null +++ b/docs/research/2026-04-01_plan-linear-decoupled-design/run.json @@ -0,0 +1,493 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-01_plan-linear-decoupled-design", + "question": "How should the plan plugin keep its old docs/plans strengths after moving storage to Linear while staying decoupled from decodex and from overly specific Linear transport details?", + "success_criteria": [ + "Identify the clean boundary between decodex, the routed Linear issue, and the plan plugin's machine authority surfaces.", + "Compare at least two viable plan-plugin architecture options for the Linear-backed design.", + "Recommend a design that preserves machine-readability and append-only history without over-coupling execution to raw Linear export shape." + ], + "constraints": [ + "Keep decodex plan-agnostic and dependent only on generic issue briefing plus normal tracker and PR surfaces.", + "Assume the canonical plan storage remains in Linear rather than reverting to repository-local docs/plans authority.", + "Use bounded repository-local evidence from decodex and the repo-local plan plugin only." + ], + "stop_rule": "Stop once the bounded pass can recommend one design shape for the Linear-backed plan plugin and name the smallest contract corrections needed to make it reliable.", + "primary_hypothesis": "The cleanest design keeps the current Linear surfaces but splits the plugin into a thin Linear ingress adapter plus a transport-agnostic plan core that consumes only a normalized execution surface.", + "rival_hypotheses": [ + "The current design is already the right long-term shape and only needs small contract wording fixes.", + "The plugin should let the execution core depend directly on the full routed Linear issue export because that is simpler and the coupling cost is acceptable." + ], + "falsifiers": [ + "If the current design already limits execution to a minimal normalized surface rather than the full routed issue export, an ingress/core split would be unnecessary.", + "If adding a normalized ingress layer would force duplicated selection logic or materially reduce reliability, the extra decoupling would not be worth it." + ], + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 2, + "independent_option_questions": [ + "Should the plan execution core depend directly on the routed Linear issue export or on a normalized execution surface derived from it?", + "Which contract elements must stay strict machine-only versus which should remain generic human-readable briefing surfaces?" + ], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "Decodex now treats the routed issue description only as a generic briefing surface and explicitly redacts machine-only fenced payloads from dispatch prompts." + }, + { + "id": "E2", + "kind": "observation", + "summary": "The repo-local plan writing contract now keeps the routed Linear issue as the parent work item and generic dispatch briefing surface instead of a plugin-private JSON envelope." + }, + { + "id": "E3", + "kind": "observation", + "summary": "The repo-local plan writing contract stores the canonical tracked-plan/1 contract in exactly one linked Linear document and stores append-only execution history in plan-event/1 comments." + }, + { + "id": "E4", + "kind": "observation", + "summary": "The current execution helpers still validate against the routed issue export and require issue.documents[].content for every linked document, which means the current consumer path still depends on a fairly raw Linear export shape." + }, + { + "id": "E5", + "kind": "observation", + "summary": "The plan-event validator only accepts a strict single fenced JSON block, while the execution skill text still contains one step that implies extra prose fields in the same progress comment." + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Keeping the routed issue generic preserves decodex and human operator usability, but it means the plan plugin must not overload that issue surface with machine authority.", + "supporting_evidence_ids": [ + "E1", + "E2" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "The current direct use of routed issue exports is pragmatic and keeps the plugin thin, but it also couples execution more tightly to Linear export shape than the old docs/plans model needed.", + "supporting_evidence_ids": [ + "E3", + "E4" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Strict single-block machine contracts keep events and linked docs easy to validate, but any mixed prose requirement in the same surface immediately creates self-contradictory contracts.", + "supporting_evidence_ids": [ + "E5" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "worker_completed", + "worker": "analyst", + "target_inventory_seq": 1, + "summary": "Option B is the better long-term design if the adapter stays thin: keep the current Linear surfaces, but concentrate raw issue-export reading, canonical document selection, and cross-surface validation in an ingress layer so the execution core consumes only a validated normalized surface." + }, + { + "seq": 5, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E6", + "kind": "observation", + "summary": "After cross-surface validation, the current execution contract already treats the linked document's tracked-plan state as the source of truth rather than the raw routed issue export." + }, + { + "id": "E7", + "kind": "inference", + "summary": "A normalized ingress layer is practical because the current helpers already derive a validated execution surface; the refactor can narrow and clean that boundary instead of redesigning the external Linear surfaces." + }, + { + "id": "E8", + "kind": "inference", + "summary": "Keeping execution bound directly to issue.documents content and linked-document enumeration preserves current behavior but does not recover the old docs/plans-style transport decoupling." + } + ] + }, + { + "seq": 6, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T4", + "summary": "A thin Linear ingress adapter preserves the successful old docs/plans semantics after the move to Linear, but only if the adapter owns raw export validation and the core stops depending on full routed issue exports.", + "supporting_evidence_ids": [ + "E4", + "E6", + "E7" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T5", + "summary": "The normalized execution surface should avoid duplicating phase and task-pointer fields outside tracked-plan.state; otherwise the adapter and core would create a second competing runtime truth.", + "supporting_evidence_ids": [ + "E6", + "E7" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 7, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "Option B", + "rejected_options": [ + "Option A" + ], + "decision_claim": "Keep the current external Linear surfaces but split the plan plugin into a thin Linear ingress adapter plus a transport-agnostic execution core whose minimal contract is the validated tracked-plan plus stable ids, with latest-event retained only as validated provenance rather than a second runtime source of truth.", + "key_evidence_ids": [ + "E1", + "E3", + "E4", + "E6", + "E7", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5" + ] + }, + "judgment_hash": "sha256:277e518b517bb78f2dc5d0c49c2b40502011e1bf3b4aabf510383ad31525aa35" + }, + { + "seq": 8, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:277e518b517bb78f2dc5d0c49c2b40502011e1bf3b4aabf510383ad31525aa35", + "summary": "The first judgment was too aggressive: latest-event still participates in canonical selection, runtime progression still leaks through spec.tasks status, and a truly minimal transport-agnostic core is not yet justified by the current contract." + }, + { + "seq": 9, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E9", + "kind": "observation", + "summary": "The current contract forbids execution from mutating spec.* but still uses spec.tasks[*].status to validate legal runtime progression, which means runtime ownership is not yet fully isolated inside tracked-plan.state." + }, + { + "id": "E10", + "kind": "observation", + "summary": "The latest valid plan-event is not only audit history; it also participates in canonical tracked-plan document selection when multiple linked tracked-plan candidates exist." + }, + { + "id": "E11", + "kind": "inference", + "summary": "The clean next step is a staged design: first make plugin-local runtime ownership self-consistent, then narrow the adapter boundary, instead of promising a fully transport-agnostic core immediately." + }, + { + "id": "E12", + "kind": "observation", + "summary": "Current event ordering and canonical selection still rely on second-granularity timestamps, which means adapter extraction alone would not remove ambiguity without an accompanying contract cleanup." + } + ] + }, + { + "seq": 10, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T6", + "summary": "A staged design preserves the successful move from docs/plans to Linear while avoiding a premature promise of a transport-agnostic core before runtime ownership and canonical selection semantics are cleaned up.", + "supporting_evidence_ids": [ + "E9", + "E10", + "E11" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T7", + "summary": "Cleaning plugin-local state ownership first adds short-term contract work, but it prevents the adapter/core split from merely renaming today's internal contradictions.", + "supporting_evidence_ids": [ + "E9", + "E11", + "E12" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 11, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "Staged Option B", + "rejected_options": [ + "Option A", + "Immediate thin-core Option B" + ], + "decision_claim": "Keep the current external Linear surfaces, but treat the next design step as a staged clean-break inside the plugin: first make runtime ownership self-consistent by keeping generic issue briefing on the routed issue, one canonical tracked-plan in the linked document, strict machine-only plan-event comments, and runtime progression tracked only in runtime-owned fields; then narrow read_execution_surface into an adapter-owned validated ingress contract without promising a fully transport-agnostic execution core until canonical selection and state ownership are clean.", + "key_evidence_ids": [ + "E1", + "E3", + "E5", + "E9", + "E10", + "E11", + "E12" + ], + "key_tradeoff_ids": [ + "T1", + "T3", + "T6", + "T7" + ] + }, + "judgment_hash": "sha256:b1e137e4fe6865cb48df44d6477e2f75db7911a769460b6cd0c77499566241e3" + }, + { + "seq": 12, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:b1e137e4fe6865cb48df44d6477e2f75db7911a769460b6cd0c77499566241e3", + "summary": "The staged design was still incomplete: it did not assign a legal runtime owner for task status, and it deferred the ingress adapter even though execution already depends on linked documents and comments that current generic runtime surfaces do not provide." + }, + { + "seq": 13, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E13", + "kind": "observation", + "summary": "Execution is currently allowed to mutate only runtime-owned state while task status still lives inside spec.tasks, so the current contract does not give execution a legal way to advance plan-local task progress." + }, + { + "id": "E14", + "kind": "observation", + "summary": "Current execution depends on linked documents and plan-event comment history, but the generic runtime surfaces discussed around decodex do not themselves supply a stable ingress contract for those documents and comments." + }, + { + "id": "E15", + "kind": "inference", + "summary": "The clean next design is boundary-first repair: introduce an explicit Linear ingress adapter now while simultaneously moving runtime task progress fully into runtime-owned state, instead of deferring either half." + }, + { + "id": "E16", + "kind": "observation", + "summary": "Canonical tracked-plan selection still relies on timestamp ordering across candidate event histories, so the ingress boundary needs an explicit ordering or revision responsibility rather than leaving that ambiguity implicit." + } + ] + }, + { + "seq": 14, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T8", + "summary": "Moving task progress and task-status ownership out of spec and into runtime-owned state restores the old docs/plans-style separation between stable intent and execution state, but it requires a schema revision to tracked-plan rather than a wording-only fix.", + "supporting_evidence_ids": [ + "E9", + "E13", + "E15" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T9", + "summary": "An explicit Linear ingress adapter has to land together with the contract cleanup because execution already depends on linked docs and comments; delaying adapter extraction would keep transport coupling hidden in the core path.", + "supporting_evidence_ids": [ + "E10", + "E14", + "E15" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T10", + "summary": "Canonical selection and event ordering should become an adapter-owned responsibility with an explicit revision or ordering rule; otherwise timestamp ambiguity remains even after other decoupling work lands.", + "supporting_evidence_ids": [ + "E12", + "E16" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 15, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "Boundary-first repair", + "rejected_options": [ + "Option A", + "Immediate thin-core Option B", + "State-cleanup-before-adapter staged design" + ], + "decision_claim": "Keep the current external Linear surfaces, but make the next design step a boundary-first repair inside the plugin: routed issue remains generic briefing only; one linked document remains the canonical tracked-plan; plan-event comments stay strict single JSON blocks; runtime task progress and task status move fully into runtime-owned state rather than spec.tasks; and an explicit Linear ingress adapter becomes responsible for reading issue, documents, and comments, selecting the canonical tracked-plan with a defined ordering rule, and emitting one validated execution-surface contract for the execution path. A narrower transport-agnostic core may follow later, but this boundary-first repair is the decision-ready next step.", + "key_evidence_ids": [ + "E1", + "E3", + "E5", + "E10", + "E13", + "E14", + "E15", + "E16" + ], + "key_tradeoff_ids": [ + "T1", + "T3", + "T8", + "T9", + "T10" + ] + }, + "judgment_hash": "sha256:f5cff28ca34dcc7e6660b0e05c70d17dd0744824b78f25953d0d75ee7f465ee9" + }, + { + "seq": 16, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:f5cff28ca34dcc7e6660b0e05c70d17dd0744824b78f25953d0d75ee7f465ee9", + "summary": "Boundary-first repair is directionally right but still incomplete unless it also defines cross-document lineage, a write-side consistency rule, a concrete runtime task-state schema, and explicit semantics for plan-local done versus decodex lane completion." + }, + { + "seq": 17, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E17", + "kind": "observation", + "summary": "Current plan-event contracts do not carry a cross-document revision or supersedes primitive, so canonical selection still falls back to timestamp ordering across histories." + }, + { + "id": "E18", + "kind": "observation", + "summary": "Current execution updates the linked document and append-only event separately while validation later requires exact agreement, which means transient write skew can block execution." + }, + { + "id": "E19", + "kind": "observation", + "summary": "The current contract relies on spec.tasks status for plan-local progress, but execution is not allowed to mutate spec.*, so a replacement runtime task-state shape is required rather than implied." + }, + { + "id": "E20", + "kind": "observation", + "summary": "The plan plugin treats done as plan-local while decodex's current operator guidance warns agents not to mark a saved plan done before terminal finalize succeeds, so outer-lifecycle semantics are not yet aligned." + }, + { + "id": "E21", + "kind": "inference", + "summary": "A decision-ready design needs one more level of specificity: event lineage and write protocol must be first-class, not residual implementation details." + } + ] + }, + { + "seq": 18, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T11", + "summary": "Treating events as the authoritative runtime log and the linked document as a checkpointed snapshot reduces write-skew fragility, but it makes readers responsible for replay or lag-tolerant validation.", + "supporting_evidence_ids": [ + "E18", + "E21" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T12", + "summary": "Adding an explicit plan revision or lineage contract is more verbose than timestamp selection, but it restores deterministic authority after replans and preserves one of the old docs/plans strengths: clear canonical ownership.", + "supporting_evidence_ids": [ + "E17", + "E21" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T13", + "summary": "Separating plan-local done from lane terminal completion preserves plugin autonomy, but it requires a small decodex wording change so plan.done is no longer treated as forbidden before terminal finalize in every context.", + "supporting_evidence_ids": [ + "E20" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 19, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "Boundary-first repair with event-log authority", + "rejected_options": [ + "Option A", + "Immediate thin-core Option B", + "State-cleanup-before-adapter staged design", + "Boundary-first repair without lineage/write-protocol changes" + ], + "decision_claim": "Keep the current external Linear surfaces, but define the next architecture as a boundary-first repair with event-log authority: the routed issue remains generic briefing only; the linked document keeps stable plan intent and a checkpointed runtime snapshot; append-only plan-event comments become the authoritative runtime log and must stay strict single JSON blocks; a new explicit plan revision/lineage field replaces timestamp-only canonical selection across replans; runtime task progress moves into a runtime-owned task-state structure rather than spec.tasks status; and a Linear ingress adapter becomes responsible for reading issue, documents, and comments, selecting the canonical document and latest authoritative event by lineage plus seq, tolerating checkpoint lag under a defined write protocol, and emitting one validated execution surface for execution. Decodex stays plan-agnostic, but its wording should distinguish plan-local done from lane terminal completion instead of banning plan.done outright before terminal finalize.", + "key_evidence_ids": [ + "E1", + "E3", + "E5", + "E13", + "E14", + "E17", + "E18", + "E19", + "E20", + "E21" + ], + "key_tradeoff_ids": [ + "T1", + "T3", + "T8", + "T9", + "T11", + "T12", + "T13" + ] + }, + "judgment_hash": "sha256:6897bb1133bc40d498e1aaf0b1edea57d688814ee4d0e08866fa56dda4056f0c" + }, + { + "seq": 20, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:6897bb1133bc40d498e1aaf0b1edea57d688814ee4d0e08866fa56dda4056f0c", + "summary": "The boundary-first repair with event-log authority is directionally coherent, but it is still missing implementation-blocking contract details: the runtime task-state schema, lineage/revision semantics, and the exact lag-tolerant write protocol." + }, + { + "seq": 21, + "type": "finalized_not_decision_ready", + "reason": "The bounded pass identified a coherent architecture direction, but the design is still missing concrete contract definitions for runtime task-state replacement, cross-document lineage and canonical-selection semantics, and a lag-tolerant write protocol between checkpointed documents and authoritative plan-event history.", + "missing_evidence": [ + "A concrete runtime task-state schema that can replace spec.tasks[*].status while preserving deterministic next-task and dependency validation.", + "A precise lineage or revision contract for replans and cross-document canonical selection, including how a new revision becomes authoritative before it has execution history.", + "A defined write protocol and validator rule for document and event skew, including write order, crash recovery, and when a stale checkpoint is acceptable versus blocking.", + "An explicit alignment rule between plan-local done and decodex lane terminal completion semantics." + ] + } + ] +} diff --git a/docs/research/2026-04-02_delivery-skill-removal-with-agents.json b/docs/research/2026-04-02_delivery-skill-removal-with-agents.json new file mode 100644 index 00000000..f2fc9bb6 --- /dev/null +++ b/docs/research/2026-04-02_delivery-skill-removal-with-agents.json @@ -0,0 +1,320 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-02_delivery-skill-removal-with-agents", + "question": "How should decodex remove the delivery skill family while staying consistent with the current AGENTS.md workflow and minimizing breakage?", + "success_criteria": [ + "Recommend a concrete target design for removing delivery-prepare and delivery-closeout as skills.", + "Explain which responsibilities move into Decodex runtime/spec/prompting versus WORKFLOW.md or other repo policy surfaces.", + "Give a phased migration route that preserves the existing review -> land -> closeout lifecycle while reducing skill authority." + ], + "constraints": [ + "Use the Research plugin's run-file workflow and host-level subagents.", + "Prefer an incremental migration over a big-bang rewrite.", + "Do not rely on human-readable commit history as a requirement." + ], + "stop_rule": "Stop once there is a challenge-ready recommendation for delivery-skill removal, or once bounded evidence shows the recommendation is still unsafe.", + "primary_hypothesis": "The delivery skill family can be removed cleanly by moving lifecycle authority into Decodex runtime and post-review protocol surfaces while moving repo-specific validation and docs gates into WORKFLOW.md-backed policy.", + "rival_hypotheses": [ + "A thin delivery shim skill is still required because AGENTS, landing, and repair flows depend too directly on delivery-specific routing.", + "Delivery removal should wait until review-prepare, review-repair, pr-land, and review-loop are all redesigned together in one larger protocol rewrite." + ], + "falsifiers": [ + "If current runtime or prompting still depends on delivery skill-local semantics that cannot be expressed by existing retained-lane protocol surfaces, removing delivery skills first is not safe.", + "If repo-specific validation, docs automation, or merge policy cannot be externalized without reintroducing delivery-specific skills, the proposed split is incomplete." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 2, + "session_id": "2026-04-02_delivery-skill-removal-with-agents" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "What is the smallest migration slice that removes delivery skills without breaking the retained review -> land -> closeout lifecycle?", + "Which current delivery responsibilities belong in Decodex runtime/spec/prompting versus WORKFLOW.md or AGENTS policy?" + ], + "external_slices": [ + { + "id": "S1", + "question": "Which delivery responsibilities are already modeled as Decodex runtime, prompt, or tracker-tool authority rather than skill-local behavior?" + }, + { + "id": "S2", + "question": "Which current skill, AGENTS, and dev-smoke references still hard-depend on delivery-prepare or delivery-closeout?" + }, + { + "id": "S3", + "question": "Which repo-specific validation and docs gates can be represented by WORKFLOW.md or adjacent policy without keeping delivery skills?" + } + ] + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "Decodex runtime, prompting, and tracker-tool bridge already model retained delivery_closeout as a first-class post-review phase with dedicated dispatch, prompt contract, and terminal completion handling.", + "source_family": "repo_code" + }, + { + "id": "E2", + "kind": "observation", + "summary": "Repo policy already has machine-readable surfaces for completed tracker state and repo-native validation commands through WORKFLOW.md, so delivery-specific test gating is not required as a skill-local authority.", + "source_family": "repo_spec" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The strongest remaining delivery dependencies are in .codex/AGENTS.md workflow prose, pr-land and review-repair skill instructions, and dev smoke coverage rather than in the retained-lane runtime model itself.", + "source_family": "repo_code" + }, + { + "id": "E4", + "kind": "contradiction", + "summary": "A retained delivery_closeout continuation boundary currently requires the issue to remain in success_state even though dispatch policy and retry tests allow delivery_closeout scheduling after the issue reaches completed state.", + "source_family": "repo_code" + }, + { + "id": "E5", + "kind": "observation", + "summary": "Current delivery history semantics such as delivery/1 merge preservation and merge-commit payload generation live in pr-land and AGENTS policy, and WORKFLOW.md does not yet expose an equivalent merge-history policy field.", + "source_family": "repo_code" + } + ], + "seq": 2 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Removing delivery skills first has low runtime-model risk because retained delivery_closeout authority already exists, but it still requires coordinated policy and test cleanup across AGENTS, skill docs, and dev smoke.", + "supporting_evidence_ids": [ + "E1", + "E3" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Moving repo-native validation and docs gates into WORKFLOW.md or adjacent repo policy simplifies the runtime/skill boundary, but merge-history semantics need a new home because they are not represented in the current workflow contract.", + "supporting_evidence_ids": [ + "E2", + "E5" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "An incremental delivery-only removal slice minimizes blast radius, but the continuation-boundary contradiction should be resolved or explicitly constrained before the final closeout protocol is declared complete.", + "supporting_evidence_ids": [ + "E4" + ], + "disconfirming_evidence_ids": [ + "E1" + ] + } + ], + "seq": 3 + }, + { + "type": "worker_completed", + "worker": "scout", + "summary": "Scout confirmed that delivery_closeout lifecycle authority already lives in Decodex runtime/spec, while remaining delivery dependencies are concentrated in AGENTS, skill prose, and dev smoke rather than in the retained-lane model.", + "seq": 4, + "target_inventory_seq": 1 + }, + { + "type": "inventory_updated", + "remaining_option_count": 2, + "independent_option_questions": [ + "Should delivery skill removal be executed now as its own first migration slice, or delayed until a larger review/landing protocol rewrite?", + "Which delivery responsibilities need new runtime or workflow surfaces before the skill pair can be deleted without leaving policy gaps?" + ], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5" + ], + "seq": 5 + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E6", + "kind": "inference", + "summary": "The preferred migration order is to remove delivery-prepare and delivery-closeout first, because Decodex already owns closeout lifecycle authority while the remaining skill dependencies are concentrated in repo policy and smoke coverage.", + "source_family": "repo_analysis" + }, + { + "id": "E7", + "kind": "missing_evidence", + "summary": "The only unresolved design check that could change the migration shape is whether retained delivery_closeout continuation after transition to completed_state is intended to be allowed or whether closeout is intentionally single-turn before completion." + } + ], + "seq": 6 + }, + { + "type": "worker_completed", + "worker": "analyst", + "summary": "Analyst prefers a delivery-first removal slice now, with merge-history policy and closeout-boundary semantics externalized before direct skill deletion.", + "seq": 7, + "target_inventory_seq": 5 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T4", + "summary": "A delivery-first removal slice makes immediate progress with bounded blast radius, but it still requires adding a new runtime or workflow surface for merge-history policy before pr-land can stop depending on delivery/1 semantics.", + "supporting_evidence_ids": [ + "E3", + "E5", + "E6" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T5", + "summary": "Waiting for a larger protocol rewrite reduces duplicate policy edits, but it delays removal even though the retained delivery_closeout lifecycle is already internal to Decodex.", + "supporting_evidence_ids": [ + "E1", + "E6" + ], + "disconfirming_evidence_ids": [ + "E4" + ] + } + ], + "seq": 8 + }, + { + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "delivery_first_runtime_externalization", + "rejected_options": [ + "wait_for_full_review_landing_protocol_rewrite" + ], + "decision_claim": "Remove delivery-prepare and delivery-closeout as the first migration slice now. Move retained delivery lifecycle authority fully into Decodex runtime/spec/prompting, move repo-native validation and docs gates into WORKFLOW.md-backed policy, add a replacement merge-history policy surface before removing pr-land's delivery/1 assumptions, and resolve or explicitly constrain completed-state continuation semantics during the slice.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5" + ] + }, + "seq": 9, + "judgment_hash": "sha256:feaeeeb23ca74758ab00810c5aac2f41bcaaed5c40d6f89b796004f8ac1f1f31" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "Skeptic agrees with a delivery-first slice only if the migration explicitly handles the current GitHub-mirroring responsibility and the completed-state continuation mismatch.", + "objections": [ + { + "id": "O1", + "summary": "The current recommendation leaves a behavior gap unless GitHub mirroring is explicitly de-scoped or replaced before deleting delivery-closeout." + } + ], + "seq": 10, + "target_judgment_hash": "sha256:feaeeeb23ca74758ab00810c5aac2f41bcaaed5c40d6f89b796004f8ac1f1f31" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E8", + "kind": "observation", + "summary": "The lifecycle spec still treats GitHub mirroring as part of delivery_closeout completion, while the retained-lane runtime evidence gathered for this run centered on merged PR lineage and tracker completed-state validation rather than an explicit mirroring implementation path.", + "source_family": "repo_spec" + } + ], + "seq": 11 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T6", + "summary": "A delivery-first slice remains sound only if GitHub mirroring is either moved into runtime-owned closeout behavior or explicitly removed from lifecycle expectations during the same slice.", + "supporting_evidence_ids": [ + "E8" + ], + "disconfirming_evidence_ids": [ + "E1" + ] + } + ], + "seq": 12 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "O1" + ], + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "delivery_first_runtime_externalization", + "rejected_options": [ + "wait_for_full_review_landing_protocol_rewrite" + ], + "decision_claim": "Remove delivery-prepare and delivery-closeout as the first migration slice now, but make the slice explicitly include three policy transfers before direct skill deletion: (1) move retained delivery lifecycle authority fully into Decodex runtime/spec/prompting, (2) move repo-native validation and docs gates into WORKFLOW.md-backed policy, and (3) either move GitHub mirroring into runtime-owned closeout behavior or explicitly remove it from lifecycle expectations. Also add a replacement merge-history policy surface before removing pr-land's delivery/1 assumptions, and resolve or explicitly constrain completed-state continuation semantics during the slice.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5", + "T6" + ] + }, + "seq": 13, + "judgment_hash": "sha256:28a47aa43f99a3b7eaee7532aad438fe97226fc42ffae69aafddd11954dcb243" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "Replacement judgment survives challenge. No remaining material objections remain if the slice explicitly covers GitHub mirroring disposition, replacement merge-history policy, and the completed-state continuation acceptance gate.", + "objections": [], + "seq": 14, + "target_judgment_hash": "sha256:28a47aa43f99a3b7eaee7532aad438fe97226fc42ffae69aafddd11954dcb243" + }, + { + "type": "finalized_decision_ready", + "confidence": "medium", + "missing_evidence": [ + "Confirm whether retained delivery_closeout continuation after completed_state is intended runtime behavior or a bug to fix during the slice." + ], + "seq": 15, + "judgment_hash": "sha256:28a47aa43f99a3b7eaee7532aad438fe97226fc42ffae69aafddd11954dcb243" + } + ] +} diff --git a/docs/research/2026-04-02_drop-delivery-skills.json b/docs/research/2026-04-02_drop-delivery-skills.json new file mode 100644 index 00000000..f4c2f0e5 --- /dev/null +++ b/docs/research/2026-04-02_drop-delivery-skills.json @@ -0,0 +1,152 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-02_drop-delivery-skills", + "question": "What runtime-native design should replace the delivery skills if decodex discards them completely?", + "success_criteria": [ + "Eliminate delivery-prepare and delivery-closeout as workflow skills.", + "Keep post-review landing and closeout machine-checkable inside decodex.", + "Preserve explicit closeout semantics without relying on skill-local commit-message workflows.", + "Produce a migration shape that fits the current orchestrator, tracker bridge, and workflow policy surfaces." + ], + "constraints": [ + "Use the current checkout as the primary evidence source.", + "Do not require delivery-specific skills or skill shims in the target design.", + "Keep tracker writes issue-scoped and aligned with the existing decodex runtime ownership model.", + "Do not depend on branch-name or PR-title heuristics for delivery decisions." + ], + "stop_rule": "Stop once the replacement design is concrete enough to implement or once the bounded pass is blocked by a required worker gate.", + "primary_hypothesis": "The delivery skills should be replaced by a decodex-owned delivery protocol: workflow policy declares the closeout target and merge behavior, the runtime owns post-review state transitions and validation, and the agent uses only issue-scoped tracker tools plus repo-native git and GitHub actions.", + "rival_hypotheses": [ + "A thin delivery skill shim is still required as the user-facing entrypoint even after the protocol moves into decodex.", + "The existing delivery commit-message contract should remain the primary authority even if the runtime owns closeout." + ], + "falsifiers": [ + "If decodex lacks the runtime surfaces needed to validate merged PR lineage and completed tracker state, removing the skills would create an unowned gap.", + "If merge policy still requires a delivery-specific commit contract that cannot move into runtime or workflow policy, deleting the skills would remove essential authority." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "stop", + "attempt": 1, + "max_attempts": 1, + "session_id": "2026-04-02_drop-delivery-skills" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The normative post-review lifecycle already treats `delivery_closeout` as a retained-lane phase whose meaning must not depend on a helper name, so the domain boundary already belongs to decodex rather than to a skill wrapper.", + "source_family": "repo_spec" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The orchestrator already has a dedicated `DeliveryCloseout` dispatch mode plus runtime validation for GitHub availability and retained PR lineage, which shows the runtime already models delivery closeout as first-class execution state.", + "source_family": "repo_code" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The tracker tool bridge already exposes issue-scoped transition and comment tools and validates that delivery closeout only completes after the merged PR matches the retained lane and the issue has reached the resolved completed state.", + "source_family": "repo_code" + }, + { + "id": "E4", + "kind": "inference", + "summary": "The current delivery skills duplicate runtime concerns by carrying closeout semantics, merge-history semantics, and tracker-sync rules in skill prose even though the runtime and workflow specs already own the same lifecycle boundary.", + "source_family": "repo_code" + }, + { + "id": "E5", + "kind": "observation", + "summary": "The current delivery-prepare skill adds extra report-generation and commit-message-contract obligations that are not required by the tracker tool bridge or the retained-lane runtime model.", + "source_family": "repo_code" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Internalizing delivery into decodex removes duplicated skill/runtime authority, but decodex must then own a small explicit delivery protocol surface in workflow policy and prompt contracts instead of relying on skill prose.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E4" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Dropping the delivery commit-message contract simplifies commit and push flow, but any semantics currently carried by `delivery_mode` and typed refs must move to runtime-owned state or issue-scoped tool calls before merge and closeout.", + "supporting_evidence_ids": [ + "E3", + "E5" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "If delivery becomes runtime-native, merge policy should derive from PR state plus workflow policy rather than from whether commit subjects happen to match a delivery JSON schema.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E5" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "runtime-native-delivery-protocol", + "rejected_options": [ + "keep-delivery-skills-as-thin-shims", + "keep-delivery-commit-contract-as-primary-authority" + ], + "decision_claim": "Discard the delivery skills entirely and replace them with a decodex-owned delivery protocol: workflow policy declares the resolved completed state and merge/closeout policy, the runtime owns landing and closeout validation, and the coding agent uses only issue-scoped tracker tools plus repo-native git and GitHub actions.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3" + ] + }, + "judgment_hash": "sha256:dec502b46d7fb2c49715ccd5ff9d8b3531c068c20bf312c0ce0496135e4b6b09" + }, + { + "seq": 5, + "type": "finalized_error", + "reason_code": "worker_unavailable", + "details": "The research plugin requires a skeptic worker for the current judgment candidate, but this turn did not include explicit child-agent authorization, so the bounded run could not complete the required adversarial challenge.", + "failed_workers": [ + "skeptic" + ] + } + ] +} diff --git a/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent-decision.json b/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent-decision.json new file mode 100644 index 00000000..be9cc7f1 --- /dev/null +++ b/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent-decision.json @@ -0,0 +1,233 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-02_plan-plugin-to-progress-skill-multiagent-decision", + "question": "Should decodex remove the plan plugin and replace it with one Linear-backed development progress state skill optimized for AI/agent/LLM execution state?", + "success_criteria": [ + "Determine whether removing the plan plugin is preferable to keeping or simplifying it.", + "Use explicit child-agent evaluation and challenge passes before finalizing.", + "Define the replacement as an issue-scoped Linear execution-state skill that preserves lifecycle boundaries." + ], + "constraints": [ + "Keep the routed Linear issue description as a generic briefing surface rather than plugin-private state.", + "Prefer one skill over a writing/execution split unless evidence shows the split is required.", + "Do not let the replacement skill substitute for review handoff, delivery closeout, or terminal finalize." + ], + "stop_rule": "Stop once the repository evidence plus explicit analyst and skeptic passes support a decision-ready recommendation or show that removing the plan plugin would leave unresolved lifecycle gaps.", + "primary_hypothesis": "The current repo needs one Linear-backed development progress state skill more than it needs a separate plan plugin, so the best fit is to remove the plan plugin and replace it with an issue-scoped progress-state skill.", + "rival_hypotheses": [ + "The plan plugin should stay and only be simplified because the runtime still needs durable strategy authority.", + "The best fit is to keep the plan plugin internals but collapse writing and execution into one skill instead of removing it." + ], + "falsifiers": [ + "If the runtime depends on linked-plan lineage or linked-plan documents for normal execution continuity, removing the plan plugin is insufficient.", + "If an issue-scoped Linear checkpoint cannot cover the state an agent needs for retries, verification, and review repair, the replacement skill is insufficient." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 2, + "session_id": "2026-04-02_plan-plugin-to-progress-skill-multiagent-decision" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "Does normal decodex execution actually require durable strategy authority beyond issue-scoped execution checkpoints?", + "Would one Linear-backed progress-state skill fit the runtime better than either keeping plan or collapsing writing and execution into one skill?" + ], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The runtime and README both position tracked planning as an execution overlay inside the retained lane rather than as decodex's primary lifecycle authority.", + "source_family": "repo_docs" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The runtime says Linear is the source of truth for issue lifecycle and coarse outcomes, forbids a second long-lived business workflow model outside Linear, and requires the issue description to remain a generic briefing surface.", + "source_family": "repo_docs" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The tracker tool contract already establishes an issue-scoped structured-checkpoint pattern via issue_review_checkpoint, which decodex treats as the only authoritative structured review-policy signal.", + "source_family": "repo_docs" + }, + { + "id": "E4", + "kind": "observation", + "summary": "The current plan plugin is explicitly split into writing and execution, with linked tracked-plan documents, append-only plan-event comments, and cross-surface validation before execution can proceed.", + "source_family": "repo_code" + }, + { + "id": "E5", + "kind": "observation", + "summary": "The current plan plugin surface is about 2082 lines across skills, templates, and contract helpers, which is large for a feature now described as an execution overlay.", + "source_family": "repo_code" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Prior bounded research on the plan plugin concluded that the design still had unresolved runtime task-state, write-skew, lineage, and plan-local-done semantic gaps.", + "source_family": "repo_docs" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Removing the plan plugin aligns the common path with decodex's existing issue-scoped tracker authority and reduces control-plane complexity, but it gives up durable strategy lineage across replans.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E5" + ], + "disconfirming_evidence_ids": [ + "E4" + ] + }, + { + "id": "T2", + "summary": "Keeping the plan plugin preserves a stronger durable planning contract for rare multi-session planning cases, but it forces a dual-surface strategy/runtime model onto the common path and carries known semantic debt.", + "supporting_evidence_ids": [ + "E4", + "E6" + ], + "disconfirming_evidence_ids": [ + "E1" + ] + }, + { + "id": "T3", + "summary": "Collapsing writing and execution into one skill would reduce entrypoints, but it would leave the linked-document plus event-log machinery mostly intact and therefore would not remove most of the complexity.", + "supporting_evidence_ids": [ + "E4", + "E5", + "E6" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "inventory_updated", + "remaining_option_count": 2, + "independent_option_questions": [ + "Which option better matches decodex's existing authority boundaries: removing plan in favor of one issue-scoped Linear state skill, or keeping a durable planning subsystem?", + "Which option better minimizes common-path complexity without losing execution continuity the runtime actually depends on?" + ], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6" + ] + }, + { + "seq": 5, + "type": "worker_completed", + "worker": "analyst", + "summary": "The analyst preferred removing the plan plugin in favor of one Linear-backed development progress state skill because it better matches decodex's existing Linear authority and issue-scoped checkpoint model, while the strongest counterargument is that removing plan also removes durable multi-step planning authority for minority replan-heavy workflows. Confidence: 0.81.", + "target_inventory_seq": 4 + }, + { + "seq": 6, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E7", + "kind": "observation", + "summary": "The analyst found that option 1 fits better because decodex already centers Linear lifecycle authority and issue-scoped checkpoints, so a single Linear-backed progress-state skill aligns with the existing architecture more directly than a separate plan subsystem.", + "source_family": "repo_docs" + }, + { + "id": "E8", + "kind": "observation", + "summary": "The analyst's strongest counterargument is that deleting plan also deletes a real capability: durable multi-step plan authority and replan lineage for minority workflows, so the simplification is a deliberate scope reduction rather than a free cleanup.", + "source_family": "repo_docs" + } + ] + }, + { + "seq": 7, + "type": "inventory_updated", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7", + "E8" + ] + }, + { + "seq": 8, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "remove-plan-plugin-and-adopt-linear-progress-skill", + "rejected_options": [ + "keep-plan-plugin-and-simplify-it", + "collapse-plan-writing-and-execution-into-one-skill" + ], + "decision_claim": "Remove the plan plugin and replace it with one Linear-backed development progress state skill. The replacement should manage only issue-scoped execution state on the current Linear issue, keep the issue description as generic briefing, reuse the issue-scoped checkpoint pattern rather than overloading issue_review_checkpoint itself, and stay strictly below decodex lifecycle authority so it cannot substitute for review handoff, delivery closeout, or terminal finalize.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3" + ] + }, + "judgment_hash": "sha256:bfe3105d292798eb2c3ec8ac8b72cd9c1b744ca917cc233c4f058053b378c135" + }, + { + "seq": 9, + "type": "worker_completed", + "worker": "skeptic", + "target_judgment_hash": "sha256:bfe3105d292798eb2c3ec8ac8b72cd9c1b744ca917cc233c4f058053b378c135", + "summary": "The strongest objection is non-blocking: the replacement must not overload issue_review_checkpoint itself. The judgment remains valid if the new skill uses a separate issue-scoped progress overlay on the current issue, keeps the issue description generic, and preserves issue_review_handoff, issue_terminal_finalize, and delivery_closeout as the only authoritative lifecycle signals.", + "objections": [] + }, + { + "seq": 10, + "type": "finalized_decision_ready", + "judgment_hash": "sha256:bfe3105d292798eb2c3ec8ac8b72cd9c1b744ca917cc233c4f058053b378c135", + "confidence": "medium", + "missing_evidence": [ + "The exact field list and write protocol for the replacement development progress state skill still need to be specified.", + "A migration plan for removing the plan plugin and replacing any current callers still needs to be written." + ] + } + ] +} diff --git a/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent.json b/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent.json new file mode 100644 index 00000000..4e5da824 --- /dev/null +++ b/docs/research/2026-04-02_plan-plugin-to-progress-skill-multiagent.json @@ -0,0 +1,224 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-02_plan-plugin-to-progress-skill-multiagent", + "question": "Should decodex remove the plan plugin and replace it with one Linear-backed development progress state skill optimized for AI/agent/LLM execution state?", + "success_criteria": [ + "Determine whether removing the plan plugin is preferable to keeping or simplifying it.", + "Use explicit child-agent evaluation and challenge passes before finalizing.", + "Define the replacement as an issue-scoped Linear execution-state skill that preserves lifecycle boundaries." + ], + "constraints": [ + "Keep the routed Linear issue description as a generic briefing surface rather than plugin-private state.", + "Prefer one skill over a writing/execution split unless evidence shows the split is required.", + "Do not let the replacement skill substitute for review handoff, delivery closeout, or terminal finalize." + ], + "stop_rule": "Stop once the repository evidence plus explicit analyst and skeptic passes support a decision-ready recommendation or show that removing the plan plugin would leave unresolved lifecycle gaps.", + "primary_hypothesis": "The current repo needs one Linear-backed development progress state skill more than it needs a separate plan plugin, so the best fit is to remove the plan plugin and replace it with an issue-scoped progress-state skill.", + "rival_hypotheses": [ + "The plan plugin should stay and only be simplified because the runtime still needs durable strategy authority.", + "The best fit is to keep the plan plugin internals but collapse writing and execution into one skill instead of removing it." + ], + "falsifiers": [ + "If the runtime depends on linked-plan lineage or linked-plan documents for normal execution continuity, removing the plan plugin is insufficient.", + "If an issue-scoped Linear checkpoint cannot cover the state an agent needs for retries, verification, and review repair, the replacement skill is insufficient." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 2, + "session_id": "2026-04-02_plan-plugin-to-progress-skill-multiagent" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "Does normal decodex execution actually require durable strategy authority beyond issue-scoped execution checkpoints?", + "Would one Linear-backed progress-state skill fit the runtime better than either keeping plan or collapsing writing and execution into one skill?" + ], + "external_slices": [] + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The runtime and README both position tracked planning as an execution overlay inside the retained lane rather than as decodex's primary lifecycle authority.", + "source_family": "repo_docs" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The runtime says Linear is the source of truth for issue lifecycle and coarse outcomes, forbids a second long-lived business workflow model outside Linear, and requires the issue description to remain a generic briefing surface.", + "source_family": "repo_docs" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The tracker tool contract already establishes an issue-scoped structured-checkpoint pattern via issue_review_checkpoint, which decodex treats as the only authoritative structured review-policy signal.", + "source_family": "repo_docs" + }, + { + "id": "E4", + "kind": "observation", + "summary": "The current plan plugin is explicitly split into writing and execution, with linked tracked-plan documents, append-only plan-event comments, and cross-surface validation before execution can proceed.", + "source_family": "repo_code" + }, + { + "id": "E5", + "kind": "observation", + "summary": "The current plan plugin surface is about 2082 lines across skills, templates, and contract helpers, which is large for a feature now described as an execution overlay.", + "source_family": "repo_code" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Prior bounded research on the plan plugin concluded that the design still had unresolved runtime task-state, write-skew, lineage, and plan-local-done semantic gaps.", + "source_family": "repo_docs" + } + ], + "seq": 2 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Removing the plan plugin aligns the common path with decodex's existing issue-scoped tracker authority and reduces control-plane complexity, but it gives up durable strategy lineage across replans.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E5" + ], + "disconfirming_evidence_ids": [ + "E4" + ] + }, + { + "id": "T2", + "summary": "Keeping the plan plugin preserves a stronger durable planning contract for rare multi-session planning cases, but it forces a dual-surface strategy/runtime model onto the common path and carries known semantic debt.", + "supporting_evidence_ids": [ + "E4", + "E6" + ], + "disconfirming_evidence_ids": [ + "E1" + ] + }, + { + "id": "T3", + "summary": "Collapsing writing and execution into one skill would reduce entrypoints, but it would leave the linked-document plus event-log machinery mostly intact and therefore would not remove most of the complexity.", + "supporting_evidence_ids": [ + "E4", + "E5", + "E6" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 3 + }, + { + "type": "inventory_updated", + "remaining_option_count": 2, + "independent_option_questions": [ + "Which option better matches decodex's existing authority boundaries: removing plan in favor of one issue-scoped Linear state skill, or keeping a durable planning subsystem?", + "Which option better minimizes common-path complexity without losing execution continuity the runtime actually depends on?" + ], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6" + ], + "seq": 4 + }, + { + "type": "worker_completed", + "worker": "analyst", + "summary": "The analyst preferred removing the plan plugin in favor of one Linear-backed development progress state skill because it better matches decodex's existing Linear authority and issue-scoped checkpoint model, while the strongest counterargument is that removing plan also removes durable multi-step planning authority for minority replan-heavy workflows. Confidence: 0.81.", + "target_inventory_seq": 4, + "seq": 5 + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E7", + "kind": "observation", + "summary": "The analyst found that option 1 fits better because decodex already centers Linear lifecycle authority and issue-scoped checkpoints, so a single Linear-backed progress-state skill aligns with the existing architecture more directly than a separate plan subsystem.", + "source_family": "repo_docs" + }, + { + "id": "E8", + "kind": "observation", + "summary": "The analyst's strongest counterargument is that deleting plan also deletes a real capability: durable multi-step plan authority and replan lineage for minority workflows, so the simplification is a deliberate scope reduction rather than a free cleanup.", + "source_family": "repo_docs" + } + ], + "seq": 6 + }, + { + "type": "inventory_updated", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7", + "E8" + ], + "seq": 7 + }, + { + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "remove-plan-plugin-and-adopt-linear-progress-skill", + "rejected_options": [ + "keep-plan-plugin-and-simplify-it", + "collapse-plan-writing-and-execution-into-one-skill" + ], + "decision_claim": "Remove the plan plugin and replace it with one Linear-backed development progress state skill. The replacement should manage only issue-scoped execution state on the current Linear issue, keep the issue description as generic briefing, reuse the checkpoint pattern already established by issue_review_checkpoint, and stay strictly below decodex lifecycle authority so it cannot substitute for review handoff, delivery closeout, or terminal finalize.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3" + ] + }, + "seq": 8, + "judgment_hash": "sha256:5fd11b97dd08ac25be36c93ce15d03e2bd1e649cf0fd58ad669328dc22137f71" + }, + { + "type": "finalized_error", + "reason_code": "worker_timeout", + "details": "The explicit skeptic child-agent dispatch for the current judgment did not produce a collected result within the bounded collection window, so the multiagent research run cannot be finalized as decision-ready.", + "failed_workers": [ + "skeptic" + ], + "seq": 9 + } + ] +} diff --git a/docs/research/2026-04-02_plan-plugin-to-progress-skill.json b/docs/research/2026-04-02_plan-plugin-to-progress-skill.json new file mode 100644 index 00000000..b85fe2dc --- /dev/null +++ b/docs/research/2026-04-02_plan-plugin-to-progress-skill.json @@ -0,0 +1,206 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-02_plan-plugin-to-progress-skill", + "question": "Should decodex remove the plan plugin and replace it with one Linear-backed development progress state skill optimized for AI/agent/LLM execution state?", + "success_criteria": [ + "Determine whether removing the plan plugin is preferable to keeping or simplifying it.", + "Define the smallest Linear-backed state-management surface that fits the current decodex lifecycle.", + "Preserve lifecycle authority boundaries so progress state cannot replace review handoff, closeout, or terminal finalize." + ], + "constraints": [ + "Keep the routed Linear issue description as a generic briefing surface rather than plugin-private state.", + "Prefer one skill over a writing/execution split unless evidence shows the split is required.", + "Do not require linked-plan documents or a separate durable strategy authority unless the runtime truly needs them." + ], + "stop_rule": "Stop once the repository evidence supports a decision-ready recommendation or shows that removing the plan plugin would leave unresolved lifecycle gaps.", + "primary_hypothesis": "The current repo needs a lightweight Linear-backed execution-state skill more than it needs a separate plan plugin, so the best fit is to remove the plan plugin and replace it with one issue-scoped progress-state skill.", + "rival_hypotheses": [ + "The plan plugin should stay and only be simplified because the runtime still needs durable strategy authority.", + "The best fit is to keep the plan plugin internals but collapse writing and execution into one skill instead of removing it." + ], + "falsifiers": [ + "If the current runtime depends on linked-plan revision lineage for normal execution continuity, removing the plan plugin is insufficient.", + "If a lightweight Linear-backed checkpoint cannot represent the state needed for retries, review repair, and verification boundaries, the replacement skill is insufficient." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 2, + "session_id": "2026-04-02_plan-plugin-to-progress-skill" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "Does the runtime need durable strategy authority beyond execution-state checkpoints for normal issue execution?", + "Can one issue-scoped Linear checkpoint cover the state transitions that agents actually need without reintroducing plan complexity?" + ], + "external_slices": [] + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The runtime and README both position tracked planning as an execution overlay inside the retained lane rather than as decodex's primary lifecycle authority.", + "source_family": "repo_docs" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The runtime says Linear is the source of truth for issue lifecycle and coarse outcomes, forbids a second long-lived business workflow model outside Linear, and requires the issue description to remain a generic briefing surface.", + "source_family": "repo_docs" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The tracker tool contract already establishes an issue-scoped structured-checkpoint pattern via issue_review_checkpoint, which decodex treats as the only authoritative structured review-policy signal.", + "source_family": "repo_docs" + }, + { + "id": "E4", + "kind": "observation", + "summary": "The current plan plugin is explicitly split into writing and execution, with linked tracked-plan documents, append-only plan-event comments, and cross-surface validation before execution can proceed.", + "source_family": "repo_code" + }, + { + "id": "E5", + "kind": "observation", + "summary": "The current plan plugin surface is about 2082 lines across skills, templates, and contract helpers, which is large for a feature now described as an execution overlay.", + "source_family": "repo_code" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Prior bounded research on the plan plugin concluded that the design still had unresolved runtime task-state, write-skew, lineage, and plan-local-done semantic gaps.", + "source_family": "repo_docs" + }, + { + "id": "E7", + "kind": "inference", + "summary": "The repo's higher-frequency need is a lightweight issue-scoped execution-state contract on Linear rather than a separate durable strategy-authority subsystem.", + "source_family": "repo_docs" + } + ], + "seq": 2 + }, + { + "type": "inventory_updated", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [], + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7" + ], + "seq": 3 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Removing the plan plugin aligns the common path with decodex's existing issue-scoped tracker authority and reduces control-plane complexity, but it gives up durable strategy lineage across replans.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E5" + ], + "disconfirming_evidence_ids": [ + "E4" + ] + }, + { + "id": "T2", + "summary": "A single Linear-backed progress-state skill fits the existing checkpoint pattern and the agent's real execution needs, but it must stay below lifecycle authority so it cannot replace review handoff, closeout, or terminal finalize.", + "supporting_evidence_ids": [ + "E2", + "E3", + "E7" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Keeping the plan plugin preserves a stronger durable planning contract for rare multi-session planning cases, but it forces a dual-surface strategy/runtime model onto the common path and carries known semantic debt.", + "supporting_evidence_ids": [ + "E4", + "E6" + ], + "disconfirming_evidence_ids": [ + "E1", + "E7" + ] + }, + { + "id": "T4", + "summary": "Collapsing writing and execution into one skill would reduce surface count, but most of the complexity is in the linked-document plus event-log contract rather than in the number of skill entrypoints.", + "supporting_evidence_ids": [ + "E4", + "E5", + "E6" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 4 + }, + { + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "remove-plan-plugin-and-adopt-linear-progress-skill", + "rejected_options": [ + "keep-plan-plugin-and-simplify-it", + "collapse-plan-writing-and-execution-into-one-skill" + ], + "decision_claim": "Remove the plan plugin and replace it with one Linear-backed development progress state skill. The new skill should manage only issue-scoped execution state on the current Linear issue, keep the issue description as generic briefing, reuse the checkpoint pattern already established by issue_review_checkpoint, and stay strictly below decodex lifecycle authority so it cannot substitute for review handoff, delivery closeout, or terminal finalize.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E5", + "E6", + "E7" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3", + "T4" + ] + }, + "seq": 5, + "judgment_hash": "sha256:6f37876de0e33898f204fe56e1144418894233df0ab79337b043f8e5956acebd" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The strongest objection is that removing the plan plugin also removes durable strategy lineage for rare replan-heavy workflows, but the current repo evidence shows that normal decodex execution does not depend on that authority. The replacement remains safe if the new skill is limited to issue-scoped execution state on Linear and cannot substitute for lifecycle signals such as review handoff or terminal finalize.", + "objections": [], + "seq": 6, + "target_judgment_hash": "sha256:6f37876de0e33898f204fe56e1144418894233df0ab79337b043f8e5956acebd" + }, + { + "type": "finalized_decision_ready", + "confidence": "medium", + "missing_evidence": [ + "The exact field list and write protocol for the replacement development progress state skill still need to be specified.", + "A migration plan for removing the plan plugin and replacing any current callers still needs to be written." + ], + "seq": 7, + "judgment_hash": "sha256:6f37876de0e33898f204fe56e1144418894233df0ab79337b043f8e5956acebd" + } + ] +} diff --git a/docs/research/2026-04-03_machine-first-landing-receipt.json b/docs/research/2026-04-03_machine-first-landing-receipt.json new file mode 100644 index 00000000..9a8ea746 --- /dev/null +++ b/docs/research/2026-04-03_machine-first-landing-receipt.json @@ -0,0 +1,309 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-03_machine-first-landing-receipt", + "question": "How should Decodex encode machine-readable landing authority when the repository wants LLM-first history, does not care about human readability, and does not want empty commits for fast-forward landings?", + "success_criteria": [ + "Preserve machine-readable history that LLMs can inspect directly from Git.", + "Avoid squash-based loss of commit-level authority.", + "Avoid requiring empty commits when a landing can fast-forward cleanly." + ], + "constraints": [ + "Design for repository-local Git authority rather than markdown authority.", + "Assume human-readable history is not a goal.", + "Keep the protocol aligned with fully autonomous Decodex-controlled landing." + ], + "stop_rule": "Stop once one protocol shape is decision-ready with bounded repo-grounded evidence and a skeptic pass.", + "primary_hypothesis": "The best protocol separates content authority from landing receipt authority: keep machine-readable commit history on normal commits and record landing as a separate machine-readable receipt attached to the landed head without forcing a merge or empty commit.", + "rival_hypotheses": [ + "Always require a merge commit carrying the landing contract, even when fast-forward would otherwise be valid.", + "Keep squash landing and accept that landed history is not the machine authority.", + "Store landing receipts in git notes instead of a first-class Git object." + ], + "falsifiers": [ + "If the receipt cannot be fetched and enumerated reliably by default Git flows, the design is insufficient.", + "If the protocol still forces an artificial commit on a clean fast-forward landing, the design is insufficient.", + "If the protocol makes landed machine history less inspectable than the current commit history, the design is insufficient." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 1, + "session_id": "2026-04-03_machine-first-landing-receipt" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 1, + "independent_option_questions": [ + "Which Git-native object should carry landing receipt authority without forcing an empty commit on fast-forward landings?" + ], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The current repository workflow now sets `[landing].merge_method = \"squash\"`, making GitHub-generated landed commit messages the effective mainline history shape instead of preserving machine-readable commit authority.", + "source_family": "repo_code" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The merged mainline commit for PR #40 was rewritten by GitHub into a title-plus-bullets message, and the original single-line `delivery/1` JSON survived only as a bullet in the squash body rather than as the landed authority surface.", + "source_family": "repo_code" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The repository allows merge commits, squash merges, and rebase merges, so preserving machine-readable landed history does not require squash and is not constrained to a single GitHub merge mode.", + "source_family": "repo_code" + }, + { + "id": "E4", + "kind": "inference", + "summary": "If landing authority is attached to the landed head as a separate first-class Git object, fast-forward landings can remain fast-forward while still carrying an explicit machine-readable receipt.", + "source_family": "repo_code" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Binding landing authority to a merge commit preserves everything in commit history, but it forces an otherwise unnecessary commit object whenever fast-forward landing is valid.", + "supporting_evidence_ids": [ + "E2", + "E4" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Keeping commit history as the primary authority and storing a separate landing receipt preserves fast-forward behavior, but the receipt carrier must still be Git-native and easy for Decodex to enumerate.", + "supporting_evidence_ids": [ + "E1", + "E3", + "E4" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "commit-history-plus-annotated-tag-receipt", + "rejected_options": [ + "always-merge-commit-land-receipt", + "git-notes-landing-receipt", + "squash-based-landing" + ], + "decision_claim": "Use machine-readable per-commit history as the primary authority and represent landing as a separate receipt, preferably an annotated tag attached to the landed head. Preserve fast-forward when possible, avoid squash entirely, and never require an empty commit just to encode landing metadata.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4" + ], + "key_tradeoff_ids": [ + "T1", + "T2" + ] + }, + "judgment_hash": "sha256:00db9e7beed46597b414e7ea5bc03bf2d6e3a9c9d7b617280b1650b3d4517051" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The original candidate was directionally right but not yet repo-grounded enough: this repo still sets `[landing].merge_method = \"squash\"`, and neither workflow contract nor runtime currently define receipt-tag authority or enumeration rules.", + "objections": [ + { + "id": "O1", + "summary": "Current repo policy still sets `[landing].merge_method = \"squash\"`, so the recommendation cannot be adopted immediately without an explicit workflow/spec policy change." + }, + { + "id": "O2", + "summary": "Annotated-tag receipts are not yet part of the repo-grounded contract; workflow/spec/runtime still need explicit receipt-tag fields, naming, fetch, and validation rules." + } + ], + "seq": 5, + "target_judgment_hash": "sha256:00db9e7beed46597b414e7ea5bc03bf2d6e3a9c9d7b617280b1650b3d4517051" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E5", + "kind": "contradiction", + "summary": "The active repo contract still says `[landing].merge_method = \"squash\"`, so the machine-first receipt design is not an immediate description of current behavior; it is a proposed replacement contract.", + "source_family": "repo_code" + }, + { + "id": "E6", + "kind": "missing_evidence", + "summary": "The repository does not yet define how `decodex/land/1` receipt tags are named, fetched, enumerated, validated, or cleaned up, so those rules must be added before the design can be implemented safely.", + "source_family": "repo_code" + } + ], + "seq": 6 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T3", + "summary": "Annotated-tag landing receipts solve the fast-forward empty-commit problem only if the repo contract explicitly teaches Decodex where receipts live and how to fetch and validate them; otherwise the design remains under-specified.", + "supporting_evidence_ids": [ + "E5", + "E6" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 7 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "O1", + "O2" + ], + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "machine-first-ff-or-merge-with-annotated-tag-receipt", + "rejected_options": [ + "always-merge-commit-land-receipt", + "git-notes-landing-receipt", + "squash-based-landing" + ], + "decision_claim": "Adopt a machine-first landing contract in two layers: keep machine-readable per-commit history as the primary authority, and add a separate `decodex/land/1` landing receipt attached to the landed head as an annotated tag. To make that design valid in this repo, change `[landing]` away from `squash`, add explicit workflow/spec support for receipt tags and their fetch/enumeration rules, and teach Decodex runtime to create and validate them for both fast-forward and merge landings without requiring empty commits.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3" + ] + }, + "judgment_hash": "sha256:7cd311b70536a0903e8d166b24089f350f98b756da8fbc1e8d7340ebb5084a08", + "seq": 8 + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The revised candidate still needs one more repo-owned machine-readable policy surface: current `[landing]` only models merge, squash, or rebase, so an explicit fast-forward-vs-merge rule must be added rather than inferred.", + "objections": [ + { + "id": "O3", + "summary": "Add a machine-readable landing-mode field for `ff_then_merge`; current `[landing]` only models merge method, not the fast-forward-vs-merge choice." + } + ], + "seq": 9, + "target_judgment_hash": "sha256:7cd311b70536a0903e8d166b24089f350f98b756da8fbc1e8d7340ebb5084a08" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E7", + "kind": "observation", + "summary": "The current workflow contract only exposes `[landing].merge_method = merge|squash|rebase`, so a deterministic fast-forward-when-possible policy needs a new machine-readable field instead of runtime inference.", + "source_family": "repo_code" + } + ], + "seq": 10 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T4", + "summary": "Supporting fast-forward without empty commits requires a separate machine-readable landing-mode field, because merge method alone cannot express 'ff when possible, otherwise merge'.", + "supporting_evidence_ids": [ + "E7" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 11 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "O3" + ], + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "machine-first-landing-with-ff-then-merge-and-annotated-tag-receipt", + "rejected_options": [ + "always-merge-commit-land-receipt", + "git-notes-landing-receipt", + "squash-based-landing" + ], + "decision_claim": "Design the protocol in three explicit layers. First, keep machine-readable per-commit history as the primary authority. Second, add `[landing].mode = \"ff_then_merge\"` so repository policy can say 'fast-forward when possible, otherwise create a merge commit', and treat existing `merge_method` as the fallback merge style when fast-forward is not available. Third, add `decodex/land/1` as a landing receipt carried by an annotated tag attached to the landed head, with explicit workflow/spec/runtime rules for naming, fetch, enumeration, validation, and cleanup. This preserves machine-readable history, avoids squash, and never requires an empty commit for fast-forward landings.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6", + "E7" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3", + "T4" + ] + }, + "judgment_hash": "sha256:b50de0fb7daaf465d94cc86e6e15822ff55d9d067451a0e48be6b8f0ff91204a", + "seq": 12 + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The protocol direction is coherent, but two machine-authoritative contract pieces are still unspecified: the durable per-commit schema itself and the contract rules between a new fast-forward landing mode and the existing merge-method field.", + "objections": [ + { + "id": "O4", + "summary": "The repository no longer defines a durable machine-readable per-commit schema, so layer 1 needs an explicit replacement contract rather than relying on an ad hoc restored `delivery/1` commit." + }, + { + "id": "O5", + "summary": "A new `[landing].mode = \"ff_then_merge\"` field also needs explicit contract rules with `merge_method`, including which combinations are valid and which ones are forbidden." + } + ], + "seq": 13, + "target_judgment_hash": "sha256:b50de0fb7daaf465d94cc86e6e15822ff55d9d067451a0e48be6b8f0ff91204a" + }, + { + "type": "finalized_not_decision_ready", + "reason": "Bounded research found a strong direction, but the commit-level schema and landing-mode contract matrix still need explicit contract design before the protocol is safe to adopt.", + "missing_evidence": [ + "Choose the durable per-commit schema that replaces the deleted delivery contract, including required fields and authority semantics.", + "Define the exact contract rules between `[landing].mode` and `merge_method`, including which fallback merge styles remain valid when fast-forward is unavailable.", + "Specify how `decodex/land/1` annotated-tag receipts are named, fetched, enumerated, validated, and cleaned up." + ], + "seq": 14 + } + ] +} diff --git a/docs/research/2026-04-05_workspace-lifecycle-hooks-surface.json b/docs/research/2026-04-05_workspace-lifecycle-hooks-surface.json new file mode 100644 index 00000000..f330a029 --- /dev/null +++ b/docs/research/2026-04-05_workspace-lifecycle-hooks-surface.json @@ -0,0 +1,182 @@ +{ + "schema": "research-run/2", + "run_id": "2026-04-05_workspace-lifecycle-hooks-surface", + "question": "What is the smallest repo-owned workspace lifecycle hook surface that fits Decodex's goals without duplicating Codex host hooks or turning WORKFLOW.md into a generic plugin system?", + "success_criteria": [ + "Recommend a lifecycle-hook contract that is easy for repo owners to understand and audit.", + "Preserve Decodex runtime ownership of lane lifecycle while allowing limited repo-specific bootstrap and cleanup behavior.", + "Avoid duplicating host-level Codex controls or introducing a generic script escape hatch." + ], + "constraints": [ + "Keep the first slice aligned with the current WORKFLOW.md machine-readable style.", + "Prefer fail-closed behavior for destructive or state-changing phases.", + "Do not widen the design into a run-hook framework, plugin system, or background-service launcher." + ], + "stop_rule": "Stop once one narrow hook contract is decision-ready with repo-grounded and official-doc-backed tradeoffs.", + "primary_hypothesis": "The best first slice is an extensible workspace hook schema that only enables after_create and before_remove, leaving run-phase hooks out of scope until there is concrete repo pressure for them.", + "rival_hypotheses": [ + "Expose a fully generic lifecycle hook surface now, including before_run and after_run, so repos do not need another schema change later.", + "Do not add any repo-owned lifecycle hooks because host-level Codex hooks or runtime logic should absorb the need." + ], + "falsifiers": [ + "If current repo-specific bootstrap and cleanup needs cannot be expressed by after_create and before_remove, the proposed first slice is too narrow.", + "If the proposed hook surface lets repos smuggle orchestration policy, review policy, or host controls back into shell scripts, the design is too broad." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 1, + "session_id": "2026-04-05_workspace-lifecycle-hooks-surface" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 3, + "independent_option_questions": [ + "Should Decodex expose only workspace bootstrap and cleanup hooks, or also run-phase hooks such as before_run and after_run?", + "What boundary keeps repo-owned lifecycle policy from overlapping host-level Codex controls?" + ], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The current workflow contract already prefers narrow, explicit, fail-closed machine-readable surfaces such as named gate profiles rather than a generic execution DSL.", + "source_family": "repo_spec" + }, + { + "id": "E2", + "kind": "observation", + "summary": "Decodex runtime already owns linked-worktree lane planning, creation, reuse, cleanup, retries, and reconciliation; repo policy should not replace that lifecycle authority.", + "source_family": "repo_spec" + }, + { + "id": "E3", + "kind": "observation", + "summary": "The current repo has no machine-readable WORKFLOW.md surface for repo-specific worktree bootstrap or pre-removal cleanup, so those behaviors still tend to live in runtime code or ad hoc local practice.", + "source_family": "repo_code" + }, + { + "id": "E4", + "kind": "observation", + "summary": "Kubernetes lifecycle hooks treat startup and shutdown hooks as explicit phase-bound handlers and emphasize that handlers may run more than once, so implementations should be idempotent and should not be treated as arbitrary orchestration logic.", + "source_family": "official_docs" + }, + { + "id": "E5", + "kind": "observation", + "summary": "systemd separates pre-start, start, stop, and post-stop actions explicitly, which keeps lifecycle sequencing understandable and keeps cleanup attached to shutdown semantics instead of general runtime scriptability.", + "source_family": "official_docs" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Dev Container lifecycle scripts provide a useful cautionary pattern: named phases are workable, but once many phases exist, users must reason about subtle ordering and re-entry behavior.", + "source_family": "official_docs" + }, + { + "id": "E7", + "kind": "observation", + "summary": "npm lifecycle scripts show the downside of a broad phase matrix: once many script slots exist, repo behavior becomes harder to audit and script hooks become a catch-all escape hatch instead of a narrow contract.", + "source_family": "official_docs" + }, + { + "id": "E8", + "kind": "observation", + "summary": "Local Codex configuration exposes host-level controls such as approval policy, sandbox mode, and a codex_hooks feature flag, but there is no repo-visible stable contract here for expressing repo-owned worktree lifecycle policy.", + "source_family": "local_env" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "A generic lifecycle hook system would reduce future schema additions, but it would immediately create ambiguity around attempt boundaries, continuation turns, review repair, and whether after_run failures should affect the primary run result.", + "supporting_evidence_ids": [ + "E2", + "E6", + "E7" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Restricting the first slice to after_create and before_remove captures the most legitimate repo-specific needs, namely workspace bootstrap and cleanup, without reopening review, landing, or retry policy through shell hooks.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E5" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Using an extensible schema with only two enabled phases keeps the architecture future-proof while still forcing the first release to earn additional phases through a separate design decision.", + "supporting_evidence_ids": [ + "E1", + "E6" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T4", + "summary": "Relying on Codex host hooks instead of WORKFLOW.md would hide repo policy in host configuration and make repo bootstrap or cleanup behavior non-portable and harder to audit in Git.", + "supporting_evidence_ids": [ + "E3", + "E8" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "recommend", + "preferred_option": "extensible-schema-with-only-after-create-and-before-remove", + "rejected_options": [ + "generic-run-and-workspace-hook-system", + "host-hook-only-no-workflow-surface" + ], + "decision_claim": "Add a repo-owned workspace hook surface under WORKFLOW.md [execution.workspace_hooks], but only enable after_create and before_remove in the first slice. Keep runtime ownership of lane lifecycle, run hooks out of scope, hook execution serial and fail-closed, and limit the surface to lightweight idempotent bootstrap and cleanup commands rooted in the worktree.", + "key_evidence_ids": [ + "E1", + "E2", + "E3", + "E4", + "E5", + "E6", + "E7", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3", + "T4" + ] + }, + "judgment_hash": "sha256:7d9c597ea687e8c72297cde8a65057f4dc7a0c1d8db6e0705463e5eeb0b4e5de" + }, + { + "seq": 5, + "type": "finalized_decision_ready", + "judgment_hash": "sha256:7d9c597ea687e8c72297cde8a65057f4dc7a0c1d8db6e0705463e5eeb0b4e5de", + "summary": "Decision-ready: use a narrow workspace bootstrap and cleanup hook contract, not a generic lifecycle hook framework." + } + ] +} diff --git a/docs/research/2026-05-03_codex-cloud-task-apis.json b/docs/research/2026-05-03_codex-cloud-task-apis.json new file mode 100644 index 00000000..4f000378 --- /dev/null +++ b/docs/research/2026-05-03_codex-cloud-task-apis.json @@ -0,0 +1,625 @@ +{ + "schema": "research-run/2", + "run_id": "2026-05-03_codex-cloud-task-apis", + "question": "Should Codex Cloud task APIs become part of Decodex execution modes, and under what constraints?", + "success_criteria": [ + "Map Codex Cloud task list, create, status, diff, and apply capabilities from source.", + "Compare those capabilities with Decodex's local app-server execution model and owned-lane lifecycle.", + "End with a decision-ready adopt, defer, or reject judgment while keeping local execution authoritative for the MVP." + ], + "constraints": [ + "No account-pool implementation changes.", + "No replacement of local codex app-server execution.", + "No credential or account-pool schema changes.", + "Any later implementation must be split into a separate issue." + ], + "stop_rule": "Stop once source evidence is sufficient to decide adopt, defer, or reject for Decodex's current execution model.", + "primary_hypothesis": "Codex Cloud task APIs are useful future remote-execution inputs, but should be deferred from Decodex until a separate remote-execution issue can preserve owned-lane lifecycle authority.", + "rival_hypotheses": [ + "Adopt Cloud tasks now as an alternate Decodex execution backend.", + "Reject Cloud task APIs permanently because they do not match Decodex's local app-server model." + ], + "falsifiers": [ + "If Cloud task APIs already expose Decodex-equivalent issue ownership, repo gate, and PR handoff semantics, immediate adoption would be viable.", + "If Cloud task APIs cannot expose task status, diff, or local apply primitives at all, future integration should be rejected rather than deferred.", + "If Decodex's local app-server model no longer owns execution, the local-MVP constraint would be obsolete." + ], + "coverage": { + "mode": "standard", + "min_source_families": 0 + }, + "continuation": { + "mode": "auto_if_not_decision_ready", + "attempt": 1, + "max_attempts": 3, + "session_id": "xy-451-attempt-4-1777808209" + }, + "events": [ + { + "seq": 1, + "type": "probe_completed", + "remaining_option_count": 1, + "independent_option_questions": [], + "external_slices": [] + }, + { + "seq": 2, + "type": "evidence_recorded", + "evidence": [ + { + "id": "E1", + "kind": "observation", + "summary": "The Cloud task client trait exposes list, summary/status, diff, messages/text, sibling attempts, preflight apply, apply, and create operations.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/api.rs:133-163" + }, + { + "id": "E2", + "kind": "observation", + "summary": "The HTTP client maps Cloud list and create to backend task endpoints, with Codex API and ChatGPT backend path styles, and supports limit, task_filter, environment_id, and cursor query parameters.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/backend-client/src/client.rs:320-424" + }, + { + "id": "E3", + "kind": "observation", + "summary": "Task summary/status derives from task_status_display/latest_turn_status_display, while task text exposes prompt, assistant messages, current turn id, sibling turn ids, placement, and attempt status.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/http.rs:183-317" + }, + { + "id": "E4", + "kind": "observation", + "summary": "Cloud diff/apply fetches a unified diff from task details or uses an override, then applies locally from the current directory through codex_git_utils and git apply; preflight is git apply --check.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/http.rs:252-520; /private/tmp/openai-codex/codex-rs/git-utils/src/apply.rs:37-104" + }, + { + "id": "E5", + "kind": "observation", + "summary": "The Cloud CLI wrapper surfaces exec, status, list, apply, and diff commands; exec requires an environment, can select a branch, and supports one to four attempts.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks/src/cli.rs:15-120" + }, + { + "id": "E6", + "kind": "observation", + "summary": "Decodex's MVP app-server contract is a local stdio JSON-RPC session using initialize, thread/start or thread/resume, turn/start, turn completion notifications, and dynamic tool calls for issue-scoped tracker writes.", + "source_family": "decodex_repo", + "source_path": "docs/spec/app-server.md:9-80; docs/spec/app-server.md:102-152" + }, + { + "id": "E7", + "kind": "observation", + "summary": "Decodex runtime authority is one local service, one isolated linked worktree lane per eligible issue, one direct app-server session per attempt, runtime DB ownership for active state, and Linear/GitHub as mirrors rather than live runtime backends.", + "source_family": "decodex_repo", + "source_path": "docs/spec/runtime.md:9-60; docs/spec/runtime.md:110-150" + }, + { + "id": "E8", + "kind": "observation", + "summary": "Decodex's successful completion path requires PR-backed review handoff, repo gate validation, and explicit terminal finalization; the runtime must not infer completion or bypass the handoff state.", + "source_family": "decodex_repo", + "source_path": "docs/spec/runtime.md:183-241; src/orchestrator/execution.rs:658-705" + }, + { + "id": "E9", + "kind": "observation", + "summary": "The owned-lane policy restricts runtime decisions to fixed action classes and requires manual intervention when retained lane, tracker, or PR signals disagree enough to require guessing.", + "source_family": "decodex_repo", + "source_path": "docs/spec/owned-lane-policy.md:26-69; docs/spec/owned-lane-policy.md:116-160" + }, + { + "id": "E10", + "kind": "observation", + "summary": "The operator control plane treats the runtime SQLite database as the active execution source of truth; Linear and GitHub remain collaboration and validation surfaces, not queue or lane ownership authority.", + "source_family": "decodex_repo", + "source_path": "docs/reference/operator-control-plane.md:16-57" + } + ] + }, + { + "seq": 3, + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T1", + "summary": "Cloud task list/status/diff data could help a future remote-observation or remote-execution mode, but the source surface does not carry Decodex's issue lease, repo gate, or PR handoff authority.", + "supporting_evidence_ids": [ + "E1", + "E2", + "E3", + "E6", + "E7", + "E8" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T2", + "summary": "Importing Cloud apply directly into Decodex would be unsafe unless wrapped by lane ownership, dirty-tree protection, repo gate validation, and PR-backed handoff, because Cloud apply is currently a local git apply primitive.", + "supporting_evidence_ids": [ + "E4", + "E7", + "E8", + "E9" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T3", + "summary": "Cloud create is plausible future work only as an explicit remote-execution mode that binds environment, branch, issue, and attempt identity up front, rather than an account-pool side effect.", + "supporting_evidence_ids": [ + "E2", + "E5", + "E7", + "E9" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T4", + "summary": "Deferring keeps the account-pool slice from becoming a remote-execution redesign while preserving the evidence that Cloud tasks have reusable task, diff, and apply primitives for a later scoped issue.", + "supporting_evidence_ids": [ + "E1", + "E4", + "E6", + "E7", + "E10" + ], + "disconfirming_evidence_ids": [] + } + ] + }, + { + "seq": 4, + "type": "judgment_candidate_created", + "judgment_payload": { + "judgment_type": "defer", + "preferred_option": "defer-cloud-task-integration", + "rejected_options": [ + "adopt-cloud-tasks-as-decodex-execution-backend-now", + "reject-cloud-tasks-permanently" + ], + "decision_claim": "Defer Codex Cloud task integration. Local codex app-server remains the authoritative MVP path. Cloud task APIs may belong later only behind an explicit, separately scoped remote-execution mode that maps one Cloud task to one Decodex-owned issue/lane, preserves repo-gate and PR-backed handoff, and does not reuse account-pool work as a hidden remote-execution redesign.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E6", + "E7", + "E8" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T3", + "T4" + ] + }, + "judgment_hash": "sha256:5a4707197057d9106e828560d302c9191a5ab44b150b1517bf6c226ae0f04019" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The defer recommendation is directionally correct, but the future-mode constraints must explicitly require issue-scoped tracker-tool and terminal-finalization parity, runtime liveness parity, and a comparison against app-server remote-environment support rather than assuming Cloud tasks are the only remote shape.", + "objections": [ + { + "id": "OBJ-001-tracker-tool-parity-missing", + "summary": "Preserving repo gate and PR-backed handoff is insufficient unless any future remote mode also preserves issue-scoped tracker tool calls and explicit issue_terminal_finalize semantics." + }, + { + "id": "OBJ-002-cloud-task-shape-may-be-the-wrong-remote-shape", + "summary": "The future path should not assume one Cloud task per lane is the right abstraction because app-server already exposes environment selection plus dynamic tools while Decodex relies on repeated turn/start calls on one thread." + }, + { + "id": "OBJ-003-operator-liveness-parity-omitted", + "summary": "The constraints must require runtime-observability parity for protocol events, liveness fields, stall detection, and recovery; Cloud task summaries and diffs alone do not satisfy Decodex's operator model." + } + ], + "seq": 5, + "target_judgment_hash": "sha256:5a4707197057d9106e828560d302c9191a5ab44b150b1517bf6c226ae0f04019" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E11", + "kind": "observation", + "summary": "Decodex's tracker bridge requires issue-scoped tool calls for transition, comment, progress checkpoint, review checkpoint, review handoff, label add, and terminal finalization; invalid or missing terminal signals must fail the attempt.", + "source_family": "decodex_repo", + "source_path": "docs/spec/tracker-tools.md:18-59; docs/spec/tracker-tools.md:63-83; docs/spec/tracker-tools.md:85-130" + }, + { + "id": "E12", + "kind": "observation", + "summary": "The Codex app-server protocol already exposes experimental thread/start and turn/start environment selections alongside dynamic tools and raw-event support, so future remote work must compare Cloud tasks with an app-server environment extension before choosing an execution shape.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/app-server-protocol/src/protocol/v2.rs:3548-3618; /private/tmp/openai-codex/codex-rs/app-server-protocol/src/protocol/v2.rs:5519-5544; /private/tmp/openai-codex/codex-rs/app-server/src/codex_message_processor.rs:2788-2837; /private/tmp/openai-codex/codex-rs/app-server/src/codex_message_processor.rs:3001-3019" + }, + { + "id": "E13", + "kind": "observation", + "summary": "Decodex operator visibility depends on runtime DB protocol journals, active-lane heartbeat, liveness fields, stall detection, and event-count hydration, which task summaries and diffs do not provide by themselves.", + "source_family": "decodex_repo", + "source_path": "docs/spec/runtime.md:290-316; docs/spec/runtime.md:318-338; docs/spec/runtime.md:348-378" + }, + { + "id": "E14", + "kind": "observation", + "summary": "The Cloud task command runtime wires exec to create_task, status to get_task_summary, list to paginated list_tasks, diff to collected attempt diffs, and apply to selected attempt diff plus local apply_task; the TUI also supports preflight/apply and detail loading around those primitives.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks/src/lib.rs:157-180; /private/tmp/openai-codex/codex-rs/cloud-tasks/src/lib.rs:493-603; /private/tmp/openai-codex/codex-rs/cloud-tasks/src/lib.rs:614-724; /private/tmp/openai-codex/codex-rs/cloud-tasks/src/lib.rs:1845-1965" + } + ], + "seq": 6 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T5", + "summary": "A future remote-execution mode cannot be accepted merely because it can create a task and return a patch; it must preserve issue-scoped tracker tools, terminal finalization, repo gate validation, PR handoff, and lane-scoped Git ownership.", + "supporting_evidence_ids": [ + "E4", + "E7", + "E8", + "E11" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T6", + "summary": "Codex app-server environment support may be a better future remote abstraction than Cloud tasks because it is closer to Decodex's existing thread, turn, dynamic-tool, and event-stream model; a later issue should compare both shapes before implementation.", + "supporting_evidence_ids": [ + "E6", + "E7", + "E11", + "E12", + "E13" + ], + "disconfirming_evidence_ids": [] + }, + { + "id": "T7", + "summary": "Until Cloud task APIs can provide tracker-tool parity and liveness/event parity, their safest Decodex use is read-only observation or explicit patch import/apply inside an already owned local lane, not authoritative execution replacement.", + "supporting_evidence_ids": [ + "E1", + "E3", + "E4", + "E11", + "E13", + "E14" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 7 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "OBJ-001-tracker-tool-parity-missing", + "OBJ-002-cloud-task-shape-may-be-the-wrong-remote-shape", + "OBJ-003-operator-liveness-parity-omitted" + ], + "judgment_payload": { + "judgment_type": "defer", + "preferred_option": "defer-cloud-task-integration-and-keep-local-app-server-mvp", + "rejected_options": [ + "adopt-cloud-tasks-as-decodex-execution-backend-now", + "reject-cloud-task-apis-permanently", + "hide-remote-execution-inside-account-pool-work", + "choose-cloud-task-backend-before-comparing-app-server-remote-environments" + ], + "decision_claim": "Defer Codex Cloud task integration. Local codex app-server remains the authoritative Decodex MVP path. Cloud task APIs are useful future evidence for list/create/status/diff/apply and may support read-only observation or explicit patch import later, but authoritative remote execution must be a separate issue that compares Cloud tasks with app-server remote-environment support and preserves issue-scoped tracker tools, explicit terminal finalization, runtime liveness/event hydration, repo-gate validation, PR-backed handoff, and lane-scoped Git ownership. Account-pool work must not carry Cloud task execution, credential schema, or remote-execution lifecycle changes.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E6", + "E7", + "E8", + "E11", + "E12", + "E13", + "E14" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5", + "T6", + "T7" + ] + }, + "seq": 8, + "judgment_hash": "sha256:45bf9826204b1a2d32600f03cdde58526f7feeb0f5117fcef1f3952acc360c37" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The revised defer judgment addresses the earlier remote-execution objections, but its patch-import clause still needs to carry the owned-lane, dirty-tree, routed-identity, repo-gate, and PR-handoff qualifiers in the decision claim itself.", + "objections": [ + { + "id": "OBJ-004-patch-import-safety-omitted", + "summary": "The decision claim's patch-import wording must explicitly restrict any future Cloud diff/apply use to an already owned local lane with clean/checked worktree state, routed identity, repo-gate validation, and PR-backed handoff." + } + ], + "seq": 9, + "target_judgment_hash": "sha256:45bf9826204b1a2d32600f03cdde58526f7feeb0f5117fcef1f3952acc360c37" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E15", + "kind": "observation", + "summary": "Decodex lane Git and review operations depend on routed GitHub credentials and worktree-local identity inheritance; patch import/apply must therefore remain inside a Decodex-owned linked worktree with the correct routed identity rather than operating on an arbitrary cwd.", + "source_family": "decodex_repo", + "source_path": "docs/spec/runtime.md:152-166; docs/spec/tracker-tools.md:108-122; src/worktree.rs:1420-1481" + } + ], + "seq": 10 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T8", + "summary": "Patch import from Cloud task diffs is only acceptable as a future lane-owned workflow: preflight against the owned linked worktree, protect dirty state, preserve routed Git/Linear identity, run the repo gate, and complete PR-backed handoff through Decodex rather than treating Cloud apply as an external authority.", + "supporting_evidence_ids": [ + "E4", + "E8", + "E11", + "E15" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 11 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "OBJ-004-patch-import-safety-omitted" + ], + "judgment_payload": { + "judgment_type": "defer", + "preferred_option": "defer-cloud-task-integration-and-keep-local-app-server-mvp", + "rejected_options": [ + "adopt-cloud-tasks-as-decodex-execution-backend-now", + "reject-cloud-task-apis-permanently", + "hide-remote-execution-inside-account-pool-work", + "choose-cloud-task-backend-before-comparing-app-server-remote-environments", + "allow-cloud-diff-apply-outside-an-owned-clean-decodex-lane" + ], + "decision_claim": "Defer Codex Cloud task integration. Local codex app-server remains the authoritative Decodex MVP path. Cloud task APIs are useful future evidence for list/create/status/diff/apply and may support read-only observation later. Explicit Cloud patch import/apply is only a possible future lane-owned workflow, and only if a separate issue proves preflight against the owned linked worktree, dirty-tree protection, routed Git/Linear identity, repo-gate validation, and PR-backed handoff through Decodex. Authoritative remote execution must be a separate issue that compares Cloud tasks with app-server remote-environment support and preserves issue-scoped tracker tools, explicit terminal finalization, runtime liveness/event hydration, repo-gate validation, PR-backed handoff, and lane-scoped Git ownership. Account-pool work must not carry Cloud task execution, credential schema, or remote-execution lifecycle changes.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E6", + "E7", + "E8", + "E11", + "E12", + "E13", + "E14", + "E15" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5", + "T6", + "T7", + "T8" + ] + }, + "seq": 12, + "judgment_hash": "sha256:483bc8f58842f2f113359833a3c80e63ae1b3c431c2de413a207ac80b73d2614" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The judgment is close, but the future remote-execution identity contract must explicitly bind Cloud task creation to Decodex issue, run, attempt, branch, and environment identity.", + "objections": [ + { + "id": "OBJ-005-remote-run-identity-binding-omitted", + "summary": "Any future Cloud create/execution mode must bind environment, branch, issue identity, run_id, and attempt_number up front so task, sibling-attempt, review, and closeout recovery can reconcile safely against Decodex runtime authority." + } + ], + "seq": 13, + "target_judgment_hash": "sha256:483bc8f58842f2f113359833a3c80e63ae1b3c431c2de413a207ac80b73d2614" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E16", + "kind": "observation", + "summary": "Cloud task creation currently binds environment_id, branch/git_ref, qa mode, and best_of_n attempts, while Decodex lane recovery and handoff identity are keyed by issue identity, run_id, attempt_number, branch, PR head, and runtime DB ownership; a future remote mode must bridge both identity models explicitly.", + "source_family": "mixed_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/api.rs:162-169; /private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/http.rs:319-379; /private/tmp/openai-codex/codex-rs/cloud-tasks/src/lib.rs:157-180; docs/spec/runtime.md:72-80; docs/spec/runtime.md:110-150; docs/spec/runtime.md:217-241; docs/spec/runtime.md:348-378" + } + ], + "seq": 14 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T9", + "summary": "A future Cloud execution mode must establish task-to-lane identity at creation time by binding environment, branch, issue identifier/id, run_id, and attempt_number, otherwise sibling attempts, liveness, review handoff, retained recovery, and closeout cannot be reconciled safely.", + "supporting_evidence_ids": [ + "E2", + "E7", + "E8", + "E13", + "E16" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 15 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "OBJ-005-remote-run-identity-binding-omitted" + ], + "judgment_payload": { + "judgment_type": "defer", + "preferred_option": "defer-cloud-task-integration-and-keep-local-app-server-mvp", + "rejected_options": [ + "adopt-cloud-tasks-as-decodex-execution-backend-now", + "reject-cloud-task-apis-permanently", + "hide-remote-execution-inside-account-pool-work", + "choose-cloud-task-backend-before-comparing-app-server-remote-environments", + "allow-cloud-diff-apply-outside-an-owned-clean-decodex-lane", + "create-cloud-tasks-without-decodex-run-and-attempt-identity" + ], + "decision_claim": "Defer Codex Cloud task integration. Local codex app-server remains the authoritative Decodex MVP path. Cloud task APIs are useful future evidence for list/create/status/diff/apply and may support read-only observation later. Explicit Cloud patch import/apply is only a possible future lane-owned workflow, and only if a separate issue proves preflight against the owned linked worktree, dirty-tree protection, routed Git/Linear identity, repo-gate validation, and PR-backed handoff through Decodex. Authoritative remote execution must be a separate issue that compares Cloud tasks with app-server remote-environment support and preserves issue-scoped tracker tools, explicit terminal finalization, runtime liveness/event hydration, repo-gate validation, PR-backed handoff, and lane-scoped Git ownership. Any future Cloud create/execution path must bind environment_id, branch/git_ref, Linear issue identity, Decodex run_id, and attempt_number before task creation so task status, sibling attempts, liveness, review handoff, retained recovery, and closeout can reconcile to one owned lane. Account-pool work must not carry Cloud task execution, credential schema, or remote-execution lifecycle changes.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E6", + "E7", + "E8", + "E11", + "E12", + "E13", + "E14", + "E15", + "E16" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5", + "T6", + "T7", + "T8", + "T9" + ] + }, + "seq": 16, + "judgment_hash": "sha256:1b33cd90ee31a9d3898380f658af5a18b5e8895688f4095ba4926d4d8565766b" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "The defer judgment is still plausible, but read-only observation also needs an explicit provenance requirement because current Cloud list/create surfaces do not carry Decodex issue/run/attempt identity by themselves.", + "objections": [ + { + "id": "OBJ-006-read-only-observation-provenance-gap", + "summary": "Read-only Cloud task observation must not hydrate Decodex operator state unless a future issue proves a durable Decodex-owned mapping or backend metadata channel for issue, run_id, and attempt_number provenance." + } + ], + "seq": 17, + "target_judgment_hash": "sha256:1b33cd90ee31a9d3898380f658af5a18b5e8895688f4095ba4926d4d8565766b" + }, + { + "type": "evidence_recorded", + "evidence": [ + { + "id": "E17", + "kind": "observation", + "summary": "Current Cloud task list summaries expose task id, title, status, timestamp, environment label, diff summary, review flag, and attempt_total, with environment_id unset in list mapping; create sends metadata only for best_of_n and returns only the created task id, so Decodex provenance is not available from those surfaces alone.", + "source_family": "codex_source", + "source_path": "/private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/api.rs:33-50; /private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/http.rs:319-380; /private/tmp/openai-codex/codex-rs/cloud-tasks-client/src/http.rs:717-732" + } + ], + "seq": 18 + }, + { + "type": "tradeoffs_recorded", + "tradeoffs": [ + { + "id": "T10", + "summary": "Even read-only Cloud task observation must remain non-authoritative until a future issue proves provenance through a Decodex-owned task mapping or backend metadata channel that round-trips issue id, issue identifier, run_id, and attempt_number on list/details.", + "supporting_evidence_ids": [ + "E7", + "E10", + "E13", + "E16", + "E17" + ], + "disconfirming_evidence_ids": [] + } + ], + "seq": 19 + }, + { + "type": "judgment_candidate_created", + "addresses_objection_ids": [ + "OBJ-006-read-only-observation-provenance-gap" + ], + "judgment_payload": { + "judgment_type": "defer", + "preferred_option": "defer-cloud-task-integration-and-keep-local-app-server-mvp", + "rejected_options": [ + "adopt-cloud-tasks-as-decodex-execution-backend-now", + "reject-cloud-task-apis-permanently", + "hide-remote-execution-inside-account-pool-work", + "choose-cloud-task-backend-before-comparing-app-server-remote-environments", + "allow-cloud-diff-apply-outside-an-owned-clean-decodex-lane", + "create-cloud-tasks-without-decodex-run-and-attempt-identity", + "hydrate-decodex-operator-state-from-cloud-tasks-without-provenance" + ], + "decision_claim": "Defer Codex Cloud task integration. Local codex app-server remains the authoritative Decodex MVP path. Cloud task APIs are useful future evidence for list/create/status/diff/apply, but even read-only observation must stay non-authoritative until a separate issue proves a Decodex-owned task mapping or backend metadata channel that round-trips Linear issue id/identifier plus Decodex run_id and attempt_number on task list/details. Explicit Cloud patch import/apply is only a possible future lane-owned workflow, and only if a separate issue proves preflight against the owned linked worktree, dirty-tree protection, routed Git/Linear identity, repo-gate validation, and PR-backed handoff through Decodex. Authoritative remote execution must be a separate issue that compares Cloud tasks with app-server remote-environment support and preserves issue-scoped tracker tools, explicit terminal finalization, runtime liveness/event hydration, repo-gate validation, PR-backed handoff, and lane-scoped Git ownership. Any future Cloud create/execution path must bind environment_id, branch/git_ref, Linear issue identity, Decodex run_id, and attempt_number before task creation so task status, sibling attempts, liveness, review handoff, retained recovery, and closeout can reconcile to one owned lane. Account-pool work must not carry Cloud task execution, credential schema, or remote-execution lifecycle changes.", + "key_evidence_ids": [ + "E1", + "E2", + "E4", + "E6", + "E7", + "E8", + "E11", + "E12", + "E13", + "E14", + "E15", + "E16", + "E17" + ], + "key_tradeoff_ids": [ + "T1", + "T2", + "T4", + "T5", + "T6", + "T7", + "T8", + "T9", + "T10" + ] + }, + "seq": 20, + "judgment_hash": "sha256:9d5873aefa6a6e9e5ceef5c8f340dbe008f8666483c1666b9c9afec48ab9c9ac" + }, + { + "type": "worker_completed", + "worker": "skeptic", + "summary": "No remaining objection blocks decision-ready finalization after the judgment requires separate issue scope, tracker/terminal/liveness parity, task-to-lane identity binding, provenance for read-only observation, and owned-lane safeguards for any patch import/apply path.", + "objections": [], + "seq": 21, + "target_judgment_hash": "sha256:9d5873aefa6a6e9e5ceef5c8f340dbe008f8666483c1666b9c9afec48ab9c9ac" + }, + { + "type": "finalized_decision_ready", + "confidence": "medium", + "missing_evidence": [ + "A live disposable Cloud task metadata round-trip was not performed; the future issue must verify whether list/details can carry Decodex issue/run/attempt provenance.", + "A live non-local codex app-server environment run with dynamic tools and event hydration was not performed; the future issue must compare that app-server shape against Cloud tasks before implementation." + ], + "seq": 22, + "judgment_hash": "sha256:9d5873aefa6a6e9e5ceef5c8f340dbe008f8666483c1666b9c9afec48ab9c9ac" + } + ] +} diff --git a/docs/guide/github_pages_deploy.md b/docs/runbook/github-pages-deploy.md similarity index 100% rename from docs/guide/github_pages_deploy.md rename to docs/runbook/github-pages-deploy.md diff --git a/docs/runbook/index.md b/docs/runbook/index.md new file mode 100644 index 00000000..db3df088 --- /dev/null +++ b/docs/runbook/index.md @@ -0,0 +1,37 @@ +# Runbook Index + +Purpose: Route agents to procedural documents that tell them which sequence to execute. + +Question this index answers: "which sequence should I execute?" + +## Use this index when + +- You need a runbook, how-to, migration sequence, validation flow, troubleshooting path, + or maintenance procedure. +- You already know the relevant spec and need the operational steps. +- You need explicit prerequisites, commands, checkpoints, or verification. + +## Do not use this index when + +- You need the authoritative contract, schema, or invariant. +- You need current repository layout or implementation boundaries. +- You need durable design rationale rather than operator steps. + +## What belongs in `docs/runbook/` + +- Task-oriented operator procedures. +- Validation and inspection sequences. +- Rollout, rollback, and recovery flows. +- Bounded recipes that depend on a governing spec. + +## Current runbooks + +- [`github-pages-deploy.md`](./github-pages-deploy.md) for GitHub Pages deployment and + `decodex.space` custom-domain setup for the static public site. +- [`linear-archive-hygiene.md`](./linear-archive-hygiene.md) for dry-run-first + archive hygiene of old terminal Linear issues by repo label. +- [`local-github-signal-workflow.md`](./local-github-signal-workflow.md) for collecting + GitHub change bundles, running Codex editorial analysis, validating signal entries, + and publishing static site content. +- [`self-dogfood-pilot.md`](./self-dogfood-pilot.md) for the retained-lane pilot run + against `decodex` itself and the bounded live-operation sequence. diff --git a/docs/runbook/linear-archive-hygiene.md b/docs/runbook/linear-archive-hygiene.md new file mode 100644 index 00000000..6ffe0782 --- /dev/null +++ b/docs/runbook/linear-archive-hygiene.md @@ -0,0 +1,75 @@ +# Linear Archive Hygiene + +Goal: Archive old terminal Linear issues without touching active Decodex lanes, +queued intake, review handoff, recovery ownership, or unrelated repo labels. + +Read this when: + +- Linear issue volume is high before a demo, large issue seed, or backlog reset. +- You need a dry-run list of terminal issues that are old enough to archive. +- You need to scope cleanup to a repo label such as `repo:decodex` or + `repo:ashen-vale`. + +Preconditions: + +- Register the target project first with `decodex project add ` or pass `--config`. +- The registered project config tracker credential must point at the routed Linear + workspace identity, such as `LINEAR_API_KEY_HACKINK` for the `y`/`hackink` + route. +- The issue must carry a repo label beginning with `repo:`. + +Depends on: + +- Terminal states from `WORKFLOW.md` `[tracker].terminal_states`. +- Protected labels from the same workflow policy: + `decodex:queued:`, `decodex:active:`, + `decodex:needs-attention`, and `decodex:manual-only`. + +Verification: + +- Dry run first and inspect every candidate. +- Re-run the dry run after execution if you need an empty candidate list. + +## Dry Run + +Use the dry-run default before demos or large issue seeding: + +```sh +cargo run -p decodex -- archive-linear --repo-label repo:decodex --older-than-days 30 +``` + +The command prints the terminal issues that would be archived, using `updatedAt` +as the age cutoff. It does not mutate Linear unless `--execute` is present. + +For another Decodex-managed repo, run from that registered checkout or pass its centralized config: + +```sh +cargo run -p decodex -- archive-linear --config ~/.codex/decodex/projects/ashen-vale --repo-label repo:ashen-vale --older-than-days 30 +``` + +## Execute + +After the dry run shows only issues that should leave the active tracker view, +repeat the command with `--execute`: + +```sh +cargo run -p decodex -- archive-linear --repo-label repo:decodex --older-than-days 30 --execute +``` + +This archives issues through Linear `issueArchive` with `trash = false`. It does +not delete issues. + +## Exclusions + +The archive plan skips an issue when any of these are true: + +- The issue state is not one of the configured terminal states, such as `Done`, + `Canceled`, or `Duplicate`. +- The issue was updated after the cutoff. +- The issue still has `decodex:active:` ownership. +- The issue is still queued with `decodex:queued:`. +- The issue is marked `decodex:needs-attention`. +- The issue is marked `decodex:manual-only`. + +These exclusions keep active, queued, in-review, needs-attention, manual-only, +and retained recovery lanes out of archive hygiene. diff --git a/docs/guide/local_github_signal_workflow.md b/docs/runbook/local-github-signal-workflow.md similarity index 92% rename from docs/guide/local_github_signal_workflow.md rename to docs/runbook/local-github-signal-workflow.md index 0819a673..bb678088 100644 --- a/docs/guide/local_github_signal_workflow.md +++ b/docs/runbook/local-github-signal-workflow.md @@ -13,11 +13,11 @@ Inputs: - The governing specs for repo layout, GitHub change bundles, and signal entries Depends on: -- `docs/spec/repo_layout.md` -- `docs/spec/github_change_bundle.md` -- `docs/spec/signal_entry.md` -- `docs/spec/release_delta.md` -- `docs/spec/site_contract.md` +- `docs/reference/workspace-layout.md` +- `docs/spec/github-change-bundle.md` +- `docs/spec/signal-entry.md` +- `docs/spec/release-delta.md` +- `docs/spec/site-contract.md` Outputs: - A validated signal entry committed to the repo @@ -27,7 +27,7 @@ Outputs: 1. Build a normalized GitHub change bundle under `tools/github/bundles/`. 2. Review the bundle and decide whether the change is signal-worthy. -3. Run Codex analysis against the bundle with `skills/decodex-github-signal/` and save the editorial draft JSON. +3. Run Codex analysis against the bundle with `plugins/decodex/skills/github-signal/` and save the editorial draft JSON. 4. Render the resulting signal entry into `site/src/content/signals/`. 5. Validate the signal entry shape and collection consistency. 6. Regenerate the release-delta artifact so the homepage compares the latest stable release to the latest prerelease using the updated signal set. @@ -88,7 +88,7 @@ The repository already includes a real sample for this flow: Repo-local skill entrypoint: -- `skills/decodex-github-signal/SKILL.md` +- `plugins/decodex/skills/github-signal/SKILL.md` Automated hourly sync entrypoint: diff --git a/docs/runbook/self-dogfood-pilot.md b/docs/runbook/self-dogfood-pilot.md new file mode 100644 index 00000000..ab6ab83c --- /dev/null +++ b/docs/runbook/self-dogfood-pilot.md @@ -0,0 +1,710 @@ +# Self-Dogfood Pilot + +Goal: Run the `decodex` MVP against one target repository and a bounded set of queued Linear issues, with `decodex` itself as the default first pilot target. +Read this when: You are preparing a dry run or live self-dogfood pilot and need the bounded operator procedure for config, target-repo requirements, and expected run behavior. +Preconditions: `codex app-server` is available locally; `gh` is available locally for live PR-backed handoff validation, merge inspection, and retained branch cleanup; the target repository exists on disk; the project contract exists under `~/.codex/decodex/projects//`; referenced `WORKFLOW.md [context.read_first]` files exist in `[paths].repo_root`; the Linear team exposes the required workflow states; and the tracker and GitHub token env-var names are configured through `tracker.api_key_env_var` and `github.token_env_var` in the centralized project config. +Depends on: `docs/spec/runtime.md`, `docs/spec/workflow-file.md`, `docs/spec/app-server.md`, the registered project `WORKFLOW.md`, and `Makefile.toml` for repo-native verification tasks. +Verification: `cargo run -p decodex -- probe`; `cargo run -p decodex -- project add ~/.codex/decodex/projects/decodex`; `cargo run -p decodex -- project list`; `cargo run -p decodex -- run --dry-run`; and, when the environment is ready, `cargo run -p decodex -- run`. + +## Alignment note + +- Normal-path tracker writes now belong to the coding agent through issue-scoped tools. +- `decodex` still owns startup reconciliation, local leases, worktree lifecycle, retries, and fallback tracker writes when a run never reaches the normal agent-owned path. +- Every live pass now starts with reconciliation of stale local leases and terminal worktree mappings before issue selection. + +## Preconditions + +- `codex app-server` is available locally. +- `gh` is available locally for live runs that must validate PR-backed review handoff. +- The host is macOS or Linux; Decodex does not support Windows. +- The target repository already exists on disk as a normal Git checkout. +- The registered project directory has a `WORKFLOW.md` beside `project.toml`. +- The target repository files referenced by `WORKFLOW.md [context.read_first]` exist under `[paths].repo_root`. +- The Linear team already has the workflow states used by the registered `WORKFLOW.md`. +- The Linear API token env-var name is configured through `tracker.api_key_env_var` in the centralized project config. +- GitHub auth for lane Git pushes, PR creation, review handoff, and post-review status is configured through `github.token_env_var` in the centralized project config; `decodex` does not fall back to ambient `GH_TOKEN` or an existing `gh auth login` session. + +Recommended first-run check: + +```sh +cargo run -p decodex -- probe +``` + +If `decodex probe` does not return `PROBE_OK`, stop there. The orchestrator loop depends on the same direct `app-server` contract. + +## Recommended layout + +For the recommended first deployment, collect each project contract under `~/.codex/decodex/projects//`, put service paths and credentials in `project.toml`, put execution policy in `WORKFLOW.md`, and set `[paths].repo_root` explicitly. If you need a redacted template for another checkout or workspace, start from `decodex.example.toml`. + +```text +~/.codex/decodex/ + config.toml + runtime.sqlite3 + logs/ + projects/ + decodex/ + project.toml + WORKFLOW.md + +/path/to/hack-ink/decodex/.worktrees/ + XY-123/ + XY-124/ +``` + +`decodex` resolves config in this order: + +1. `--config ` +2. the global project registry entry whose `repo_root` or `worktree_root` owns the current directory + +Projects must be registered explicitly. Keep project configs in the canonical +directory `~/.codex/decodex/projects//`, make sure each `project.toml` +includes `[paths].repo_root`, then register and verify the entry: + +```sh +decodex project add +decodex project list +``` + +`decodex serve` schedules enabled projects from that registry. It does not scan +`.codex` history, repo-local config files, or currently open worktrees to infer +projects. + +After restarting `decodex serve`, verify the registry still points at the centralized +project directory: + +- `decodex project list` should show the project config as + `~/.codex/decodex/projects//project.toml`. +- `GET /state` or the operator UI should show no project backed by a flat `*.toml` + config path inside a checkout or lane worktree. + +Runtime state now lives in the Decodex-owned SQLite database at `~/.codex/decodex/runtime.sqlite3`, and logs live under `~/.codex/decodex/logs/`. On restart, `decodex` reloads retained worktree knowledge and active-lane recovery intent from that database, then refreshes low-frequency Linear and GitHub state as connector budgets allow. + +That recovery is still scoped by configured `service_id`, so reconciliation and cleanup stay within the single service instance represented by the registered project config. + +## Sample service config + +```toml +service_id = "decodex" + +[tracker] +api_key_env_var = "LINEAR_API_KEY" + +[github] +token_env_var = "GITHUB_TOKEN" + +[codex] +internal_review_mode = "prompt" +external_review_enabled = false + +[paths] +repo_root = "/path/to/hack-ink/decodex" +``` + +Notes: + +- `service_id` scopes service-owned labels, reconciliation, and retained local state. Pick one stable service namespace per project config. +- `paths.repo_root` is required. Decodex does not derive it from the config file location. +- `paths.worktree_root` is optional. If omitted, Decodex uses `/.worktrees`. Relative worktree overrides are resolved from `repo_root`; relative `repo_root` overrides are resolved from the config file location. +- `transport` is defined in the registered project `WORKFLOW.md` and should normally be `stdio://`. +- Decodex does not expose repo-local model or reasoning overrides. `codex app-server` inherits those defaults from `~/.codex/config.toml`. +- `api_key_env_var` is required and must name the environment variable that stores the Linear API token. +- `github.token_env_var` is required for PR-backed review handoff validation and post-review PR-state inspection and must name the environment variable that stores the GitHub token. +- For the self-dogfood pilot, use `codex.internal_review_mode = "loop"` for the runtime-owned self-review checkpoint loop, `"prompt"` to add only `Review your work repeatedly and fix any logic bugs until no new issues are found.`, or `"off"` to skip internal self-review. If omitted, the default is `"loop"`. +- Keep `codex.external_review_enabled = false` when the retained lane should skip the runtime-owned `@codex review` request and rely on the PR-backed handoff plus the normal PR landing checks. +- With `codex.external_review_enabled = false`, a one-shot `decodex run` may continue draining the same retained lane after review handoff if the retained landing gates are already satisfied. If those gates are still pending, the run exits cleanly at the retained waiting boundary instead of spinning. +- Automatic intake is driven by the service-scoped Linear label `decodex:queued:` derived from the registered project config `service_id`. Keep the pilot bounded by applying that label only to the small issue set you want `decodex` to own. + +## Target repository contract + +The registered project directory must provide a parseable `WORKFLOW.md` with TOML frontmatter. For the MVP, the frontmatter contract lives in [`docs/spec/workflow-file.md`](../spec/workflow-file.md). For the first pilot, that means `~/.codex/decodex/projects/decodex/WORKFLOW.md`. + +At minimum, the target repo should define: + +- `[tracker] provider = "linear"` +- `[tracker] startable_states = ["Todo"]` or another explicit start set +- `[tracker] terminal_states = ["Done", "Canceled", "Duplicate"]` or another explicit terminal set +- `[tracker] in_progress_state = ""` +- `[tracker] success_state = ""` +- `[tracker] failure_state = ""` +- `[tracker] opt_out_label = "