Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion bin/fm-supervision-instructions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,36 @@ render_snippet() {
done < "$SNIPPET"
}

# Away delivery is the daemon, not a session stub, so this branch names the
# daemon relaunch the /afk skill prescribes for the resolved harness: the
# no-separate-terminal native path where the harness owns a tracked background
# job, and the terminal-backed launcher everywhere else. Draining stays out of
# it: the away daemon reads the durable queue through its own cursor.
away_repair_line() {
local relaunch native_tool
case "$HARNESS" in
claude) native_tool='Claude Code background task' ;;
grok) native_tool='Grok tracked background task' ;;
*) native_tool='' ;;
esac
if [ -n "$native_tool" ]; then
relaunch="prepare the lifecycle with bin/fm-afk-launch.sh start-native, then run FM_AFK_STATE_PREPARED=1 bin/fm-afk-start.sh as its own $native_tool (never shell &); if that native launch fails, roll the preparation back with bin/fm-afk-launch.sh stop, which EXITS away mode by clearing state/.afk and therefore must be followed immediately by a fresh away entry so the captain is not left unattended without it"
else
relaunch='restart its non-visible tracked terminal with bin/fm-afk-launch.sh start'
fi
printf '%s%s%s\n' \
'Away mode owns wake delivery and no live identity-matched away daemon is reading the durable queue: ' \
"$relaunch" \
'. Then confirm state/.supervise-daemon.pid names a live pid matching the daemon lock identity; do not arm a session delivery wait instead.'
}

repair_line() {
if [ "$READ_ONLY" -eq 1 ]; then
printf '%s\n' 'Watcher repair belongs to the session holding the fleet lock; do not drain, arm, or repair from this read-only session.'
return 0
fi
if [ "$AFK" -eq 1 ]; then
printf '%s\n' 'Away mode owns wake delivery; load /afk and ensure the daemon is reading the durable queue instead of arming a session delivery wait.'
away_repair_line
return 0
fi

Expand Down
39 changes: 29 additions & 10 deletions bin/fm-turnend-guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,40 +83,59 @@ fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0
. "$SCRIPT_DIR/fm-wake-lib.sh"

fm_supervision_status "$STATE" "$GRACE"
[ "$FM_SUP_IN_FLIGHT" -gt 0 ] || exit 0
# A queued durable wake is protected work in its own right: state/*.meta records
# can be cleaned up while terminal wakes still sit unread in state/.wake-queue.
if [ "$FM_SUP_IN_FLIGHT" -eq 0 ] && [ "$FM_SUP_QUEUE_PENDING" != true ]; then
exit 0
fi
daemon_healthy=0
delivery_armed=0
afk=0
[ -e "$STATE/.afk" ] && afk=1
fm_watcher_healthy "$STATE" "$WATCH" "$GRACE" "$FM_HOME" && daemon_healthy=1
if [ -e "$STATE/.afk" ]; then
if [ "$afk" -eq 1 ]; then
# Away mode's own daemon consumes new durable queue records; normal session
# delivery waits remain intentionally paused under the /afk contract.
delivery_armed=1
fm_pusher_healthy "$STATE" && delivery_armed=1
elif fm_wake_stub_armed "$STATE" "$STUB" "$FM_HOME"; then
delivery_armed=1
fi
[ "$daemon_healthy" -eq 1 ] && [ "$delivery_armed" -eq 1 ] && exit 0

afk=0
[ -e "$STATE/.afk" ] && afk=1
x_mode=0
[ -f "$CONFIG/x-mode.env" ] && x_mode=1
DELIVERY_REASON=$("$SCRIPT_DIR/fm-supervision-instructions.sh" --afk "$afk" --x-mode "$x_mode" --repair-line 2>/dev/null \
queue_pending=0
[ "$FM_SUP_QUEUE_PENDING" = true ] && queue_pending=1
DELIVERY_REASON=$("$SCRIPT_DIR/fm-supervision-instructions.sh" --afk "$afk" --x-mode "$x_mode" \
--queue-pending "$queue_pending" --repair-line 2>/dev/null \
|| printf '%s\n' 're-arm wake delivery according to the session-start operating block before ending the turn')
DAEMON_REASON=$("$SCRIPT_DIR/fm-watcher-service.sh" repair-command 2>/dev/null \
|| printf '%s\n' 'bin/fm-watcher-service.sh restart')
if [ "$FM_SUP_IN_FLIGHT" -gt 0 ]; then
protected_desc="$FM_SUP_IN_FLIGHT task(s) in flight"
else
protected_desc='queued wake(s) pending in the durable queue'
fi
rule='━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
{
printf '●%s\n' "$rule"
printf '● TURN WOULD END BLIND - SUPERVISION IS INCOMPLETE\n'
if [ "$daemon_healthy" -eq 0 ]; then
printf '● Watcher daemon down: %s task(s) in flight and no healthy daemon holds this home lock (last beat: %s).\n' "$FM_SUP_IN_FLIGHT" "$FM_SUP_BEACON_DESC"
printf '● Watcher daemon down: %s and no healthy daemon holds this home lock (last beat: %s).\n' "$protected_desc" "$FM_SUP_BEACON_DESC"
printf '● Daemon repair: %s; treat this as a supervision incident and verify the beacon+lock predicate.\n' "$DAEMON_REASON"
fi
if [ "$delivery_armed" -eq 0 ]; then
printf '● Wake delivery missing: no identity-matched delivery stub is armed for this session.\n'
if [ "$afk" -eq 1 ]; then
printf '● Away wake delivery missing: no live identity-matched pusher holds the supervise-daemon lock.\n'
else
printf '● Wake delivery missing: no identity-matched delivery stub is armed for this session.\n'
fi
printf '● Delivery repair: %s\n' "$DELIVERY_REASON"
fi
printf '● This forced continuation is internal maintenance; after draining and re-arming, end silently unless a queued wake is captain-relevant under AGENTS.md section 9.\n'
if [ "$afk" -eq 1 ]; then
printf '● This forced continuation is internal maintenance; after restoring away delivery, end silently unless a queued wake is captain-relevant under AGENTS.md section 9.\n'
else
printf '● This forced continuation is internal maintenance; after draining and restoring delivery, end silently unless a queued wake is captain-relevant under AGENTS.md section 9.\n'
fi
printf '●%s\n' "$rule"
} >&2
exit 2
23 changes: 23 additions & 0 deletions bin/fm-wake-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,29 @@ fm_watcher_healthy() {
return 0
}

fm_pusher_lock_matches_pid() {
local state=$1 pid=$2 lockdir lock_pid lock_identity current_identity
lockdir="$state/.supervise-daemon.lock"
lock_pid=$(cat "$lockdir/pid" 2>/dev/null || true)
lock_identity=$(cat "$lockdir/pid-identity" 2>/dev/null || true)
[ "$lock_pid" = "$pid" ] || return 1
[ -n "$lock_identity" ] || return 1
current_identity=$(fm_pid_identity "$pid") || return 1
[ "$current_identity" = "$lock_identity" ]
}

FM_PUSHER_HEALTHY_PID=
fm_pusher_healthy() {
local state=$1 pid
FM_PUSHER_HEALTHY_PID=
pid=$(cat "$state/.supervise-daemon.pid" 2>/dev/null || true)
fm_pid_alive "$pid" || return 1
fm_pusher_lock_matches_pid "$state" "$pid" || return 1
# shellcheck disable=SC2034 # Read by callers after fm_pusher_healthy returns.
FM_PUSHER_HEALTHY_PID=$pid
return 0
}

fm_lock_clean_known_files() {
local lockdir=$1
rm -f \
Expand Down
2 changes: 1 addition & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ Optional direct Telegram receive is armed separately from watcher supervision wh
A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, if tasks are in flight and the daemon lock plus beacon are unhealthy, if the current session's delivery-stub identity is absent, or if queued wakes are waiting to be drained.
The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness.
It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the stale-watcher banner/reminder policy so repeated guarded commands stay noisy without reprinting the full watcher-down banner in the same episode.
On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work is in flight and either the daemon is unhealthy or the session delivery stub is unarmed, direct Stop hooks block and passive turn-end hooks force one bounded follow-up.
On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work is in flight or a durable queued wake is still unread, and either the daemon is unhealthy or wake delivery is unarmed (the session delivery stub normally, a live identity-matched away daemon while `state/.afk` is present), direct Stop hooks block and passive turn-end hooks force one bounded follow-up.
The guard covers the main primary and genuinely marked secondmate homes, exempts child crewmate/scout worktrees, is loop-safe per harness, and is documented in [turnend-guard.md](turnend-guard.md).

A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) extends this for walk-away supervision: the `/afk` skill starts it through the tracked foreground helper `bin/fm-afk-start.sh`, after which the external watcher keeps enqueuing and the away daemon reads new queue records without draining the queue.
Expand Down
2 changes: 1 addition & 1 deletion docs/scripts.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize
| `fm-tasks-axi-lib.sh` | Shared backlog-backend selector and `tasks-axi` compatibility probe |
| `fm-currency-base-lib.sh` | Shared resolution and validation of the two upstream-check comparison bases |
| `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations, then assert watcher liveness |
| `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher identity/health helpers |
| `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher/away-daemon identity/health helpers |
| `fm-classify-lib.sh` | Shared captain-relevant and declared-external-wait wake classification vocabulary |
| `fm-send.sh` | Send one verified literal line or supported key through the target's recorded backend |
| `fm-tmux-lib.sh` | Shared tmux pane primitives for busy detection, composer capture, and verified submit |
Expand Down
26 changes: 22 additions & 4 deletions docs/turnend-guard.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,10 @@ It also requires `AGENTS.md`, `bin/`, and the effective state directory to exist

For an in-scope primary checkout, it counts supervision-relevant work from `state/*.meta`.
A `kind=secondmate` record with explicit `state=resting` remains registered but is excluded; every ordinary task, active secondmate, and legacy secondmate with no state still counts.
If no supervision-relevant work remains, it exits silently.
If work is in flight, the first half requires `fm_watcher_healthy <state-dir> <watch-path> [grace-seconds] [home]` from `bin/fm-wake-lib.sh`.
An unread record in `state/.wake-queue` is protected work in its own right, because a terminal wake can outlive the `state/*.meta` record it came from: the guard stays active whenever `FM_SUP_IN_FLIGHT` is above zero **or** `FM_SUP_QUEUE_PENDING` is true, and the daemon-down banner then names the queued wakes instead of an in-flight count.
That queue state is also passed to the repair-line renderer as `--queue-pending`, exactly as `bin/fm-guard.sh` does, so a session-delivery repair says the queued wakes must be drained before re-arming.
If neither remains, it exits silently.
If work is protected, the first half requires `fm_watcher_healthy <state-dir> <watch-path> [grace-seconds] [home]` from `bin/fm-wake-lib.sh`.
That daemon predicate is unchanged: it requires an identity-matched live watcher lock and a fresh beacon.
A stale beacon blocks even if the watcher pid is still live, and a fresh leftover beacon blocks if the watcher lock is missing, dead, or identity-mismatched.
The second half requires `fm_wake_stub_armed <state-dir> <stub-path> [home]` from the same library.
Expand All @@ -37,7 +39,23 @@ The stub needs no beacon because it is a pure blocking wait over the durable que
Daemon failure and delivery failure produce separate repair lines.
The daemon line names the scoped systemd instance restart or tmux keeper repair and is treated as a real supervision incident.
The delivery line points to the active harness protocol and costs one cheap re-arm.
While `state/.afk` is present, the away daemon consumes new durable queue records and intentionally satisfies the delivery half without a session stub; daemon health remains mandatory.
While `state/.afk` is present, the away daemon consumes new durable queue records and intentionally replaces the session stub.
The away flag alone does not satisfy the delivery half.
`fm_pusher_healthy <state-dir>` requires `state/.supervise-daemon.pid` to name a live pid that matches the portable lock's pid and recorded process identity.
Those are exactly the three files `bin/fm-supervise-daemon.sh` publishes at startup, and `tests/fm-daemon.test.sh` pins the predicate against that real publication rather than a fixture.
Watcher health remains independently mandatory.
The away delivery repair line names the concrete daemon relaunch the `/afk` skill prescribes for the resolved harness, and never tells an away session to arm a session delivery wait.
A harness with a native tracked-background tool (`claude`, `grok`) is sent down the no-separate-terminal path: `bin/fm-afk-launch.sh start-native`, then `FM_AFK_STATE_PREPARED=1 bin/fm-afk-start.sh` through that tool, with `bin/fm-afk-launch.sh stop` as the rollback when that native launch fails.
Because the repair runs mid-away-session rather than at a fresh away entry, the line states in place that `stop` exits away mode by clearing `state/.afk` and must be followed immediately by a fresh away entry; that consequence is never left implicit.
Every other harness gets the terminal-backed `bin/fm-afk-launch.sh start`.
The away branch is deliberately never prefixed with the drain instruction, because the away daemon reads the durable queue through its own cursor and the session must not drain it.
For the same reason the banner's closing maintenance line also branches on away mode: an away turn is told to restore away delivery and end silently, while a normal session keeps the drain-and-restore wording.
An away banner that named a drain would be a data-loss instruction, since `bin/fm-wake-drain.sh` would consume records the daemon's cursor has not read yet; `bin/fm-afk-return.sh` owns that drain at return time instead.

If the away daemon dies during an unattended stretch, the next attempted turn end forces one maintenance continuation that directs the agent to restore away delivery.
The hook loop guard then permits that turn to stop, so daemon death cannot create an autonomous endless continuation loop.
A later independent turn alerts again until delivery is repaired.
The daemon-owned max-defer alarm cannot run after the daemon itself dies, so it is not treated as substitute coverage; queued wakes remain durable and the turn-end force is the recovery signal.

`FM_STATE_OVERRIDE` wins over `FM_HOME/state`, and `FM_HOME` wins over repo-root `state/`.
`FM_GUARD_GRACE` controls only the daemon beacon freshness window and defaults to 300 seconds.
Expand Down Expand Up @@ -197,7 +215,7 @@ Firstmate deliberately does not track `.claude/settings.local.json` anywhere, an

## Tests

`tests/fm-turnend-guard.test.sh` covers the split daemon-and-delivery predicate, independent repair lines, delivery-stub pid and identity matching, primary scoping, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, away-mode delivery ownership, fail-open behavior without `jq`, tracked hook registration for all five harnesses, and passive-adapter loop guards.
`tests/fm-turnend-guard.test.sh` covers the split daemon-and-delivery predicate, independent repair lines, delivery-stub pid and identity matching, primary scoping, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, dead and healthy away-pusher states with queued wakes, queued wakes with no `state/*.meta` record left, the drain-first delivery repair a queued-wake activation prints, both closing-banner branches (away restores delivery and never names a drain anywhere in the banner; a normal session keeps drain-and-restore), fail-open behavior without `jq`, tracked hook registration for all five harnesses, and passive-adapter loop guards.
The per-task crewmate overlay recorded in the 2026-07-25 subsection is pinned from the spawn side by `tests/fm-spawn-dispatch-profile.test.sh` (a repository-tracked `.claude/settings.local.json` survives a claude spawn, the overlay is written under the distinct name and git-excluded, and the launch line carries `--settings` for it) and from the teardown side by `tests/fm-teardown.test.sh` cases (z) through (dd) (a tracked-and-dirty overlay still tears down, that tolerance does not mask other genuine uncommitted work, and a repository-tracked legacy `settings.local.json` is never removed).
The default behavior suite does not invoke live language-model harnesses.
`FM_PI_LIVE_E2E=1 tests/fm-pi-primary-live-e2e.test.sh` opts into the isolated interactive Pi regression recorded above.
Loading
Loading