From 47cbfcf2e5aaa5f65b2334569c6e440877df28bc Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Mon, 18 May 2026 23:57:38 -0500 Subject: [PATCH 01/46] docs: phase 1.b.1 HiTL implementation plan --- .../2026-05-18-compass-phase-1b1-hitl.md | 2235 +++++++++++++++++ 1 file changed, 2235 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md diff --git a/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md new file mode 100644 index 0000000..b63e928 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md @@ -0,0 +1,2235 @@ +# Compass Phase 1.B.1 — Real HiTL via `interrupt()` + `AsyncSqliteSaver` (Implementation Plan) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the Phase 0.B/1.A auto-approve `hitl_node` with a real LangGraph `interrupt()`-based human approval flow, backed by `AsyncSqliteSaver` checkpointing and a SQLite-backed pending-approval queue. Add MCP tools (`pending_approvals`, `approve`) so the human approves/rejects scored-but-pre-tailor jobs from Claude Code or Cursor. Add a `timeout_checker` module that any cron (Modal cron lands in 1.B.3) can call to auto-cancel approvals older than `HITL_TIMEOUT_HOURS`. End state: above-threshold jobs pause at `hitl`, sit in `pending_approvals()` until a human acts, then resume into `tailor` + `vault_write` (or skip tailor on reject). Below-threshold jobs continue to auto-reject the same way they do today — no interrupt, no human prompt — so the cost of running the pipeline is unchanged for the long tail. + +**Architecture:** Three new modules under `compass/hitl/`: `state_store.py` (aiosqlite-backed pending queue, one row per paused thread), `resume.py` (re-opens the checkpointer + recompiles the graph + calls `graph.ainvoke(Command(resume=...), config={"configurable": {"thread_id": ...}})`), `timeout_checker.py` (polls the queue, resumes timed-out threads with `{"approved": False}`). `hitl_node` becomes a one-line `interrupt(...)` call wrapped with a below-threshold short-circuit. `run_pipeline()` is restructured so the `AsyncSqliteSaver` is opened once per batch (`async with` block), the graph is compiled inside it, and the per-job invocation generates a deterministic `thread_id` from the job URL + scrape timestamp. After every batch, the orchestrator detects which jobs paused (via `__interrupt__` in returned state, registering them in the state store) versus which ran to completion. New MCP tools (`pending_approvals`, `approve`) wrap the resume entrypoint. No changes to `extract` / `score` / `reflect` / `tailor` / `vault_write` node bodies. + +**Module-level discipline (carried forward from Phase 1.A):** any module that touches `VAULT_PATH`, `HITL_STATE_DB`, or `HITL_TIMEOUT_HOURS` must reference it via `compass.config.` *inside the function body*, never as a module-level captured constant. The `temp_vault` and new `temp_hitl_db` pytest fixtures monkeypatch `compass.config`, which only affects code that re-reads the attribute each call. + +**Tech Stack:** Python 3.12 · langgraph 1.2 (`langgraph.types.interrupt`, `langgraph.types.Command`) · langgraph-checkpoint-sqlite 3.1 (`langgraph.checkpoint.sqlite.aio.AsyncSqliteSaver`) · aiosqlite 0.20 (for the pending-approval queue, kept separate from the checkpoint DB) · pytest + pytest-asyncio (`asyncio_mode = "auto"` already set). + +**Authoritative spec:** `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` § Phase 1.B +**Previous-phase handoff:** `docs/PHASE_1A_COMPLETE.md` + +**Closes these deferred items from Phase 1.A:** +1. Real `interrupt()` + `AsyncSqliteSaver` checkpointing (P1.A handoff "What's deferred", row 1) +2. The `_route_after_hitl` three-way comment in `graph.py:48–58` that already anticipates this flow +3. The `HITL_STATE_DB` + `HITL_TIMEOUT_HOURS` env vars in `compass/config.py:46–47` and `.env.example:30–31` (defined for 1.B; used here for the first time) + +**Does NOT touch in this phase:** +- RAG / Chroma — deferred to **Phase 1.B.2** +- Modal cron decorators (`modal_app.py`, `@app.function(schedule=Cron(...))`) — deferred to **Phase 1.B.3**. This phase ships `timeout_checker.py` as a callable module + plain CLI entrypoint (`python -m compass.hitl.timeout_checker`) so the 1.B.3 Modal cron just imports it. +- Langfuse callback API mismatch (bug #23) — deferred to **Phase 1.B.3** observability work +- `extract` / `score` / `reflect` / `tailor` / `vault_write` node bodies +- `compass/llm.py`, taxonomy, schemas, applications lifecycle + +--- + +## File Structure + +### New +- `compass/hitl/state_store.py` — async aiosqlite store of pending approvals; pure data access +- `compass/hitl/resume.py` — `resume_pending(thread_id, decision, feedback)` reopens the checkpointer, recompiles the graph, drives the resume +- `compass/hitl/timeout_checker.py` — `check_and_resume_timeouts()` plus `__main__` CLI entrypoint +- `compass/hitl/__init__.py` — re-export the public surface + +### Test scaffolding (new) +- `tests/hitl/__init__.py` +- `tests/hitl/conftest.py` — `temp_hitl_db` fixture (monkeypatch `compass.config.HITL_STATE_DB` to a tmp path) and `frozen_now` fixture +- `tests/hitl/test_state_store.py` +- `tests/hitl/test_resume.py` +- `tests/hitl/test_timeout_checker.py` +- `tests/pipeline/test_hitl_node_interrupt.py` — replaces (does not extend) the auto-approve tests in `tests/pipeline/test_hitl_node.py` +- `tests/pipeline/test_graph_checkpointing.py` — end-to-end: invoke graph, confirm pause; resume via `resume_pending`; confirm tailor + vault_write fire +- `tests/mcp_server/test_pending_approvals.py` + +### Modify +- `compass/pipeline/nodes/hitl.py` — replace auto-approve body with `interrupt(...)` + below-threshold short-circuit +- `compass/vault/schemas.py` — tighten `JobNote.hitl_decision` from `str | None` to `Literal["approved","rejected","auto_rejected","timed_out"] | None` (Task 2.5) +- `compass/pipeline/nodes/vault_write.py` — populate `hitl_decision` + `hitl_at` on the JobNote from `state["human_approved"]` so the vault carries the audit trail +- `compass/pipeline/graph.py` — open `AsyncSqliteSaver` once per `run_pipeline` invocation; compile graph inside the `async with`; generate deterministic `thread_id` per job; detect paused vs completed jobs; register paused jobs in the state store +- `compass/pipeline/state.py` — add `thread_id: str | None` to `CompassState` (so `hitl_node` and `vault_write_node` can read it from `RunnableConfig` and surface it in logs/state) +- `compass/mcp_server/server.py` — add `pending_approvals()` and `approve()` tools +- `compass/config.py` — `HITL_CHECKPOINT_DB` env var (defaults to `~/.compass/checkpoints.db`); keep `HITL_STATE_DB` (pending queue) and `HITL_TIMEOUT_HOURS` as-is +- `.env.example` — document `HITL_CHECKPOINT_DB` +- `tests/pipeline/test_routing.py` — remove the three existing `hitl_node` auto-approve tests (lines 46-66); the new behaviour is covered by `test_hitl_node_interrupt.py` +- `tests/pipeline/test_graph_integration.py` — add an `auto_approve_hitl` fixture (monkeypatches `compass.pipeline.nodes.hitl.interrupt` to a stub returning `{"approved": True}`); opt the three above-threshold integration tests into it so they still exercise the full pipeline without needing real checkpoint+resume + +### Untouched +- `compass/pipeline/nodes/{intake,intake_filter,extract,score,reflect,tailor,vault_write}.py` +- `compass/pipeline/role_family.py` +- `compass/vault/*`, `compass/scrapers/*`, `compass/applications/*`, `compass/analysis/*`, `compass/llm.py` +- All Phase 1.A tests (must still pass) + +### Decomposition rationale +The pending-approval queue (`state_store.py`) is intentionally separate from the LangGraph checkpoint DB. The checkpoint DB is LangGraph-owned (schema controlled by `AsyncSqliteSaver`) and stores graph node state; the pending queue is *our* schema, optimized for `pending_approvals()` (we don't need to deserialize a full checkpoint just to list pending jobs by company + title + score). Two SQLite files in `~/.compass/`. The `resume.py` module is the only place that mounts the checkpointer for resumes — keeping it focused makes it the single audit point for "did we recompile the graph correctly with the checkpointer". `timeout_checker.py` calls `resume_pending` rather than reimplementing resume, so the timeout path and the human path go through identical code. + +--- + +## Task 0: Pre-flight + +**Files:** none + +- [ ] **Step 1: Verify clean tree on `phase-1a-application-tracking` tag** + +```bash +cd ~/Documents/compass +git status # expected: clean +git describe --tags --abbrev=0 # expected: phase-1a-application-tracking +uv run pytest -q # expected: 204 passed +uv run ruff check # expected: All checks passed +``` + +If working tree is dirty, STOP and ask the user before proceeding. + +- [ ] **Step 2: Confirm LangGraph + checkpointer libs are at expected versions** + +```bash +uv run python -c " +import importlib.metadata as m +print('langgraph', m.version('langgraph')) +print('langgraph-checkpoint-sqlite', m.version('langgraph-checkpoint-sqlite')) +from langgraph.types import interrupt, Command # smoke +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver # smoke +" +``` + +Expected: `langgraph >= 1.2`, `langgraph-checkpoint-sqlite >= 3.1`, both imports succeed. If lower, ask the user before bumping — version skew bit bug #4 in Phase 0. + +- [ ] **Step 3: Create branch** + +```bash +git checkout -b phase-1b1-hitl +``` + +--- + +## Task 1: Pending-approval state store (`compass/hitl/state_store.py`) + +**Why first:** Pure data layer with no LangGraph dependency. We can TDD it without touching the graph, then plug it in from the orchestrator in Task 3. + +**Schema:** + +```sql +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, -- JSON array + missing_skills TEXT NOT NULL, -- JSON array + created_at TEXT NOT NULL, -- ISO8601 UTC + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +``` + +**Files:** +- Create: `compass/hitl/__init__.py` (empty for now) +- Create: `compass/hitl/state_store.py` +- Create: `tests/hitl/__init__.py` +- Create: `tests/hitl/conftest.py` +- Create: `tests/hitl/test_state_store.py` + +- [ ] **Step 1: Write the `temp_hitl_db` fixture** + +Create `tests/hitl/conftest.py`: + +```python +"""Shared fixtures for HiTL tests.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + + +@pytest.fixture +def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point HITL_STATE_DB at a fresh per-test SQLite file. + + The store reads `compass.config.HITL_STATE_DB` inside function bodies (per + the module-level discipline rule), so monkeypatching the attribute is + sufficient — no module reimport needed. + """ + db = tmp_path / "pending.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_STATE_DB", db) + return db + + +@pytest.fixture +def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: + """Freeze the wall clock the state store uses.""" + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.timezone.utc) + + class _FrozenDT(_dt.datetime): + @classmethod + def now(cls, tz=None): + return fixed if tz is None else fixed.astimezone(tz) + + import compass.hitl.state_store as ss + monkeypatch.setattr(ss, "_now", lambda: fixed) + return fixed +``` + +- [ ] **Step 2: Write the failing tests** + +Create `tests/hitl/test_state_store.py`: + +```python +"""state_store CRUD: add/get/list/resolve + status transitions + serialisation.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.hitl import state_store + + +pytestmark = pytest.mark.usefixtures("temp_hitl_db") + + +async def _add_one(thread_id: str = "tid-1", **overrides) -> None: + defaults = dict( + thread_id=thread_id, + job_url="https://jobs.example.com/abc", + company="Sierra", + title="Software Engineer, Agent", + score=4.2, + score_reasoning="Strong match on MCP + LangGraph.", + matched_skills=["MCP", "Python"], + missing_skills=["LangGraph"], + ) + defaults.update(overrides) + await state_store.add_pending(**defaults) + + +async def test_add_and_get_round_trips(frozen_now): + await _add_one() + row = await state_store.get_pending("tid-1") + assert row is not None + assert row["thread_id"] == "tid-1" + assert row["company"] == "Sierra" + assert row["score"] == pytest.approx(4.2) + assert row["matched_skills"] == ["MCP", "Python"] + assert row["missing_skills"] == ["LangGraph"] + assert row["status"] == "pending" + assert row["created_at"] == frozen_now.isoformat() + assert row["resolved_at"] is None + assert row["feedback"] is None + + +async def test_add_pending_is_idempotent_on_thread_id(): + """Re-running a pipeline that re-pauses the same thread_id is a no-op, not a crash.""" + await _add_one() + # Second call with same thread_id but different score should be a no-op + # (we don't overwrite — the resume must use the original checkpoint). + await _add_one(score=2.0) + row = await state_store.get_pending("tid-1") + assert row["score"] == pytest.approx(4.2) + + +async def test_list_pending_only_returns_pending_status(): + await _add_one(thread_id="tid-pending") + await _add_one(thread_id="tid-approved") + await state_store.mark_resolved("tid-approved", status="approved") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-pending"] + + +async def test_list_pending_orders_oldest_first(): + """The MCP UI shows the queue oldest-first so things don't get lost.""" + import compass.hitl.state_store as ss + # First insertion at frozen_now + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.timezone.utc) + ss._now = lambda: fixed + await _add_one(thread_id="tid-old") + ss._now = lambda: fixed + _dt.timedelta(minutes=5) + await _add_one(thread_id="tid-new") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-old", "tid-new"] + + +async def test_mark_resolved_records_status_and_timestamp(frozen_now): + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="LGTM") + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + assert row["resolved_at"] == frozen_now.isoformat() + + +async def test_mark_resolved_rejects_unknown_status(): + await _add_one() + with pytest.raises(ValueError, match="status"): + await state_store.mark_resolved("tid-1", status="bogus") + + +async def test_mark_resolved_unknown_thread_raises(): + with pytest.raises(LookupError): + await state_store.mark_resolved("tid-missing", status="approved") + + +async def test_list_timed_out_returns_only_old_pending(frozen_now): + import compass.hitl.state_store as ss + # Insert one row "5 hours ago", one "1 hour ago" + old = frozen_now - _dt.timedelta(hours=5) + young = frozen_now - _dt.timedelta(hours=1) + ss._now = lambda: old + await _add_one(thread_id="tid-old") + ss._now = lambda: young + await _add_one(thread_id="tid-young") + ss._now = lambda: frozen_now + rows = await state_store.list_timed_out(timeout_hours=4) + assert [r["thread_id"] for r in rows] == ["tid-old"] + + +async def test_get_pending_unknown_returns_none(): + assert await state_store.get_pending("nope") is None +``` + +Run: + +```bash +uv run pytest tests/hitl/test_state_store.py -v +``` + +Expected: every test fails with `ModuleNotFoundError: compass.hitl.state_store` or `AttributeError`. + +- [ ] **Step 3: Implement `state_store.py`** + +Create `compass/hitl/state_store.py`: + +```python +"""Pending-approval queue for the HiTL flow — aiosqlite, separate from the +LangGraph checkpoint DB. + +Public surface (all coroutines): + add_pending(thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills) + get_pending(thread_id) -> row | None + list_pending() -> list[row] # oldest first + list_timed_out(timeout_hours) -> list[row] # pending AND older than cutoff + mark_resolved(thread_id, status, feedback=None) + +A "row" is a plain dict with the schema documented in the implementation plan. +matched_skills / missing_skills are JSON-encoded in the DB and decoded on read. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +from typing import Any + +import aiosqlite + +logger = logging.getLogger(__name__) + +_VALID_STATUSES = {"pending", "approved", "rejected", "timed_out", "error"} + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, + missing_skills TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +""" + + +def _now() -> _dt.datetime: + """Wall clock as UTC-aware. Indirected so tests can freeze it.""" + return _dt.datetime.now(_dt.timezone.utc) + + +def _db_path(): + """Late-bound HITL_STATE_DB lookup — see module-level discipline rule.""" + import compass.config as cfg + cfg.HITL_STATE_DB.parent.mkdir(parents=True, exist_ok=True) + return cfg.HITL_STATE_DB + + +async def _connect() -> aiosqlite.Connection: + conn = await aiosqlite.connect(_db_path()) + conn.row_factory = aiosqlite.Row + # WAL mode lets concurrent readers + a single writer coexist without + # exclusive-lock contention. Phase 1.B.3 Modal cron + a human pressing + # `approve` in MCP will race on this file; cheaper to set the pragma now + # than debug a flaky lock-timeout in production. PRAGMA is per-DB and + # persists across opens. + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA busy_timeout=5000") # 5s retry on transient lock + await conn.executescript(_SCHEMA) + await conn.commit() + return conn + + +def _row_to_dict(row: aiosqlite.Row) -> dict[str, Any]: + d = dict(row) + d["matched_skills"] = json.loads(d["matched_skills"]) + d["missing_skills"] = json.loads(d["missing_skills"]) + return d + + +async def add_pending( + *, + thread_id: str, + job_url: str, + company: str, + title: str, + score: float, + score_reasoning: str, + matched_skills: list[str], + missing_skills: list[str], +) -> None: + """Insert a new pending row. INSERT OR IGNORE — re-pausing the same thread_id is a no-op.""" + async with await _connect() as conn: + await conn.execute( + """ + INSERT OR IGNORE INTO pending_approvals + (thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills, created_at, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending') + """, + ( + thread_id, + job_url, + company, + title, + float(score), + score_reasoning, + json.dumps(matched_skills), + json.dumps(missing_skills), + _now().isoformat(), + ), + ) + await conn.commit() + + +async def get_pending(thread_id: str) -> dict[str, Any] | None: + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + row = await cur.fetchone() + return _row_to_dict(row) if row else None + + +async def list_pending() -> list[dict[str, Any]]: + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE status = 'pending' " + "ORDER BY created_at ASC" + ) as cur: + rows = await cur.fetchall() + return [_row_to_dict(r) for r in rows] + + +async def list_timed_out(*, timeout_hours: int) -> list[dict[str, Any]]: + cutoff = (_now() - _dt.timedelta(hours=timeout_hours)).isoformat() + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals " + "WHERE status = 'pending' AND created_at < ? " + "ORDER BY created_at ASC", + (cutoff,), + ) as cur: + rows = await cur.fetchall() + return [_row_to_dict(r) for r in rows] + + +async def mark_resolved( + thread_id: str, + *, + status: str, + feedback: str | None = None, +) -> None: + if status not in _VALID_STATUSES or status == "pending": + raise ValueError(f"invalid resolve status: {status!r}") + async with await _connect() as conn: + async with conn.execute( + "SELECT 1 FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + if not await cur.fetchone(): + raise LookupError(f"no pending row for thread_id {thread_id!r}") + await conn.execute( + "UPDATE pending_approvals " + "SET status = ?, feedback = ?, resolved_at = ? " + "WHERE thread_id = ?", + (status, feedback, _now().isoformat(), thread_id), + ) + await conn.commit() +``` + +- [ ] **Step 4: Run the tests, confirm pass** + +```bash +uv run pytest tests/hitl/test_state_store.py -v +``` + +Expected: 9 passed. + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/__init__.py compass/hitl/state_store.py tests/hitl/__init__.py \ + tests/hitl/conftest.py tests/hitl/test_state_store.py +git commit -m "feat(hitl): aiosqlite-backed pending-approval state store" +``` + +--- + +## Task 2: Rewire `hitl_node` to call `interrupt()` + +**Files:** +- Modify: `compass/pipeline/nodes/hitl.py` +- Replace: `tests/pipeline/test_hitl_node.py` → `tests/pipeline/test_hitl_node_interrupt.py` + +**Design:** + +``` +hitl_node(state): + score = state.get("score_result") + if score is None or score.score < SCORE_THRESHOLD: + # auto-reject for missing / low-score — exactly the 1.A behaviour. + # No interrupt fires; the orchestrator never sees this thread. + return {"human_approved": False} + + decision = interrupt({ + "kind": "approval_request", + "job_url": state["current_job"].url, + "company": state["current_job"].company, + "title": state["current_job"].title, + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": score.matched_skills, + "missing_skills": score.missing_skills, + }) + + # On resume, `decision` is whatever was passed to Command(resume=...) + if not isinstance(decision, dict): + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } +``` + +The `interrupt(...)` payload above is what the orchestrator captures (via the `__interrupt__` marker on the returned graph state) to populate `state_store.add_pending(...)`. + +- [ ] **Step 1: Remove the three old auto-approve tests from `test_routing.py`** + +The auto-approve tests live in `tests/pipeline/test_routing.py:46-66` (NOT in a standalone `test_hitl_node.py`). Delete the three test functions: + +- `test_hitl_node_approves_when_score_meets_threshold` +- `test_hitl_node_rejects_when_score_below_threshold` +- `test_hitl_node_handles_missing_score` + +…along with their shared `_state(score)` helper if it's no longer referenced elsewhere in the file. The `reflect_node` tests in the same file stay. Confirm with: + +```bash +grep -n "hitl_node\|SCORE_THRESHOLD" tests/pipeline/test_routing.py +``` + +Expected: no matches after the deletion. If `_state` is still used by reflect tests, keep it; otherwise delete it too. + +- [ ] **Step 2: Write the new interrupt-based tests** + +Create `tests/pipeline/test_hitl_node_interrupt.py`: + +```python +"""hitl_node calls interrupt() above threshold; auto-rejects below threshold.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.pipeline.nodes.hitl import hitl_node +from compass.pipeline.state import CompassState, JobScore, RawJob + + +def _state(score: float | None) -> CompassState: + job = RawJob( + company="Sierra", + title="SWE, Agent", + url="https://jobs.example.com/sierra-1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + sr = ( + None + if score is None + else JobScore( + score=score, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="", + ) + ) + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": sr, + "in_scope": True, + "role_family": "agent-engineer", + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": "tid-test", + } + + +async def test_below_threshold_auto_rejects_without_interrupt(monkeypatch): + """Below SCORE_THRESHOLD short-circuits — interrupt MUST NOT fire (no human prompt cost).""" + called = {"interrupt": 0} + + def boom(_payload): + called["interrupt"] += 1 + raise AssertionError("interrupt should not have been called") + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", boom) + result = await hitl_node(_state(score=2.0)) + assert result == {"human_approved": False} + assert called["interrupt"] == 0 + + +async def test_missing_score_auto_rejects(): + result = await hitl_node(_state(score=None)) + assert result == {"human_approved": False} + + +async def test_above_threshold_calls_interrupt_with_payload(monkeypatch): + captured = {} + + def fake_interrupt(payload): + captured.update(payload) + # Simulate resume value (this is what Command(resume=...) sends back) + return {"approved": True, "feedback": "Strong fit"} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + result = await hitl_node(_state(score=4.2)) + assert captured["kind"] == "approval_request" + assert captured["company"] == "Sierra" + assert captured["score"] == pytest.approx(4.2) + assert captured["matched_skills"] == ["MCP"] + assert result == {"human_approved": True, "human_feedback": "Strong fit"} + + +async def test_resume_rejection_propagates(monkeypatch): + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda _p: {"approved": False, "feedback": "Not a fit"}, + ) + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False, "human_feedback": "Not a fit"} + + +async def test_malformed_resume_value_defaults_to_rejected(monkeypatch): + """If Command(resume=...) somehow sends a non-dict, treat as reject not crash.""" + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", lambda _p: "bogus") + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False} +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_hitl_node_interrupt.py -v +``` + +Expected: all fail (interrupt symbol not imported in hitl.py yet). + +- [ ] **Step 3: Implement the new `hitl_node`** + +Replace `compass/pipeline/nodes/hitl.py` entirely: + +```python +"""hitl_node — Phase 1.B.1 real human-in-the-loop via LangGraph interrupt(). + +Behaviour: + • score < SCORE_THRESHOLD or score missing -> auto-reject, no interrupt fires + • score >= SCORE_THRESHOLD -> interrupt() with the approval payload; the + orchestrator catches the interrupt and registers the thread in + compass.hitl.state_store. A human resumes via Command(resume={...}). +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from langgraph.types import interrupt + +from compass.config import SCORE_THRESHOLD + +if TYPE_CHECKING: + from compass.pipeline.state import CompassState + +logger = logging.getLogger(__name__) + + +async def hitl_node(state: "CompassState") -> dict: + score = state.get("score_result") + job = state.get("current_job") + + if score is None or score.score < SCORE_THRESHOLD: + logger.info( + "hitl: auto-reject %s (score=%s, threshold=%.2f)", + job.url if job else "(unknown)", + getattr(score, "score", None), + SCORE_THRESHOLD, + ) + return {"human_approved": False} + + payload = { + "kind": "approval_request", + "job_url": job.url if job else "", + "company": job.company if job else "", + "title": job.title if job else "", + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": list(score.matched_skills), + "missing_skills": list(score.missing_skills), + } + logger.info("hitl: interrupting for approval — %s (score=%.2f)", payload["job_url"], score.score) + decision = interrupt(payload) + + if not isinstance(decision, dict): + logger.warning("hitl: malformed resume value %r — defaulting to reject", decision) + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } +``` + +- [ ] **Step 4: Add `thread_id` to `CompassState`** + +Edit `compass/pipeline/state.py`. Add inside the `CompassState` TypedDict, after `errors: list[str]`: + +```python + thread_id: str | None +``` + +- [ ] **Step 5: Update the three state-dict literals to include `thread_id`** + +Three places: `compass/pipeline/graph.py:~120` (`_initial_state`), `compass/pipeline/graph.py:~190` (aggregate), `compass/mcp_server/server.py:~82` (the `score_jd` ad-hoc state). Confirm with `grep -n '"raw_jobs"' compass/` — there should be exactly three module-level matches. + +For each: add `"thread_id": None,` to the literal. (We'll set a real value in Task 3.) + +- [ ] **Step 6: Run the new hitl tests** + +```bash +uv run pytest tests/pipeline/test_hitl_node_interrupt.py -v +``` + +Expected: 5 passed. + +- [ ] **Step 7: Add the `auto_approve_hitl` fixture for existing integration tests** + +`tests/pipeline/test_graph_integration.py` has three tests that stub `score=4.2` (above threshold) and expect the pipeline to run to completion: `test_run_pipeline_end_to_end`, `test_run_pipeline_regenerates_gap_plan`, `test_run_pipeline_appends_to_run_log`. With real `interrupt()` they will now pause and the assertions on `jobs_written`, `tailored_paragraph`, etc. will fail. + +Add this fixture at the top of `tests/pipeline/test_graph_integration.py` (next to `mocked_llms`): + +```python +@pytest.fixture +def auto_approve_hitl(monkeypatch): + """Stub the `interrupt()` call in hitl_node so the integration tests can + exercise the full extract -> score -> hitl -> tailor -> vault_write path + without needing a real human resume.""" + def fake_interrupt(_payload): + return {"approved": True, "feedback": None} + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) +``` + +Then add `auto_approve_hitl` to the parameter list of each above-threshold test: + +```python +async def test_run_pipeline_end_to_end(temp_vault, mocked_llms, auto_approve_hitl): +async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms, auto_approve_hitl): +async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms, auto_approve_hitl): +``` + +Leave `test_run_pipeline_skips_tailor_when_below_threshold_but_still_writes` alone — it uses `score=2.0`, never reaches `interrupt()`. + +Leave `test_run_pipeline_skips_dedup_urls` alone — it dedup-drops the job before the graph runs. + +- [ ] **Step 8: Run the full suite — Phase 1.A tests must still pass** + +```bash +uv run pytest -q +``` + +Expected: 204 (pre-existing) − 3 (deleted auto-approve tests in `test_routing.py`) + 5 (new `test_hitl_node_interrupt.py`) = **206 passed**. + +If anything else fails: a state-literal somewhere is missing `thread_id` — find it and add `"thread_id": None,`. Or an above-threshold integration test was missed in Step 7 — opt it into `auto_approve_hitl`. + +- [ ] **Step 9: Commit** + +```bash +git add compass/pipeline/nodes/hitl.py compass/pipeline/state.py \ + compass/pipeline/graph.py compass/mcp_server/server.py \ + tests/pipeline/test_hitl_node_interrupt.py \ + tests/pipeline/test_routing.py \ + tests/pipeline/test_graph_integration.py +git commit -m "feat(hitl): replace auto-approve with LangGraph interrupt()" +``` + +--- + +## Task 2.5: Populate `hitl_decision` + `hitl_at` on the JobNote (audit trail) + +**Why:** `JobNote` has `hitl_decision: str | None` and `hitl_at: datetime | None` fields ([compass/vault/schemas.py:62-63](compass/vault/schemas.py:62)) that have been unpopulated since Phase 0. The whole point of HiTL is that the vault becomes the audit trail for human decisions — without these fields populated, an Obsidian reader can't tell whether a JobNote was approved, rejected, or auto-rejected for low score. + +**Mapping:** +- `state["human_approved"] is True` → `hitl_decision = "approved"` +- `state["human_approved"] is False` and score < threshold → `hitl_decision = "auto_rejected"` +- `state["human_approved"] is False` and score ≥ threshold → `hitl_decision = "rejected"` (human said no on resume) +- `state["human_approved"] is False` and feedback startswith `"auto-cancelled after"` → `hitl_decision = "timed_out"` (timeout_checker path) +- `hitl_at = datetime.now()` whenever any decision was made (i.e. always, except the never-reached-hitl branch which doesn't go through vault_write anyway) + +**Files:** +- Modify: `compass/pipeline/nodes/vault_write.py` +- Modify: `tests/pipeline/test_vault_write.py` + +- [ ] **Step 1: Tighten the `JobNote.hitl_decision` type** + +In `compass/vault/schemas.py`, change: + +```python +hitl_decision: str | None = None +``` + +to: + +```python +HitlDecision = Literal["approved", "rejected", "auto_rejected", "timed_out"] +# ... and on JobNote: +hitl_decision: HitlDecision | None = None +``` + +`Literal` is already imported. All existing vault JobNotes have `hitl_decision: null` (never populated since Phase 0), so the tightening doesn't break round-tripping existing notes. + +- [ ] **Step 2: Rename the test helper if it exists; otherwise create it** + +`tests/pipeline/test_vault_write.py` currently has a `_state(skills_required, skills_matched, score=4.2)` helper at the top of the file used by multiple tests. **Rename it to `_build_state_for_score`** and extend the signature: + +```python +def _build_state_for_score( + *, + score: float = 4.2, + skills_required: list[str] | None = None, + skills_matched: list[str] | None = None, + skills_missing: list[str] | None = None, + human_approved: bool = True, + human_feedback: str | None = None, +) -> "CompassState": + ... +``` + +Update the existing test call sites in the file to use the new name + kwargs (mechanical find/replace; tests that called `_state(4.2, [...], [...])` become `_build_state_for_score(score=4.2, skills_required=[...], skills_matched=[...])`). + +- [ ] **Step 3: Write the failing tests** + +Append to `tests/pipeline/test_vault_write.py`: + +```python +async def test_vault_write_records_approved_decision(temp_vault): + """state['human_approved'] = True -> hitl_decision='approved', hitl_at set.""" + import datetime as _dt + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + from compass.pipeline.state import JobScore, JobRequirements, RawJob + + state = _build_state_for_score( # existing helper in this file + score=4.5, + human_approved=True, + human_feedback="LGTM", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "approved" + assert "hitl_at" in md and md["hitl_at"] is not None + + +async def test_vault_write_records_auto_rejected_for_low_score(temp_vault): + """Below-threshold path: hitl never interrupts; decision is 'auto_rejected'.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=2.0, human_approved=False, human_feedback=None) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "auto_rejected" + + +async def test_vault_write_records_rejected_when_human_said_no(temp_vault): + """Above-threshold path with human_approved=False = explicit reject.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=4.2, human_approved=False, human_feedback="not a fit") + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "rejected" + + +async def test_vault_write_records_timed_out(temp_vault): + """Timeout-checker resume sets feedback='auto-cancelled after Xh timeout'.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.2, + human_approved=False, + human_feedback="auto-cancelled after 4h timeout", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "timed_out" +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_vault_write.py -v -k "hitl_decision or auto_rejected or rejected_when_human or timed_out" +``` + +Expected: 4 failures (the fields aren't populated yet). + +- [ ] **Step 4: Implement the mapping in `vault_write_node`** + +In `compass/pipeline/nodes/vault_write.py`, before building the `JobNote(...)` kwargs, add: + +```python +def _derive_hitl_decision(state: "CompassState") -> tuple[str | None, "datetime | None"]: + """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" + from datetime import datetime as _dt + + from compass.config import SCORE_THRESHOLD + + approved = state.get("human_approved") + if approved is None: + # hitl never reached (e.g. extract/score errored). Leave fields null. + return (None, None) + + feedback = (state.get("human_feedback") or "").lower() + score = state.get("score_result") + score_value = score.score if score is not None else 0.0 + + if approved is True: + decision = "approved" + elif feedback.startswith("auto-cancelled after"): + decision = "timed_out" + elif score_value < SCORE_THRESHOLD: + decision = "auto_rejected" + else: + decision = "rejected" + return (decision, _dt.now()) +``` + +Wire it into the JobNote construction (replace the existing `JobNote(...)` call's tailored_paragraph line region): + +```python + hitl_decision, hitl_at = _derive_hitl_decision(state) + note = JobNote( + # … all existing fields … + tailored_paragraph=state.get("tailored_paragraph"), + hitl_decision=hitl_decision, + hitl_at=hitl_at, + ) +``` + +- [ ] **Step 5: Run the new tests** + +```bash +uv run pytest tests/pipeline/test_vault_write.py -v +``` + +Expected: all previous + 4 new pass. + +- [ ] **Step 6: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 210 passed (206 from Task 2 + 4 new). + +- [ ] **Step 7: Commit** + +```bash +git add compass/pipeline/nodes/vault_write.py tests/pipeline/test_vault_write.py \ + compass/vault/schemas.py +git commit -m "feat(hitl): record hitl_decision + hitl_at on JobNote" +``` + +--- + +## Task 3: Wire `AsyncSqliteSaver` + register paused threads in `run_pipeline` + +**The risky one.** This is where the new graph topology meets the new state store. Key invariants we must preserve: + +1. **The checkpointer is opened ONCE per `run_pipeline` invocation**, inside an `async with` block. Reusing a closed checkpointer across batches is a known LangGraph foot-gun. +2. **`build_graph()` is called inside the `async with`** so the compiled graph holds the open checkpointer. Compiling at module level (or before the `async with`) silently breaks `interrupt()`. Do not refactor this for "neatness" later — the Phase 1.A handoff calls this out by name. +3. **The `thread_id` is deterministic from `(job.url, scrape_timestamp)`** so a rerun of the same batch reuses the same paused checkpoint. We hash `f"{job.url}|{start_wall.isoformat()}"` with SHA-1 truncated to 16 chars. +4. **After `graph.ainvoke(...)` returns**, inspect the result for `__interrupt__` markers. If present, the job is paused — call `state_store.add_pending(...)` and treat it as "not written, not errored, pending". Otherwise the job ran to completion as before. + +**Files:** +- Modify: `compass/pipeline/graph.py` +- Modify: `compass/config.py` (add `HITL_CHECKPOINT_DB`) +- Modify: `.env.example` +- Create: `tests/pipeline/test_graph_checkpointing.py` + +- [ ] **Step 1: Add `HITL_CHECKPOINT_DB` to config** + +Edit `compass/config.py`, in the `── HiTL ──` section: + +```python +HITL_STATE_DB: Path = Path(os.getenv("HITL_STATE_DB", "~/.compass/hitl.db")).expanduser() +HITL_CHECKPOINT_DB: Path = Path( + os.getenv("HITL_CHECKPOINT_DB", "~/.compass/checkpoints.db") +).expanduser() +HITL_TIMEOUT_HOURS: int = int(os.getenv("HITL_TIMEOUT_HOURS", "4")) +``` + +Edit `.env.example`, in the `── HiTL ──` section: + +``` +HITL_STATE_DB=~/.compass/hitl.db +HITL_CHECKPOINT_DB=~/.compass/checkpoints.db +HITL_TIMEOUT_HOURS=4 +``` + +- [ ] **Step 2: Write the integration test for graph checkpointing** + +Create `tests/pipeline/test_graph_checkpointing.py`: + +```python +"""End-to-end: graph pauses at hitl, state_store gets the row, + resume_pending finishes the run. + + These tests STUB the LLM-touching nodes (extract, score, tailor) — we only + exercise the graph machinery + interrupt + checkpointer + state_store + interaction. Real LLM calls live in the live-smoke check in Task 7.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + """Replace extract / score / tailor / vault_write with deterministic stubs.""" + async def fake_extract(state: CompassState) -> dict: + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP", "Python"], + nice_to_have_skills=["LangGraph"], + seniority="mid", + remote_policy="remote", + summary="An agent role.", + ) + } + + async def fake_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=4.2, + reasoning="Strong match.", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="Lead with MCP.", + ) + } + + async def fake_tailor(state: CompassState) -> dict: + return {"tailored_paragraph": "Tailored: lead with MCP work."} + + async def fake_vault_write(state: CompassState) -> dict: + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(state: CompassState) -> dict: + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +def _job(url: str = "https://jobs.example.com/sierra-1") -> RawJob: + return RawJob( + company="Sierra", + title="SWE, Agent", + url=url, + source="ashby", + description="An agent engineering role at Sierra.", + date_posted=_dt.date(2026, 5, 18), + ) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_above_threshold_job_pauses_and_registers_in_state_store(monkeypatch): + """run_pipeline with a single above-threshold job: paused=1, written=0, + state_store has a row.""" + from compass.pipeline.graph import run_pipeline + + # No real scraping — pass the job in directly. + result = await run_pipeline(raw_jobs=[_job()]) + + assert result["jobs_processed"] == 1 + assert result["jobs_written"] == 0 # paused before vault_write + assert result["jobs_paused"] == 1 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["company"] == "Sierra" + assert rows[0]["score"] == pytest.approx(4.2) + assert rows[0]["job_url"] == "https://jobs.example.com/sierra-1" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_below_threshold_job_runs_to_completion_no_state_store_row(monkeypatch): + """Below-threshold path is unchanged from 1.A — no interrupt, vault_write fires.""" + async def low_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=2.0, + reasoning="Mismatched.", + matched_skills=[], + missing_skills=["A", "B"], + tailoring_notes="", + ) + } + monkeypatch.setattr("compass.pipeline.graph.score_node", low_score) + from compass.pipeline.graph import run_pipeline + + result = await run_pipeline(raw_jobs=[_job()]) + assert result["jobs_written"] == 1 + assert result["jobs_paused"] == 0 + assert await state_store.list_pending() == [] + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_thread_id_is_deterministic_for_same_url_and_batch(): + """Re-running the SAME batch (same start_wall) reuses the same thread_id — + second run hits the idempotent INSERT OR IGNORE path.""" + from compass.pipeline.graph import _thread_id_for + tid_a = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_b = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_c = _thread_id_for("https://jobs.example.com/y", _dt.datetime(2026, 5, 19, 9, 0, 0)) + assert tid_a == tid_b + assert tid_a != tid_c + assert len(tid_a) == 16 +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_graph_checkpointing.py -v +``` + +Expected: all three fail (run_pipeline doesn't take `raw_jobs` paths through this flow yet, `_thread_id_for` doesn't exist, `jobs_paused` not in aggregate). + +- [ ] **Step 3: Rewrite the relevant chunks of `graph.py`** + +Apply the following surgical edits to `compass/pipeline/graph.py`: + +**3a. Add new imports near the top:** + +```python +import hashlib + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + +from compass.hitl import state_store +``` + +**3b. Add `_thread_id_for` as a module-level helper, just above `_initial_state`:** + +```python +def _thread_id_for(job_url: str, batch_started_at: datetime) -> str: + """Deterministic 16-char SHA-1 of (url, batch start) — same batch + same job = same thread.""" + raw = f"{job_url}|{batch_started_at.isoformat()}" + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] +``` + +**3c. Extend `_initial_state` to accept and store `thread_id`:** + +```python +def _initial_state(job: RawJob, thread_id: str | None = None) -> CompassState: + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": None, + "in_scope": None, + "role_family": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": thread_id, + } +``` + +**3d. Replace `_process_one` with a checkpointer-aware version that detects paused state:** + +```python +async def _process_one( + graph, + job: RawJob, + sem: asyncio.Semaphore, + thread_id: str, +) -> tuple[CompassState, bool]: + """Invoke the graph for one job. Returns (final_state, was_paused).""" + config = { + "configurable": {"thread_id": thread_id}, + **_langfuse_config(), + } + async with sem: + try: + result = await graph.ainvoke(_initial_state(job, thread_id=thread_id), config=config) + except Exception as e: + logger.exception("pipeline: graph crashed on %s", job.url) + return ( + {**_initial_state(job, thread_id=thread_id), "errors": [f"graph: {type(e).__name__}: {e}"]}, + False, + ) + + # When interrupt() fires, ainvoke returns with state containing the + # __interrupt__ marker AND without progressing past the hitl node. + # vault_written stays False, jobs_written stays 0 — that's our signal. + if "__interrupt__" in result and result["__interrupt__"]: + interrupts = result["__interrupt__"] + payload = interrupts[0].value if hasattr(interrupts[0], "value") else interrupts[0] + if isinstance(payload, dict) and payload.get("kind") == "approval_request": + await state_store.add_pending( + thread_id=thread_id, + job_url=payload["job_url"], + company=payload["company"], + title=payload["title"], + score=float(payload["score"]), + score_reasoning=payload["score_reasoning"], + matched_skills=list(payload["matched_skills"]), + missing_skills=list(payload["missing_skills"]), + ) + logger.info( + "pipeline: paused %s for approval (thread_id=%s, score=%.2f)", + job.url, thread_id, payload["score"], + ) + return (result, True) + # Unknown interrupt shape — DO NOT silently swallow. Phase 0 bug pattern: + # a future interrupt() added elsewhere in the graph would otherwise + # disappear into a "succeeded with jobs_paused=0" black hole. Log loudly + # and still count as paused so the caller's bookkeeping reflects reality. + logger.error( + "pipeline: graph paused at UNKNOWN interrupt kind for %s — payload=%r", + job.url, payload, + ) + return (result, True) + return (result, False) +``` + +**3e. Restructure `run_pipeline` to mount the checkpointer:** + +```python +async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: + """Scrape (or accept) jobs, dedup, run per-job graph under a single + AsyncSqliteSaver, regenerate gap plan.""" + from compass.config import HITL_CHECKPOINT_DB + + start_monotonic = time.monotonic() + start_wall = datetime.now() + if raw_jobs is None: + raw_jobs = await _scrape_all() + + seen_urls = _vault_url_set() + fresh = [j for j in raw_jobs if j.url not in seen_urls] + dropped = len(raw_jobs) - len(fresh) + if dropped: + logger.info("pipeline: dropping %d/%d jobs already in vault", dropped, len(raw_jobs)) + + HITL_CHECKPOINT_DB.parent.mkdir(parents=True, exist_ok=True) + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + # Enable WAL on the checkpoint DB once per process. Cheap if already set. + # See state_store._connect for the rationale. + try: + await checkpointer.conn.execute("PRAGMA journal_mode=WAL") + await checkpointer.conn.execute("PRAGMA busy_timeout=5000") + except Exception: + logger.debug("checkpoint: WAL pragma set already or unsupported; continuing") + graph = build_graph(checkpointer=checkpointer) + sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) + coros = [ + _process_one(graph, j, sem, thread_id=_thread_id_for(j.url, start_wall)) + for j in fresh + ] + results = await asyncio.gather(*coros) + + pairs = results # list of (state, was_paused) + paused_count = sum(int(p) for _, p in pairs) + final_states = [s for s, _ in pairs] + + aggregate: CompassState = { + "raw_jobs": raw_jobs, + "current_job": None, + "extracted_requirements": None, + "score_result": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": any(r.get("vault_written") for r in final_states), + "jobs_processed": len(fresh), + "jobs_written": sum(int(bool(r.get("vault_written"))) for r in final_states), + "errors": [e for r in final_states for e in r.get("errors", [])], + "in_scope": None, + "role_family": None, + "thread_id": None, + } + # `jobs_paused` is informational only — not in CompassState TypedDict. + # Returned via the aggregate dict; orchestrators (CLI, MCP) read it. + aggregate["jobs_paused"] = paused_count # type: ignore[typeddict-unknown-key] + + if aggregate["jobs_written"] > 0: + gap_aggregator.regenerate(write=True) + + duration_s = time.monotonic() - start_monotonic + _append_run_log(aggregate, duration_s) + unknown_count = _count_unknown_skills_seen_this_run(start_wall) + logger.info( + "pipeline: processed=%d written=%d paused=%d errors=%d unknown_skills_seen=%d duration=%.1fs", + aggregate["jobs_processed"], aggregate["jobs_written"], paused_count, + len(aggregate["errors"]), unknown_count, duration_s, + ) + return aggregate +``` + +**3f. Make `build_graph` accept a checkpointer:** + +```python +def build_graph(checkpointer=None): + builder = StateGraph(CompassState) + # ... unchanged node + edge wiring ... + return builder.compile(checkpointer=checkpointer) +``` + +**3g. Update `_append_run_log` to record the paused column:** + +```python +def _append_run_log(state: CompassState, duration_s: float) -> None: + log_path = VAULT_PATH / "_meta" / "pipeline-runs.md" + log_path.parent.mkdir(parents=True, exist_ok=True) + if not log_path.exists(): + log_path.write_text( + "# Pipeline Run Log\n\n" + "| Timestamp | Processed | Written | Paused | Errors | Duration |\n" + "|---|---|---|---|---|---|\n", + encoding="utf-8", + ) + ts = datetime.now().isoformat(timespec="seconds") + paused = state.get("jobs_paused", 0) # type: ignore[typeddict-item] + row = ( + f"| {ts} | {state['jobs_processed']} | {state['jobs_written']} | " + f"{paused} | {len(state['errors'])} | {duration_s:.1f}s |\n" + ) + with log_path.open("a", encoding="utf-8") as f: + f.write(row) +``` + +**3h. Update the `__main__` print:** + +```python +if __name__ == "__main__": + result = asyncio.run(run_pipeline()) + print( + f"Processed: {result['jobs_processed']} | " + f"Written: {result['jobs_written']} | " + f"Paused: {result.get('jobs_paused', 0)} | " + f"Errors: {len(result['errors'])}" + ) +``` + +- [ ] **Step 4: Run the new checkpointing tests** + +```bash +uv run pytest tests/pipeline/test_graph_checkpointing.py -v +``` + +Expected: 3 passed. If a test fails because `__interrupt__` isn't in the returned state, drop a `print(result)` after the ainvoke to inspect the exact shape — LangGraph 1.x has been seen to also expose interrupts via `graph.get_state(config).next` instead. If so, switch the detection to `state = await graph.aget_state(config); was_paused = bool(state.next)`. + +- [ ] **Step 5: Run the full suite — Phase 1.A tests must still pass** + +```bash +uv run pytest -q +``` + +Expected: 213 passed (210 from Task 2.5 + 3 new). + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/graph.py compass/config.py .env.example \ + tests/pipeline/test_graph_checkpointing.py +git commit -m "feat(hitl): mount AsyncSqliteSaver in run_pipeline; register paused threads" +``` + +--- + +## Task 4: Resume entrypoint (`compass/hitl/resume.py`) + +**Files:** +- Create: `compass/hitl/resume.py` +- Create: `tests/hitl/test_resume.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/hitl/test_resume.py`: + +```python +"""resume_pending re-opens the checkpointer, recompiles the graph, and drives + the resume via Command(resume=...). Verifies vault_write fires on approve + and tailor is skipped on reject.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="agent role")} + + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes="")} + + async def fake_tailor(_): return {"tailored_paragraph": "lead with MCP."} + + written = {"calls": 0, "with_tailor": 0} + async def fake_vault_write(state): + written["calls"] += 1 + if state.get("tailored_paragraph"): + written["with_tailor"] += 1 + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + return written + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_approve_runs_tailor_then_vault_write(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + + pending = await state_store.list_pending() + tid = pending[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": True, "feedback": "LGTM"}) + assert final["vault_written"] is True + assert final["human_approved"] is True + assert stub_llm_nodes["with_tailor"] == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_reject_skips_tailor_writes_to_vault(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/2", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": False}) + assert final["human_approved"] is False + assert stub_llm_nodes["with_tailor"] == 0 + assert stub_llm_nodes["calls"] == 1 # vault_write still fired (rejected branch) + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_unknown_thread_raises(): + from compass.hitl.resume import resume_pending + with pytest.raises(LookupError, match="thread"): + await resume_pending("nope-1234", decision={"approved": True}) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_already_resolved_is_noop(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/3", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + await resume_pending(tid, decision={"approved": True}) + + # Second resume call should not crash and should not re-fire vault_write + before = stub_llm_nodes["calls"] + with pytest.raises(ValueError, match="already resolved"): + await resume_pending(tid, decision={"approved": True}) + assert stub_llm_nodes["calls"] == before +``` + +Run — expect all 4 failing with `ModuleNotFoundError`. + +- [ ] **Step 2: Implement `resume.py`** + +Create `compass/hitl/resume.py`: + +```python +"""Resume a paused LangGraph thread by re-opening the checkpointer and + recompiling the graph. Single source of truth for resumes — the timeout + checker and MCP `approve` tool both go through here.""" + +from __future__ import annotations + +import logging +from typing import Any + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver +from langgraph.types import Command + +from compass.hitl import state_store + +logger = logging.getLogger(__name__) + + +async def resume_pending( + thread_id: str, + *, + decision: dict[str, Any], + status_override: str | None = None, +) -> dict: + """Drive a paused thread to completion. + + decision: passed as `Command(resume=decision)`. Expected shape + {"approved": bool, "feedback": str | None}. + status_override: optional explicit status to write to the state store + (timeout_checker uses "timed_out"). If None, derived from + decision["approved"]: True -> "approved", False -> "rejected". + Raises LookupError if the thread isn't in the state store. + Raises ValueError if the thread is already resolved. + """ + from compass.config import HITL_CHECKPOINT_DB + + row = await state_store.get_pending(thread_id) + if row is None: + raise LookupError(f"no pending thread {thread_id!r}") + if row["status"] != "pending": + raise ValueError(f"thread {thread_id!r} already resolved ({row['status']})") + + # Late import to avoid circular: graph.py imports state_store + from compass.pipeline.graph import build_graph + + config = {"configurable": {"thread_id": thread_id}} + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + resolved_status = "approved" if decision.get("approved") else "rejected" + await state_store.mark_resolved( + thread_id, + status=resolved_status, + feedback=decision.get("feedback"), + ) + logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) + return final +``` + +- [ ] **Step 3: Run resume tests** + +```bash +uv run pytest tests/hitl/test_resume.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 217 passed (213 + 4). + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/resume.py tests/hitl/test_resume.py +git commit -m "feat(hitl): resume_pending entrypoint via Command(resume=...)" +``` + +--- + +## Task 5: Timeout checker + +**Files:** +- Create: `compass/hitl/timeout_checker.py` +- Create: `tests/hitl/test_timeout_checker.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/hitl/test_timeout_checker.py`: + +```python +"""timeout_checker resumes pending rows older than HITL_TIMEOUT_HOURS with + {"approved": False}, marks them as 'timed_out' in the state store, and + leaves young pending rows untouched.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="x")} + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes="")} + async def fake_vault_write(_): return {"vault_written": True, "jobs_written": 1} + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + async def fake_tailor(_): return {"tailored_paragraph": "..."} + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_resumes_old_pending_as_rejected(monkeypatch): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the row's created_at to look 5h old + import aiosqlite + from compass.config import HITL_STATE_DB + five_h_ago = (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute( + "UPDATE pending_approvals SET created_at = ? WHERE thread_id = ?", + (five_h_ago, tid), + ) + await conn.commit() + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "timed_out" + # Spec DoD line 230 — timeout must be logged to _meta/agent-log.md + from compass.config import AGENT_LOG_PATH + log_text = AGENT_LOG_PATH.read_text(encoding="utf-8") + assert "hitl-timeout" in log_text + assert tid in log_text + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_leaves_young_pending_alone(): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/2", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 0 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["status"] == "pending" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_continues_after_one_resume_failure(monkeypatch): + """A single broken thread (e.g. checkpoint corrupted) must not block the + rest of the queue from timing out.""" + from compass.hitl import timeout_checker + + # Two rows, both stale + await state_store.add_pending( + thread_id="tid-broken", job_url="x://1", company="C", title="T", + score=4.0, score_reasoning="r", matched_skills=[], missing_skills=[], + ) + await state_store.add_pending( + thread_id="tid-ok", job_url="x://2", company="C", title="T", + score=4.0, score_reasoning="r", matched_skills=[], missing_skills=[], + ) + # Backdate both + import aiosqlite + from compass.config import HITL_STATE_DB + five_h_ago = (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute("UPDATE pending_approvals SET created_at = ?", (five_h_ago,)) + await conn.commit() + + # Make resume_pending raise for tid-broken + async def flaky(thread_id, **kw): + if thread_id == "tid-broken": + raise RuntimeError("checkpoint missing") + from compass.hitl.state_store import mark_resolved + await mark_resolved(thread_id, status="timed_out") + monkeypatch.setattr("compass.hitl.timeout_checker.resume_pending", flaky) + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 # one succeeded, one failed + row_broken = await state_store.get_pending("tid-broken") + assert row_broken["status"] == "error" +``` + +- [ ] **Step 2: Implement `timeout_checker.py`** + +Create `compass/hitl/timeout_checker.py`: + +```python +"""Auto-cancel pending approvals older than HITL_TIMEOUT_HOURS. + +Module entrypoint (CLI): python -m compass.hitl.timeout_checker +Async API: check_and_resume_timeouts(timeout_hours: int | None = None) + +In Phase 1.B.1 this is human-run. In Phase 1.B.3 a Modal cron will import and +schedule it — see modal_app.py.""" + +from __future__ import annotations + +import asyncio +import logging + +from compass.hitl import state_store +from compass.hitl.resume import resume_pending + +logger = logging.getLogger(__name__) + + +async def check_and_resume_timeouts(*, timeout_hours: int | None = None) -> int: + """Resume every stale pending row with {"approved": False}. Returns the + count successfully timed-out (i.e. exclude rows that errored).""" + from compass.config import HITL_TIMEOUT_HOURS + + hrs = timeout_hours if timeout_hours is not None else HITL_TIMEOUT_HOURS + stale = await state_store.list_timed_out(timeout_hours=hrs) + if not stale: + return 0 + + logger.info("hitl: %d pending approval(s) past %dh timeout", len(stale), hrs) + timed_out = 0 + for row in stale: + tid = row["thread_id"] + try: + await resume_pending( + tid, + decision={"approved": False, "feedback": f"auto-cancelled after {hrs}h timeout"}, + status_override="timed_out", + ) + # Spec § DoD line 230: HiTL timeout must log to _meta/agent-log.md + _log_to_agent_log( + f"hitl-timeout: auto-cancelled {tid} ({row['company']} / {row['title']}, " + f"score={row['score']:.2f}) after {hrs}h" + ) + timed_out += 1 + except Exception as e: + logger.exception("hitl: timeout resume failed for %s — marking as error", tid) + try: + await state_store.mark_resolved(tid, status="error", feedback=f"resume error: {e}") + _log_to_agent_log(f"hitl-timeout-error: {tid} resume failed: {e}") + except Exception: + logger.exception("hitl: also failed to mark %s as error", tid) + return timed_out + + +def _log_to_agent_log(line: str) -> None: + """Append a timestamped row to compass-vault/_meta/agent-log.md. + + Late-binds via compass.vault.writer.append_agent_log so the temp_vault + fixture works in tests. + """ + from compass.vault.writer import append_agent_log + try: + append_agent_log(line) + except Exception: + logger.exception("hitl: failed to write to agent-log.md") + + +if __name__ == "__main__": + n = asyncio.run(check_and_resume_timeouts()) + print(f"Timed out: {n}") +``` + +- [ ] **Step 3: Run tests** + +```bash +uv run pytest tests/hitl/test_timeout_checker.py -v +``` + +Expected: 3 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 220 passed (217 + 3). + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/timeout_checker.py tests/hitl/test_timeout_checker.py +git commit -m "feat(hitl): timeout_checker auto-cancels stale approvals" +``` + +--- + +## Task 6: MCP tools (`pending_approvals` + `approve`) + +**Files:** +- Modify: `compass/mcp_server/server.py` +- Create: `tests/mcp_server/__init__.py` if missing +- Create: `tests/mcp_server/test_pending_approvals.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/mcp_server/test_pending_approvals.py`: + +```python +"""MCP wrappers around state_store + resume_pending — JSON-serializable + return shapes (no datetime objects, no Pydantic instances).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from compass.hitl import state_store + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_pending_approvals_tool_returns_jsonable_rows(): + from compass.mcp_server.server import pending_approvals + + await state_store.add_pending( + thread_id="tid-1", job_url="https://x/1", company="Sierra", + title="SWE Agent", score=4.2, score_reasoning="solid", + matched_skills=["MCP"], missing_skills=["LangGraph"], + ) + rows = await pending_approvals() + assert len(rows) == 1 + assert rows[0]["thread_id"] == "tid-1" + # Must be plain JSON-serialisable + json.dumps(rows) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_invokes_resume(monkeypatch): + from compass.mcp_server import server as mcp_server + + captured = {} + async def fake_resume(thread_id, *, decision, status_override=None): + captured["thread_id"] = thread_id + captured["decision"] = decision + captured["status_override"] = status_override + return {"vault_written": True, "human_approved": decision["approved"]} + + monkeypatch.setattr(mcp_server, "_resume_pending", fake_resume) + + result = await mcp_server.approve(thread_id="tid-x", approved=True, feedback="LGTM") + assert result == {"vault_written": True, "human_approved": True} + assert captured == { + "thread_id": "tid-x", + "decision": {"approved": True, "feedback": "LGTM"}, + "status_override": None, + } + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_handles_unknown_thread_as_error_dict(): + from compass.mcp_server.server import approve + result = await approve(thread_id="missing", approved=False) + assert "error" in result and "missing" in result["error"] +``` + +- [ ] **Step 2: Add the MCP tools** + +Append to `compass/mcp_server/server.py` (after the existing `tailor_resume` block): + +```python +# ── HiTL approvals ─────────────────────────────────────────────────────────── + +from compass.hitl import state_store as _state_store +from compass.hitl.resume import resume_pending as _resume_pending + + +@mcp.tool() +async def pending_approvals() -> list[dict]: + """List jobs paused at hitl awaiting human approval. Oldest first.""" + rows = await _state_store.list_pending() + # rows are already plain dicts of JSON-safe primitives + list[str] + return rows + + +@mcp.tool() +async def approve(thread_id: str, approved: bool, feedback: str | None = None) -> dict: + """Resume a paused thread. approved=True runs tailor + vault_write; + approved=False skips tailor and writes the rejected JobNote.""" + try: + final = await _resume_pending( + thread_id, + decision={"approved": approved, "feedback": feedback}, + ) + except (LookupError, ValueError) as e: + return {"error": str(e)} + # Strip non-JSON-safe state from the return (current_job is a Pydantic model) + return { + "vault_written": bool(final.get("vault_written")), + "human_approved": bool(final.get("human_approved")), + "human_feedback": final.get("human_feedback"), + } +``` + +Also update the module docstring's tool listing to include `pending_approvals()` and `approve(thread_id, approved, feedback)`. + +- [ ] **Step 3: Run MCP tests** + +```bash +mkdir -p tests/mcp_server && touch tests/mcp_server/__init__.py +uv run pytest tests/mcp_server/test_pending_approvals.py -v +``` + +Expected: 3 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 223 passed (220 + 3). + +- [ ] **Step 5: Lint** + +```bash +uv run ruff check +uv run ruff format --check +``` + +Expected: clean. If `format --check` complains, run `uv run ruff format` and re-stage. + +- [ ] **Step 6: Commit** + +```bash +git add compass/mcp_server/server.py tests/mcp_server/__init__.py \ + tests/mcp_server/test_pending_approvals.py +git commit -m "feat(mcp): pending_approvals + approve tools" +``` + +--- + +## Task 7: Live smoke + checkpoint-survives-restart verification + +> **PAUSE HERE before running this task.** Step 2 hits real OpenRouter + writes to the user's real vault + writes to `~/.compass/checkpoints.db`. Confirm with the user before running. This is the only LLM-touching step in the whole plan. + +**Goal:** Prove the three things tests cannot prove: +1. A real graph invocation with a real LLM score actually pauses at hitl +2. The pause SURVIVES a Python process restart (closing + re-opening the AsyncSqliteSaver from a fresh process) +3. Resume via `approve()` MCP tool produces a real JobNote with a real tailored paragraph + +**Files:** none + +- [ ] **Step 1: Snapshot vault + DBs** + +```bash +cp -r ~/Documents/compass-vault/jobs ~/Documents/compass-vault/jobs.preB1.bak 2>/dev/null || true +cp ~/.compass/hitl.db ~/.compass/hitl.db.preB1.bak 2>/dev/null || true +cp ~/.compass/checkpoints.db ~/.compass/checkpoints.db.preB1.bak 2>/dev/null || true +``` + +- [ ] **Step 2: Run pipeline against one apply-now board, expect paused jobs** + +```bash +cd ~/Documents/compass +MAX_JOBS_PER_RUN=10 \ + GREENHOUSE_BOARDS= \ + LEVER_COMPANIES= \ + ASHBY_BOARDS=sierra \ + uv run python -m compass.pipeline.graph +``` + +Expected output: +``` +Processed: N | Written: M | Paused: K | Errors: 0 +``` +where `K >= 1` (at least one above-threshold Sierra job should pause). + +- [ ] **Step 3: Confirm pending row exists from a SEPARATE Python process** + +This is the critical "checkpoint survives restart" check. Run a fresh process: + +```bash +uv run python -c " +import asyncio +from compass.hitl import state_store +async def main(): + rows = await state_store.list_pending() + print(f'Pending: {len(rows)}') + for r in rows: + print(f' {r[\"thread_id\"]} {r[\"company\"]} / {r[\"title\"]} score={r[\"score\"]}') +asyncio.run(main()) +" +``` + +Expected: at least one row printed with a real Sierra job. + +- [ ] **Step 4: Resume one thread via the MCP path, confirm JobNote written** + +Pick a `thread_id` from Step 3's output, then run (replace ``): + +```bash +uv run python -c " +import asyncio +from compass.hitl.resume import resume_pending +async def main(): + final = await resume_pending('', decision={'approved': True, 'feedback': 'smoke test'}) + print('vault_written =', final.get('vault_written')) + print('human_approved =', final.get('human_approved')) +asyncio.run(main()) +" +``` + +Expected: `vault_written = True`, `human_approved = True`. Then verify the JobNote: + +```bash +ls -ltr ~/Documents/compass-vault/jobs/ | tail -3 +``` + +The most recent file should be the Sierra job. Open it and confirm: +- `tailored_paragraph:` is populated with real Sonnet output (not empty) +- `match_score:` is the same float seen in Step 3 +- `hitl_decision: approved` in the frontmatter (NOT `human_approved` — that key lives in pipeline state, not the JobNote) +- `hitl_at:` is a recent ISO timestamp + +- [ ] **Step 5: Run `timeout_checker` against an empty queue, confirm 0 timed out** + +```bash +uv run python -m compass.hitl.timeout_checker +``` + +Expected: `Timed out: 0` (the one row we approved is no longer pending). + +- [ ] **Step 6: Restore backups if anything went wrong; otherwise delete them** + +```bash +# If anything looks wrong (no tailored paragraph, mis-sorted, etc.), restore: +# rm -rf ~/Documents/compass-vault/jobs && mv ~/Documents/compass-vault/jobs.preB1.bak ~/Documents/compass-vault/jobs +# mv ~/.compass/hitl.db.preB1.bak ~/.compass/hitl.db +# mv ~/.compass/checkpoints.db.preB1.bak ~/.compass/checkpoints.db +# If everything looks good: +rm -rf ~/Documents/compass-vault/jobs.preB1.bak ~/.compass/hitl.db.preB1.bak ~/.compass/checkpoints.db.preB1.bak +``` + +- [ ] **Step 7: Tag the phase** + +```bash +git tag phase-1b1-hitl +``` + +--- + +## Definition of Done + +All of the following must hold before declaring Phase 1.B.1 complete: + +**Code & tests** +- 223 tests passing (`uv run pytest -q`) +- Ruff clean (`uv run ruff check && uv run ruff format --check`) +- No module-level `from compass.config import HITL_*` — every reference is inside a function body +- `build_graph()` accepts an optional `checkpointer` parameter; called only from inside the `async with AsyncSqliteSaver(...)` block in `run_pipeline` AND from `resume_pending` + +**Behaviour (verified empirically in Task 7)** +- A real above-threshold job pauses at `hitl`, registers in `~/.compass/hitl.db`, does NOT write a JobNote +- The pending row is visible from a fresh Python process (proves the checkpoint isn't process-local) +- `resume_pending(tid, decision={"approved": True})` writes a JobNote with a non-empty `tailored_paragraph` and `hitl_decision: approved` +- `resume_pending(tid, decision={"approved": False})` writes a JobNote with `hitl_decision: rejected` and no `tailored_paragraph` +- `timeout_checker.check_and_resume_timeouts()` resumes only rows older than `HITL_TIMEOUT_HOURS`, marks them `timed_out`, AND appends a row to `_meta/agent-log.md` (spec § DoD line 230) +- Below-threshold jobs are unchanged from Phase 1.A — no interrupt fires, no state-store row +- An unknown `interrupt()` payload (anything not `kind == "approval_request"`) is logged at ERROR and counted as `jobs_paused`, never silently dropped + +**Docs** +- `.env.example` documents `HITL_CHECKPOINT_DB` +- This plan's "What's deferred" cross-references are accurate (RAG → 1.B.2, Modal cron → 1.B.3) +- Final commit message of Task 7: `git tag phase-1b1-hitl` + +--- + +## What's deferred (and to which sub-phase) + +| Concern | Severity | Phase | Why deferred | +|---|---|---|---| +| Chroma RAG for profile retrieval | Portfolio claim | **1.B.2** | Independent of HiTL; separate plan | +| Modal cron schedule (`@app.function(schedule=Cron(...))`) for `timeout_checker` + daily scrape + weekly assessor | Required for "no babysitting" | **1.B.3** | This phase ships the callable; 1.B.3 wires the schedule | +| Modal Secrets for `OPENROUTER_API_KEY`, `LANGFUSE_*` | Cloud deploy | **1.B.3** | Same — couples with cron deploy | +| Langfuse callback API mismatch (`host=` kwarg) | Observability | **1.B.3** | Bug #23 from Phase 0; dedicated 1.B.3 work | +| URL dedup for `intake_filter`-rejected JDs | Cost + log growth | **1.B.3** | Natural fit with caching layer | +| Hebbia Greenhouse 404 cleanup in `.env.example` | Coverage | **1.B.3** | Bundle with config restructure for Modal Secrets | +| `approve(thread_id, approved=None)` "request changes" branch | UX | post-1.B | Today binary; if needed, ship as third decision dict key | +| Per-thread auth (anyone with MCP access can approve) | Security | post-1.B | Solo-user system; revisit if hosted | +| Surface `pending_approvals` in the Dataview dashboard | UX polish | **2.B** | Dataview can't read SQLite; needs a tiny exporter | + +--- + +## Critical lessons to carry forward (from Phases 0 + 1.A) + +These bit us before; they will bite us again if we forget them: + +1. **Tests check shape; only adversarial real-data inspection catches data-correctness bugs.** Task 7 is non-optional — do not declare 1.B.1 done from green CI alone. +2. **Module-level `from compass.config import X` freezes the value at import time** and silently breaks the `temp_*_db` fixtures. Always late-bind inside function bodies (e.g. `from compass.config import HITL_CHECKPOINT_DB` *inside* `run_pipeline`). +3. **A graph compiled without a checkpointer silently breaks `interrupt()`.** Never compile `build_graph()` at module level. The only call sites are inside the `async with AsyncSqliteSaver(...)` block in `run_pipeline` and `resume_pending`. +4. **Two-stage review (spec compliance → code quality) catches ~1 issue per task.** Don't skip even on "mechanical" tasks — the CompanyNote tier race (Phase 1.A bug #1) and the Greenhouse `?content=true` flag (Phase 0 bug #1) both looked mechanical and shipped silent data bugs. +5. **JSON-serializability of MCP return shapes.** Phase 1.A bug #11: `date` objects don't serialise through FastMCP. Use `model_dump(mode="json")` or hand-build plain-dict rows (which `state_store._row_to_dict` already does). +6. **Don't trust the tag until you've resumed a real paused thread across a process restart.** Task 7 Step 3 is specifically that check. + +--- + +## Plan Review Loop + +Before executing this plan, dispatch a single plan-document-reviewer subagent with: +- Path to this plan: `docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md` +- Path to spec: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- Phase scope reference: `docs/PHASE_1A_COMPLETE.md` § "How to start Phase 1.B" + +If the reviewer flags issues, fix them in place and re-dispatch the reviewer on the whole plan. Cap at 3 iterations before escalating. + +--- + +## Execution handoff + +Once the plan is approved, two execution options: + +1. **Subagent-Driven (recommended)** — Use `superpowers:subagent-driven-development`. Fresh implementer subagent per Task 1–6, two-stage reviewer (spec compliance → code quality) between tasks. Task 7 PAUSES for human confirmation before live-LLM run. + +2. **Inline Execution** — Use `superpowers:executing-plans`. Single session, batch through Tasks 1–6, hard checkpoint before Task 7. + +Phase 1.A used Subagent-Driven and it worked — keep the pattern. Task 7 always pauses for the human regardless of approach because it costs real money and writes to the real vault. From 216b8eceea9b9d943bbd0ed9f6d2861ba6d8dc99 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:08:23 -0500 Subject: [PATCH 02/46] feat(hitl): aiosqlite-backed pending-approval state store --- compass/hitl/state_store.py | 186 +++++++++++++++++++++++++++++++++ tests/hitl/__init__.py | 0 tests/hitl/conftest.py | 42 ++++++++ tests/hitl/test_state_store.py | 112 ++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 compass/hitl/state_store.py create mode 100644 tests/hitl/__init__.py create mode 100644 tests/hitl/conftest.py create mode 100644 tests/hitl/test_state_store.py diff --git a/compass/hitl/state_store.py b/compass/hitl/state_store.py new file mode 100644 index 0000000..0e652ac --- /dev/null +++ b/compass/hitl/state_store.py @@ -0,0 +1,186 @@ +"""Pending-approval queue for the HiTL flow — aiosqlite, separate from the +LangGraph checkpoint DB. + +Public surface (all coroutines): + add_pending(thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills) + get_pending(thread_id) -> row | None + list_pending() -> list[row] # oldest first + list_timed_out(timeout_hours) -> list[row] # pending AND older than cutoff + mark_resolved(thread_id, status, feedback=None) + +A "row" is a plain dict with the schema documented in the implementation plan. +matched_skills / missing_skills are JSON-encoded in the DB and decoded on read. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +from typing import Any + +import aiosqlite + +logger = logging.getLogger(__name__) + +_VALID_STATUSES = {"pending", "approved", "rejected", "timed_out", "error"} + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, + missing_skills TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +""" + + +def _now() -> _dt.datetime: + """Wall clock as UTC-aware. Indirected so tests can freeze it.""" + return _dt.datetime.now(_dt.UTC) + + +def _db_path(): + """Late-bound HITL_STATE_DB lookup — see module-level discipline rule.""" + import compass.config as cfg + + cfg.HITL_STATE_DB.parent.mkdir(parents=True, exist_ok=True) + return cfg.HITL_STATE_DB + + +async def _connect() -> aiosqlite.Connection: + conn = await aiosqlite.connect(_db_path()) + conn.row_factory = aiosqlite.Row + # WAL mode lets concurrent readers + a single writer coexist without + # exclusive-lock contention. Phase 1.B.3 Modal cron + a human pressing + # `approve` in MCP will race on this file; cheaper to set the pragma now + # than debug a flaky lock-timeout in production. PRAGMA is per-DB and + # persists across opens. + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA busy_timeout=5000") # 5s retry on transient lock + await conn.executescript(_SCHEMA) + await conn.commit() + return conn + + +def _row_to_dict(row: aiosqlite.Row) -> dict[str, Any]: + d = dict(row) + d["matched_skills"] = json.loads(d["matched_skills"]) + d["missing_skills"] = json.loads(d["missing_skills"]) + return d + + +async def add_pending( + *, + thread_id: str, + job_url: str, + company: str, + title: str, + score: float, + score_reasoning: str, + matched_skills: list[str], + missing_skills: list[str], +) -> None: + """Insert a new pending row. INSERT OR IGNORE — re-pausing the same thread_id is a no-op.""" + conn = await _connect() + try: + await conn.execute( + """ + INSERT OR IGNORE INTO pending_approvals + (thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills, created_at, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending') + """, + ( + thread_id, + job_url, + company, + title, + float(score), + score_reasoning, + json.dumps(matched_skills), + json.dumps(missing_skills), + _now().isoformat(), + ), + ) + await conn.commit() + finally: + await conn.close() + + +async def get_pending(thread_id: str) -> dict[str, Any] | None: + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + row = await cur.fetchone() + finally: + await conn.close() + return _row_to_dict(row) if row else None + + +async def list_pending() -> list[dict[str, Any]]: + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE status = 'pending' ORDER BY created_at ASC" + ) as cur: + rows = await cur.fetchall() + finally: + await conn.close() + return [_row_to_dict(r) for r in rows] + + +async def list_timed_out(*, timeout_hours: int) -> list[dict[str, Any]]: + cutoff = (_now() - _dt.timedelta(hours=timeout_hours)).isoformat() + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals " + "WHERE status = 'pending' AND created_at < ? " + "ORDER BY created_at ASC", + (cutoff,), + ) as cur: + rows = await cur.fetchall() + finally: + await conn.close() + return [_row_to_dict(r) for r in rows] + + +async def mark_resolved( + thread_id: str, + *, + status: str, + feedback: str | None = None, +) -> None: + if status not in _VALID_STATUSES or status == "pending": + raise ValueError(f"invalid resolve status: {status!r}") + conn = await _connect() + try: + async with conn.execute( + "SELECT 1 FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + if not await cur.fetchone(): + raise LookupError(f"no pending row for thread_id {thread_id!r}") + await conn.execute( + "UPDATE pending_approvals " + "SET status = ?, feedback = ?, resolved_at = ? " + "WHERE thread_id = ?", + (status, feedback, _now().isoformat(), thread_id), + ) + await conn.commit() + finally: + await conn.close() diff --git a/tests/hitl/__init__.py b/tests/hitl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py new file mode 100644 index 0000000..f099622 --- /dev/null +++ b/tests/hitl/conftest.py @@ -0,0 +1,42 @@ +"""Shared fixtures for HiTL tests.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point HITL_STATE_DB at a fresh per-test SQLite file. + + The store reads `compass.config.HITL_STATE_DB` inside function bodies (per + the module-level discipline rule), so monkeypatching the attribute is + sufficient — no module reimport needed. + """ + db = tmp_path / "pending.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_STATE_DB", db) + return db + + +@pytest.fixture +def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: + """Freeze the wall clock the state store uses.""" + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) + + class _FrozenDT(_dt.datetime): + @classmethod + def now(cls, tz=None): + return fixed if tz is None else fixed.astimezone(tz) + + import compass.hitl.state_store as ss + + monkeypatch.setattr(ss, "_now", lambda: fixed) + return fixed diff --git a/tests/hitl/test_state_store.py b/tests/hitl/test_state_store.py new file mode 100644 index 0000000..35e8d02 --- /dev/null +++ b/tests/hitl/test_state_store.py @@ -0,0 +1,112 @@ +"""state_store CRUD: add/get/list/resolve + status transitions + serialisation.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.hitl import state_store + +pytestmark = pytest.mark.usefixtures("temp_hitl_db") + + +async def _add_one(thread_id: str = "tid-1", **overrides) -> None: + defaults = dict( + thread_id=thread_id, + job_url="https://jobs.example.com/abc", + company="Sierra", + title="Software Engineer, Agent", + score=4.2, + score_reasoning="Strong match on MCP + LangGraph.", + matched_skills=["MCP", "Python"], + missing_skills=["LangGraph"], + ) + defaults.update(overrides) + await state_store.add_pending(**defaults) + + +async def test_add_and_get_round_trips(frozen_now): + await _add_one() + row = await state_store.get_pending("tid-1") + assert row is not None + assert row["thread_id"] == "tid-1" + assert row["company"] == "Sierra" + assert row["score"] == pytest.approx(4.2) + assert row["matched_skills"] == ["MCP", "Python"] + assert row["missing_skills"] == ["LangGraph"] + assert row["status"] == "pending" + assert row["created_at"] == frozen_now.isoformat() + assert row["resolved_at"] is None + assert row["feedback"] is None + + +async def test_add_pending_is_idempotent_on_thread_id(): + """Re-running a pipeline that re-pauses the same thread_id is a no-op, not a crash.""" + await _add_one() + # Second call with same thread_id but different score should be a no-op + # (we don't overwrite — the resume must use the original checkpoint). + await _add_one(score=2.0) + row = await state_store.get_pending("tid-1") + assert row["score"] == pytest.approx(4.2) + + +async def test_list_pending_only_returns_pending_status(): + await _add_one(thread_id="tid-pending") + await _add_one(thread_id="tid-approved") + await state_store.mark_resolved("tid-approved", status="approved") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-pending"] + + +async def test_list_pending_orders_oldest_first(): + """The MCP UI shows the queue oldest-first so things don't get lost.""" + import compass.hitl.state_store as ss + + # First insertion at frozen_now + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) + ss._now = lambda: fixed + await _add_one(thread_id="tid-old") + ss._now = lambda: fixed + _dt.timedelta(minutes=5) + await _add_one(thread_id="tid-new") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-old", "tid-new"] + + +async def test_mark_resolved_records_status_and_timestamp(frozen_now): + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="LGTM") + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + assert row["resolved_at"] == frozen_now.isoformat() + + +async def test_mark_resolved_rejects_unknown_status(): + await _add_one() + with pytest.raises(ValueError, match="status"): + await state_store.mark_resolved("tid-1", status="bogus") + + +async def test_mark_resolved_unknown_thread_raises(): + with pytest.raises(LookupError): + await state_store.mark_resolved("tid-missing", status="approved") + + +async def test_list_timed_out_returns_only_old_pending(frozen_now): + import compass.hitl.state_store as ss + + # Insert one row "5 hours ago", one "1 hour ago" + old = frozen_now - _dt.timedelta(hours=5) + young = frozen_now - _dt.timedelta(hours=1) + ss._now = lambda: old + await _add_one(thread_id="tid-old") + ss._now = lambda: young + await _add_one(thread_id="tid-young") + ss._now = lambda: frozen_now + rows = await state_store.list_timed_out(timeout_hours=4) + assert [r["thread_id"] for r in rows] == ["tid-old"] + + +async def test_get_pending_unknown_returns_none(): + assert await state_store.get_pending("nope") is None From 01dad3877cf8972d1b5a6f844b8741ffd6d83f63 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:13:32 -0500 Subject: [PATCH 03/46] fix(hitl): atomic mark_resolved; observe idempotent re-pause; remove dead frozen class --- compass/hitl/state_store.py | 28 ++++++++++++++++++++-------- tests/hitl/conftest.py | 5 ----- tests/hitl/test_state_store.py | 12 ++++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/compass/hitl/state_store.py b/compass/hitl/state_store.py index 0e652ac..8d641e7 100644 --- a/compass/hitl/state_store.py +++ b/compass/hitl/state_store.py @@ -96,7 +96,7 @@ async def add_pending( """Insert a new pending row. INSERT OR IGNORE — re-pausing the same thread_id is a no-op.""" conn = await _connect() try: - await conn.execute( + cursor = await conn.execute( """ INSERT OR IGNORE INTO pending_approvals (thread_id, job_url, company, title, score, score_reasoning, @@ -116,6 +116,8 @@ async def add_pending( ), ) await conn.commit() + if cursor.rowcount == 0: + logger.info("hitl: re-pause ignored for thread_id=%s (already pending)", thread_id) finally: await conn.close() @@ -168,19 +170,29 @@ async def mark_resolved( ) -> None: if status not in _VALID_STATUSES or status == "pending": raise ValueError(f"invalid resolve status: {status!r}") + conn = await _connect() try: - async with conn.execute( - "SELECT 1 FROM pending_approvals WHERE thread_id = ?", (thread_id,) - ) as cur: - if not await cur.fetchone(): - raise LookupError(f"no pending row for thread_id {thread_id!r}") - await conn.execute( + # Atomic guarded UPDATE: only transitions rows that are still pending. + cursor = await conn.execute( "UPDATE pending_approvals " "SET status = ?, feedback = ?, resolved_at = ? " - "WHERE thread_id = ?", + "WHERE thread_id = ? AND status = 'pending'", (status, feedback, _now().isoformat(), thread_id), ) await conn.commit() + + if cursor.rowcount == 0: + # Distinguish "no such thread" from "already resolved" for the caller. + async with conn.execute( + "SELECT status FROM pending_approvals WHERE thread_id = ?", + (thread_id,), + ) as cur: + existing = await cur.fetchone() + if existing is None: + raise LookupError(f"no pending row for thread_id {thread_id!r}") + raise ValueError(f"thread {thread_id!r} already resolved ({existing['status']!r})") finally: await conn.close() + + logger.info("hitl: thread %s resolved as %s", thread_id, status) diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py index f099622..d0dd141 100644 --- a/tests/hitl/conftest.py +++ b/tests/hitl/conftest.py @@ -31,11 +31,6 @@ def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: """Freeze the wall clock the state store uses.""" fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) - class _FrozenDT(_dt.datetime): - @classmethod - def now(cls, tz=None): - return fixed if tz is None else fixed.astimezone(tz) - import compass.hitl.state_store as ss monkeypatch.setattr(ss, "_now", lambda: fixed) diff --git a/tests/hitl/test_state_store.py b/tests/hitl/test_state_store.py index 35e8d02..b7dc47a 100644 --- a/tests/hitl/test_state_store.py +++ b/tests/hitl/test_state_store.py @@ -110,3 +110,15 @@ async def test_list_timed_out_returns_only_old_pending(frozen_now): async def test_get_pending_unknown_returns_none(): assert await state_store.get_pending("nope") is None + + +async def test_mark_resolved_rejects_already_resolved_row(): + """Concurrent resolvers race: second one must see ValueError, not silent overwrite.""" + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="first") + with pytest.raises(ValueError, match="already resolved"): + await state_store.mark_resolved("tid-1", status="timed_out") + # Confirm the first resolution stuck — no silent overwrite + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "first" From a11f3d3e29632f519c3772730bafece22c9b65ef Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:18:50 -0500 Subject: [PATCH 04/46] feat(hitl): replace auto-approve with LangGraph interrupt() --- compass/mcp_server/server.py | 1 + compass/pipeline/graph.py | 2 + compass/pipeline/nodes/hitl.py | 55 ++++++++---- compass/pipeline/state.py | 2 + tests/pipeline/test_graph_integration.py | 18 +++- tests/pipeline/test_hitl_node_interrupt.py | 100 +++++++++++++++++++++ tests/pipeline/test_routing.py | 26 +----- 7 files changed, 158 insertions(+), 46 deletions(-) create mode 100644 tests/pipeline/test_hitl_node_interrupt.py diff --git a/compass/mcp_server/server.py b/compass/mcp_server/server.py index 90674b0..1760130 100644 --- a/compass/mcp_server/server.py +++ b/compass/mcp_server/server.py @@ -90,6 +90,7 @@ async def score_jd(jd_text: str) -> dict: "jobs_processed": 0, "jobs_written": 0, "errors": [], + "thread_id": None, } extract_result = await extract_node(state) if extract_result.get("errors"): diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index 7d6dd91..b0f5b33 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -130,6 +130,7 @@ def _initial_state(job: RawJob) -> CompassState: "jobs_processed": 0, "jobs_written": 0, "errors": [], + "thread_id": None, } @@ -200,6 +201,7 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: "errors": [e for r in results for e in r.get("errors", [])], "in_scope": None, "role_family": None, + "thread_id": None, } if aggregate["jobs_written"] > 0: diff --git a/compass/pipeline/nodes/hitl.py b/compass/pipeline/nodes/hitl.py index 3fc29c4..30ae342 100644 --- a/compass/pipeline/nodes/hitl.py +++ b/compass/pipeline/nodes/hitl.py @@ -1,10 +1,10 @@ -""" -hitl_node — Phase 0.B auto-approve based on SCORE_THRESHOLD. +"""hitl_node — Phase 1.B.1 real human-in-the-loop via LangGraph interrupt(). -This is INTENTIONALLY auto-approve for 0.B. Real LangGraph `interrupt()` + -`AsyncSqliteSaver` checkpointing + an MCP-tool-driven approval surface ship in -Phase 1.B per the spec. Auto-approve unblocks the end-to-end pipeline so we -can collect real eval data first. +Behaviour: + * score < SCORE_THRESHOLD or score missing -> auto-reject, no interrupt fires + * score >= SCORE_THRESHOLD -> interrupt() with the approval payload; the + orchestrator catches the interrupt and registers the thread in + compass.hitl.state_store. A human resumes via Command(resume={...}). """ from __future__ import annotations @@ -12,6 +12,8 @@ import logging from typing import TYPE_CHECKING +from langgraph.types import interrupt + from compass.config import SCORE_THRESHOLD if TYPE_CHECKING: @@ -21,20 +23,37 @@ async def hitl_node(state: CompassState) -> dict: - """Auto-approve if score >= SCORE_THRESHOLD, else reject. No human interaction in 0.B.""" score = state.get("score_result") - if score is None: - logger.info("hitl: no score_result, rejecting by default") + job = state.get("current_job") + + if score is None or score.score < SCORE_THRESHOLD: + logger.info( + "hitl: auto-reject %s (score=%s, threshold=%.2f)", + job.url if job else "(unknown)", + getattr(score, "score", None), + SCORE_THRESHOLD, + ) return {"human_approved": False} - approved = score.score >= SCORE_THRESHOLD - job = state.get("current_job") - job_id = job.url if job else "(unknown)" + payload = { + "kind": "approval_request", + "job_url": job.url if job else "", + "company": job.company if job else "", + "title": job.title if job else "", + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": list(score.matched_skills), + "missing_skills": list(score.missing_skills), + } logger.info( - "hitl: auto-%s job %s (score=%.2f, threshold=%.2f)", - "approve" if approved else "reject", - job_id, - score.score, - SCORE_THRESHOLD, + "hitl: interrupting for approval — %s (score=%.2f)", payload["job_url"], score.score ) - return {"human_approved": approved} + decision = interrupt(payload) + + if not isinstance(decision, dict): + logger.warning("hitl: malformed resume value %r — defaulting to reject", decision) + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } diff --git a/compass/pipeline/state.py b/compass/pipeline/state.py index 5053d65..e71978c 100644 --- a/compass/pipeline/state.py +++ b/compass/pipeline/state.py @@ -72,3 +72,5 @@ class CompassState(TypedDict): jobs_written: int errors: list[str] + + thread_id: str | None diff --git a/tests/pipeline/test_graph_integration.py b/tests/pipeline/test_graph_integration.py index 5188bd8..aa7533b 100644 --- a/tests/pipeline/test_graph_integration.py +++ b/tests/pipeline/test_graph_integration.py @@ -40,7 +40,19 @@ async def fake_tailor(*args, **kwargs): monkeypatch.setattr(tailor, "_tailor", fake_tailor) -async def test_run_pipeline_end_to_end(temp_vault, mocked_llms): +@pytest.fixture +def auto_approve_hitl(monkeypatch): + """Stub the `interrupt()` call in hitl_node so the integration tests can + exercise the full extract -> score -> hitl -> tailor -> vault_write path + without needing a real human resume.""" + + def fake_interrupt(_payload): + return {"approved": True, "feedback": None} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + + +async def test_run_pipeline_end_to_end(temp_vault, mocked_llms, auto_approve_hitl): """Run a single fake job through the full graph; verify vault state.""" from compass.pipeline.graph import run_pipeline @@ -100,7 +112,7 @@ async def test_run_pipeline_skips_dedup_urls(temp_vault, mocked_llms): assert state["jobs_written"] == 0 -async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms): +async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms, auto_approve_hitl): """After processing, master-gap-plan.md should be regenerated.""" from compass.pipeline.graph import run_pipeline @@ -177,7 +189,7 @@ async def fake_tailor(*args, **kwargs): assert tailor_calls["count"] == 0 -async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms): +async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms, auto_approve_hitl): """Every run_pipeline invocation appends a row to _meta/pipeline-runs.md.""" from compass.pipeline.graph import run_pipeline diff --git a/tests/pipeline/test_hitl_node_interrupt.py b/tests/pipeline/test_hitl_node_interrupt.py new file mode 100644 index 0000000..b6d0506 --- /dev/null +++ b/tests/pipeline/test_hitl_node_interrupt.py @@ -0,0 +1,100 @@ +"""hitl_node calls interrupt() above threshold; auto-rejects below threshold.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.pipeline.nodes.hitl import hitl_node +from compass.pipeline.state import CompassState, JobScore, RawJob + + +def _state(score: float | None) -> CompassState: + job = RawJob( + company="Sierra", + title="SWE, Agent", + url="https://jobs.example.com/sierra-1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + sr = ( + None + if score is None + else JobScore( + score=score, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="", + ) + ) + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": sr, + "in_scope": True, + "role_family": "agent-engineer", + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": "tid-test", + } + + +async def test_below_threshold_auto_rejects_without_interrupt(monkeypatch): + """Below SCORE_THRESHOLD short-circuits — interrupt MUST NOT fire (no human prompt cost).""" + called = {"interrupt": 0} + + def boom(_payload): + called["interrupt"] += 1 + raise AssertionError("interrupt should not have been called") + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", boom) + result = await hitl_node(_state(score=2.0)) + assert result == {"human_approved": False} + assert called["interrupt"] == 0 + + +async def test_missing_score_auto_rejects(): + result = await hitl_node(_state(score=None)) + assert result == {"human_approved": False} + + +async def test_above_threshold_calls_interrupt_with_payload(monkeypatch): + captured = {} + + def fake_interrupt(payload): + captured.update(payload) + # Simulate resume value (this is what Command(resume=...) sends back) + return {"approved": True, "feedback": "Strong fit"} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + result = await hitl_node(_state(score=4.2)) + assert captured["kind"] == "approval_request" + assert captured["company"] == "Sierra" + assert captured["score"] == pytest.approx(4.2) + assert captured["matched_skills"] == ["MCP"] + assert result == {"human_approved": True, "human_feedback": "Strong fit"} + + +async def test_resume_rejection_propagates(monkeypatch): + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda _p: {"approved": False, "feedback": "Not a fit"}, + ) + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False, "human_feedback": "Not a fit"} + + +async def test_malformed_resume_value_defaults_to_rejected(monkeypatch): + """If Command(resume=...) somehow sends a non-dict, treat as reject not crash.""" + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", lambda _p: "bogus") + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False} diff --git a/tests/pipeline/test_routing.py b/tests/pipeline/test_routing.py index 9aa5349..5a87e34 100644 --- a/tests/pipeline/test_routing.py +++ b/tests/pipeline/test_routing.py @@ -1,8 +1,7 @@ -"""Tests for reflect_node + hitl_node — control-flow nodes (no LLM in 0.B).""" +"""Tests for reflect_node — control-flow nodes (no LLM).""" from datetime import date -from compass.config import SCORE_THRESHOLD from compass.pipeline.state import CompassState, JobScore, RawJob @@ -43,29 +42,6 @@ async def test_reflect_node_is_passthrough(): assert result == {} -async def test_hitl_node_approves_when_score_meets_threshold(): - from compass.pipeline.nodes.hitl import hitl_node - - result = await hitl_node(_state(SCORE_THRESHOLD)) - assert result["human_approved"] is True - - -async def test_hitl_node_rejects_when_score_below_threshold(): - from compass.pipeline.nodes.hitl import hitl_node - - result = await hitl_node(_state(SCORE_THRESHOLD - 0.5)) - assert result["human_approved"] is False - - -async def test_hitl_node_handles_missing_score(): - from compass.pipeline.nodes.hitl import hitl_node - - state = _state(0.0) - state["score_result"] = None - result = await hitl_node(state) - assert result["human_approved"] is False - - class TestIntakeFilterRouting: def test_out_of_scope_yields_in_scope_false_and_no_write(self, temp_vault): """An out-of-scope job must short-circuit to END without invoking From 8ba413d98e83f944db1cc39e9e5d73025cf73c1f Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:23:29 -0500 Subject: [PATCH 05/46] feat(hitl): record hitl_decision + hitl_at on JobNote --- compass/pipeline/nodes/vault_write.py | 29 +++++++- compass/vault/schemas.py | 3 +- tests/pipeline/test_vault_write.py | 96 ++++++++++++++++++++++----- 3 files changed, 111 insertions(+), 17 deletions(-) diff --git a/compass/pipeline/nodes/vault_write.py b/compass/pipeline/nodes/vault_write.py index 71d0b48..fb93350 100644 --- a/compass/pipeline/nodes/vault_write.py +++ b/compass/pipeline/nodes/vault_write.py @@ -21,7 +21,7 @@ from __future__ import annotations import logging -from datetime import date +from datetime import date, datetime from typing import TYPE_CHECKING from compass.vault.schemas import CompanyNote, JobNote @@ -33,6 +33,30 @@ logger = logging.getLogger(__name__) +def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | None]: + """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" + from compass.config import SCORE_THRESHOLD + + approved = state.get("human_approved") + if approved is None: + # hitl never reached (e.g. extract/score errored). Leave fields null. + return (None, None) + + feedback = (state.get("human_feedback") or "").lower() + score = state.get("score_result") + score_value = score.score if score is not None else 0.0 + + if approved is True: + decision = "approved" + elif feedback.startswith("auto-cancelled after"): + decision = "timed_out" + elif score_value < SCORE_THRESHOLD: + decision = "auto_rejected" + else: + decision = "rejected" + return (decision, datetime.now()) + + async def vault_write_node(state: CompassState) -> dict: job = state.get("current_job") score = state.get("score_result") @@ -86,6 +110,7 @@ async def vault_write_node(state: CompassState) -> dict: company_tier_for_write = "unknown" break + hitl_decision, hitl_at = _derive_hitl_decision(state) note = JobNote( company=job.company, title=job.title, @@ -108,6 +133,8 @@ async def vault_write_node(state: CompassState) -> dict: skills_missing=score.missing_skills, jd_summary=req.summary, tailored_paragraph=state.get("tailored_paragraph"), + hitl_decision=hitl_decision, + hitl_at=hitl_at, ) write_job_note(note, full_description=job.description) diff --git a/compass/vault/schemas.py b/compass/vault/schemas.py index 09035f8..7bbffdc 100644 --- a/compass/vault/schemas.py +++ b/compass/vault/schemas.py @@ -33,6 +33,7 @@ "fine-tuning", ] Source = Literal["greenhouse", "lever", "ashby", "jobspy", "smoke", "manual"] +HitlDecision = Literal["approved", "rejected", "auto_rejected", "timed_out"] class JobNote(BaseModel): @@ -59,7 +60,7 @@ class JobNote(BaseModel): skills_missing: list[str] = [] jd_summary: str = "" tailored_paragraph: str | None = None - hitl_decision: str | None = None + hitl_decision: HitlDecision | None = None hitl_at: datetime | None = None applied_at: datetime | None = None diff --git a/tests/pipeline/test_vault_write.py b/tests/pipeline/test_vault_write.py index 7d097ef..dcb6995 100644 --- a/tests/pipeline/test_vault_write.py +++ b/tests/pipeline/test_vault_write.py @@ -7,9 +7,16 @@ from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob -def _state( - skills_required: list[str], skills_matched: list[str], score: float = 4.2 +def _build_state_for_score( + *, + score: float = 4.2, + skills_required: list[str] | None = None, + skills_matched: list[str] | None = None, + human_approved: bool = True, + human_feedback: str | None = None, ) -> CompassState: + skills_required = skills_required if skills_required is not None else ["MCP", "LangGraph"] + skills_matched = skills_matched if skills_matched is not None else ["MCP"] return { "raw_jobs": [], "current_job": RawJob( @@ -38,8 +45,8 @@ def _state( ), "in_scope": True, "role_family": "agent-engineer", - "human_approved": True, - "human_feedback": None, + "human_approved": human_approved, + "human_feedback": human_feedback, "tailored_paragraph": None, "vault_written": False, "jobs_processed": 0, @@ -51,7 +58,9 @@ def _state( async def test_vault_write_node_writes_jobnote(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - result = await vault_write_node(_state(["MCP", "LangGraph"], ["MCP"])) + result = await vault_write_node( + _build_state_for_score(skills_required=["MCP", "LangGraph"], skills_matched=["MCP"]) + ) assert result["vault_written"] is True assert result["jobs_written"] == 1 job_files = list((temp_vault / "jobs").glob("*.md")) @@ -71,7 +80,9 @@ async def test_vault_write_node_records_skills_on_jobnote(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - await vault_write_node(_state(["MCP", "LangGraph"], ["MCP"])) + await vault_write_node( + _build_state_for_score(skills_required=["MCP", "LangGraph"], skills_matched=["MCP"]) + ) job_files = list((temp_vault / "jobs").glob("*.md")) assert len(job_files) == 1 loaded = frontmatter.load(job_files[0]) @@ -82,7 +93,7 @@ async def test_vault_write_node_records_skills_on_jobnote(temp_vault): async def test_vault_write_node_writes_company_note(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - await vault_write_node(_state(["MCP"], ["MCP"])) + await vault_write_node(_build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"])) company_path = temp_vault / "companies" / "Sierra.md" assert company_path.exists() @@ -90,7 +101,7 @@ async def test_vault_write_node_writes_company_note(temp_vault): async def test_vault_write_node_handles_missing_state(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"]) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"]) state["score_result"] = None result = await vault_write_node(state) assert result["vault_written"] is False @@ -101,7 +112,7 @@ async def test_vault_write_node_persists_tailored_paragraph(temp_vault): """When state has tailored_paragraph, it lands on the JobNote.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"]) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"]) state["tailored_paragraph"] = "Lead with your Cisco MCP server work." await vault_write_node(state) job_files = list((temp_vault / "jobs").glob("*.md")) @@ -114,7 +125,7 @@ async def test_vault_write_node_persists_full_jd_body(temp_vault): raw JD is preserved in the JobNote body, not just the LLM summary.""" import compass.pipeline.nodes.vault_write as vw - state = _state(["MCP"], ["MCP"], score=4.2) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.2) state["current_job"] = state["current_job"].model_copy( update={"description": "VERY_DISTINCTIVE_RAW_JD_MARKER agentic engineering."} ) @@ -129,7 +140,7 @@ async def test_low_score_in_scope_still_writes(temp_vault): regardless of match_score. Pre-1.A this returned vault_written=False.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["Python"], ["Python"], score=1.5) + state = _build_state_for_score(skills_required=["Python"], skills_matched=["Python"], score=1.5) state["in_scope"] = True state["role_family"] = "swe-backend" result = await vault_write_node(state) @@ -142,7 +153,9 @@ async def test_role_family_threaded_to_jobnote(temp_vault): """role_family from state lands in JobNote frontmatter.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["LangGraph"], ["LangGraph"], score=4.0) + state = _build_state_for_score( + skills_required=["LangGraph"], skills_matched=["LangGraph"], score=4.0 + ) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -162,7 +175,7 @@ async def test_tier_resolved_from_target_companies(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -179,7 +192,7 @@ async def test_unknown_company_tier_remains_unknown(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" state["current_job"] = state["current_job"].model_copy(update={"company": "RandomCo"}) @@ -207,7 +220,7 @@ async def test_human_edited_company_tier_preserved(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -217,3 +230,56 @@ async def test_human_edited_company_tier_preserved(temp_vault): job_path = next((temp_vault / "jobs").glob("*Sierra*.md")) assert frontmatter.load(job_path).metadata["tier"] == "apply-now" + + +async def test_vault_write_records_approved_decision(temp_vault): + """state['human_approved'] = True -> hitl_decision='approved', hitl_at set.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.5, + human_approved=True, + human_feedback="LGTM", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "approved" + assert "hitl_at" in md and md["hitl_at"] is not None + + +async def test_vault_write_records_auto_rejected_for_low_score(temp_vault): + """Below-threshold path: hitl never interrupts; decision is 'auto_rejected'.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=2.0, human_approved=False, human_feedback=None) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "auto_rejected" + + +async def test_vault_write_records_rejected_when_human_said_no(temp_vault): + """Above-threshold path with human_approved=False = explicit reject.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=4.2, human_approved=False, human_feedback="not a fit") + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "rejected" + + +async def test_vault_write_records_timed_out(temp_vault): + """Timeout-checker resume sets feedback='auto-cancelled after Xh timeout'.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.2, + human_approved=False, + human_feedback="auto-cancelled after 4h timeout", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "timed_out" From 1c7d55605c7b2f9f6c09adb7f9a9e6b20037748e Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:30:50 -0500 Subject: [PATCH 06/46] feat(hitl): mount AsyncSqliteSaver in run_pipeline; register paused threads --- .env.example | 1 + compass/config.py | 3 + compass/pipeline/graph.py | 122 +++++++++++++++--- tests/conftest.py | 15 +++ tests/hitl/conftest.py | 25 +--- tests/pipeline/test_graph_checkpointing.py | 138 +++++++++++++++++++++ 6 files changed, 264 insertions(+), 40 deletions(-) create mode 100644 tests/pipeline/test_graph_checkpointing.py diff --git a/.env.example b/.env.example index 7295449..791dba3 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,7 @@ EMBEDDING_MODEL=all-MiniLM-L6-v2 # ── HiTL ───────────────────────────────────────────────────────────────────── HITL_STATE_DB=~/.compass/hitl.db +HITL_CHECKPOINT_DB=~/.compass/checkpoints.db HITL_TIMEOUT_HOURS=4 # ── Pipeline ───────────────────────────────────────────────────────────────── diff --git a/compass/config.py b/compass/config.py index 2e99940..f6654d6 100644 --- a/compass/config.py +++ b/compass/config.py @@ -44,6 +44,9 @@ # ── HiTL ────────────────────────────────────────────────────────────────────── HITL_STATE_DB: Path = Path(os.getenv("HITL_STATE_DB", "~/.compass/hitl.db")).expanduser() +HITL_CHECKPOINT_DB: Path = Path( + os.getenv("HITL_CHECKPOINT_DB", "~/.compass/checkpoints.db") +).expanduser() HITL_TIMEOUT_HOURS: int = int(os.getenv("HITL_TIMEOUT_HOURS", "4")) # ── Pipeline ────────────────────────────────────────────────────────────────── diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index b0f5b33..cd845ce 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -17,15 +17,18 @@ from __future__ import annotations import asyncio +import hashlib import logging import time from datetime import datetime import frontmatter +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from langgraph.graph import END, START, StateGraph from compass.analysis import gap_aggregator from compass.config import MAX_CONCURRENT_JOBS, VAULT_PATH +from compass.hitl import state_store from compass.pipeline.nodes.extract import extract_node from compass.pipeline.nodes.hitl import hitl_node from compass.pipeline.nodes.intake import intake_node @@ -61,7 +64,7 @@ def _route_after_hitl(state: CompassState) -> str: return "vault_write" -def build_graph(): +def build_graph(checkpointer=None): builder = StateGraph(CompassState) builder.add_node("intake", intake_node) builder.add_node("intake_filter", intake_filter_node) @@ -93,7 +96,7 @@ def build_graph(): builder.add_edge("tailor", "vault_write") builder.add_edge("vault_write", END) - return builder.compile() + return builder.compile(checkpointer=checkpointer) def _vault_url_set() -> set[str]: @@ -115,7 +118,13 @@ def _vault_url_set() -> set[str]: return urls -def _initial_state(job: RawJob) -> CompassState: +def _thread_id_for(job_url: str, batch_started_at: datetime) -> str: + """Deterministic 16-char SHA-1 of (url, batch start) — same batch + same job = same thread.""" + raw = f"{job_url}|{batch_started_at.isoformat()}" + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] + + +def _initial_state(job: RawJob, thread_id: str | None = None) -> CompassState: return { "raw_jobs": [], "current_job": job, @@ -130,7 +139,7 @@ def _initial_state(job: RawJob) -> CompassState: "jobs_processed": 0, "jobs_written": 0, "errors": [], - "thread_id": None, + "thread_id": thread_id, } @@ -159,19 +168,72 @@ def _langfuse_config() -> dict: return {} -async def _process_one(graph, job: RawJob, sem: asyncio.Semaphore) -> CompassState: +async def _process_one( + graph, + job: RawJob, + sem: asyncio.Semaphore, + thread_id: str, +) -> tuple[CompassState, bool]: + """Invoke the graph for one job. Returns (final_state, was_paused).""" + config = { + "configurable": {"thread_id": thread_id}, + **_langfuse_config(), + } async with sem: try: - return await graph.ainvoke(_initial_state(job), config=_langfuse_config()) + result = await graph.ainvoke(_initial_state(job, thread_id=thread_id), config=config) except Exception as e: - # logger.exception preserves the traceback to stderr; the string version - # below is still aggregated into state for the run summary. logger.exception("pipeline: graph crashed on %s", job.url) - return {**_initial_state(job), "errors": [f"graph: {type(e).__name__}: {e}"]} + return ( + { + **_initial_state(job, thread_id=thread_id), + "errors": [f"graph: {type(e).__name__}: {e}"], + }, + False, + ) + + # When interrupt() fires, ainvoke returns with state containing the + # __interrupt__ marker AND without progressing past the hitl node. + # vault_written stays False, jobs_written stays 0 — that's our signal. + interrupts = result.get("__interrupt__") + if interrupts: + payload = interrupts[0].value if hasattr(interrupts[0], "value") else interrupts[0] + if isinstance(payload, dict) and payload.get("kind") == "approval_request": + await state_store.add_pending( + thread_id=thread_id, + job_url=payload["job_url"], + company=payload["company"], + title=payload["title"], + score=float(payload["score"]), + score_reasoning=payload["score_reasoning"], + matched_skills=list(payload["matched_skills"]), + missing_skills=list(payload["missing_skills"]), + ) + logger.info( + "pipeline: paused %s for approval (thread_id=%s, score=%.2f)", + job.url, + thread_id, + payload["score"], + ) + return (result, True) + # Unknown interrupt shape — DO NOT silently swallow. Phase 0 bug pattern: + # a future interrupt() added elsewhere in the graph would otherwise + # disappear into a "succeeded with jobs_paused=0" black hole. Log loudly + # and still count as paused so the caller's bookkeeping reflects reality. + logger.error( + "pipeline: graph paused at UNKNOWN interrupt kind for %s — payload=%r", + job.url, + payload, + ) + return (result, True) + return (result, False) async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: - """Scrape (or accept) jobs, dedup, run per-job graph, regenerate gap plan.""" + """Scrape (or accept) jobs, dedup, run per-job graph under a single + AsyncSqliteSaver, regenerate gap plan.""" + from compass.config import HITL_CHECKPOINT_DB + start_monotonic = time.monotonic() start_wall = datetime.now() # captured alongside monotonic; immune to NTP/DST mid-run if raw_jobs is None: @@ -183,9 +245,24 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: if dropped: logger.info("pipeline: dropping %d/%d jobs already in vault", dropped, len(raw_jobs)) - graph = build_graph() - sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) - results = await asyncio.gather(*[_process_one(graph, j, sem) for j in fresh]) + HITL_CHECKPOINT_DB.parent.mkdir(parents=True, exist_ok=True) + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + # Enable WAL on the checkpoint DB once per process. Cheap if already set. + # See state_store._connect for the rationale. + try: + await checkpointer.conn.execute("PRAGMA journal_mode=WAL") + await checkpointer.conn.execute("PRAGMA busy_timeout=5000") + except Exception: + logger.debug("checkpoint: WAL pragma set already or unsupported; continuing") + graph = build_graph(checkpointer=checkpointer) + sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) + coros = [ + _process_one(graph, j, sem, thread_id=_thread_id_for(j.url, start_wall)) for j in fresh + ] + results = await asyncio.gather(*coros) + + paused_count = sum(int(p) for _, p in results) + final_states = [s for s, _ in results] aggregate: CompassState = { "raw_jobs": raw_jobs, @@ -195,14 +272,16 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: "human_approved": None, "human_feedback": None, "tailored_paragraph": None, - "vault_written": any(r.get("vault_written") for r in results), + "vault_written": any(r.get("vault_written") for r in final_states), "jobs_processed": len(fresh), - "jobs_written": sum(int(bool(r.get("vault_written"))) for r in results), - "errors": [e for r in results for e in r.get("errors", [])], + "jobs_written": sum(int(bool(r.get("vault_written"))) for r in final_states), + "errors": [e for r in final_states for e in r.get("errors", [])], "in_scope": None, "role_family": None, "thread_id": None, } + # `jobs_paused` is informational only — not in CompassState TypedDict. + aggregate["jobs_paused"] = paused_count # type: ignore[typeddict-unknown-key] if aggregate["jobs_written"] > 0: gap_aggregator.regenerate(write=True) @@ -211,9 +290,10 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: _append_run_log(aggregate, duration_s) unknown_count = _count_unknown_skills_seen_this_run(start_wall) logger.info( - "pipeline: processed=%d written=%d errors=%d unknown_skills_seen=%d duration=%.1fs", + "pipeline: processed=%d written=%d paused=%d errors=%d unknown_skills_seen=%d duration=%.1fs", aggregate["jobs_processed"], aggregate["jobs_written"], + paused_count, len(aggregate["errors"]), unknown_count, duration_s, @@ -228,14 +308,15 @@ def _append_run_log(state: CompassState, duration_s: float) -> None: if not log_path.exists(): log_path.write_text( "# Pipeline Run Log\n\n" - "| Timestamp | Processed | Written | Errors | Duration |\n" - "|---|---|---|---|---|\n", + "| Timestamp | Processed | Written | Paused | Errors | Duration |\n" + "|---|---|---|---|---|---|\n", encoding="utf-8", ) ts = datetime.now().isoformat(timespec="seconds") + paused = state.get("jobs_paused", 0) # type: ignore[typeddict-item] row = ( f"| {ts} | {state['jobs_processed']} | {state['jobs_written']} | " - f"{len(state['errors'])} | {duration_s:.1f}s |\n" + f"{paused} | {len(state['errors'])} | {duration_s:.1f}s |\n" ) with log_path.open("a", encoding="utf-8") as f: f.write(row) @@ -294,5 +375,6 @@ async def _scrape_all() -> list[RawJob]: print( f"Processed: {result['jobs_processed']} | " f"Written: {result['jobs_written']} | " + f"Paused: {result.get('jobs_paused', 0)} | " f"Errors: {len(result['errors'])}" ) diff --git a/tests/conftest.py b/tests/conftest.py index 8f67a76..e7f2d49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,21 @@ import pytest +@pytest.fixture +def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point HITL_STATE_DB at a fresh per-test SQLite file. + + The store reads `compass.config.HITL_STATE_DB` inside function bodies (per + the module-level discipline rule), so monkeypatching the attribute is + sufficient — no module reimport needed. + """ + db = tmp_path / "pending.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_STATE_DB", db) + return db + + @pytest.fixture def temp_vault(tmp_path: Path, monkeypatch): """Create a minimal compass-vault structure in a tmp dir and point config at it.""" diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py index d0dd141..8870b79 100644 --- a/tests/hitl/conftest.py +++ b/tests/hitl/conftest.py @@ -1,30 +1,15 @@ -"""Shared fixtures for HiTL tests.""" +"""Shared fixtures for HiTL tests. + +`temp_hitl_db` lives in the top-level tests/conftest.py so pipeline tests can +reuse it without duplication. +""" from __future__ import annotations import datetime as _dt -from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: - from pathlib import Path - - -@pytest.fixture -def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Point HITL_STATE_DB at a fresh per-test SQLite file. - - The store reads `compass.config.HITL_STATE_DB` inside function bodies (per - the module-level discipline rule), so monkeypatching the attribute is - sufficient — no module reimport needed. - """ - db = tmp_path / "pending.db" - import compass.config as cfg - - monkeypatch.setattr(cfg, "HITL_STATE_DB", db) - return db - @pytest.fixture def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: diff --git a/tests/pipeline/test_graph_checkpointing.py b/tests/pipeline/test_graph_checkpointing.py new file mode 100644 index 0000000..f69eba3 --- /dev/null +++ b/tests/pipeline/test_graph_checkpointing.py @@ -0,0 +1,138 @@ +"""End-to-end: graph pauses at hitl, state_store gets the row, +resume_pending finishes the run. + +These tests STUB the LLM-touching nodes (extract, score, tailor) — we only +exercise the graph machinery + interrupt + checkpointer + state_store +interaction. Real LLM calls live in the live-smoke check in Task 7.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + """Replace extract / score / tailor / vault_write / intake_filter with deterministic stubs.""" + + async def fake_extract(state: CompassState) -> dict: + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP", "Python"], + nice_to_have_skills=["LangGraph"], + seniority="mid", + remote_policy="remote", + summary="An agent role.", + ) + } + + async def fake_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=4.2, + reasoning="Strong match.", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="Lead with MCP.", + ) + } + + async def fake_tailor(state: CompassState) -> dict: + return {"tailored_paragraph": "Tailored: lead with MCP work."} + + async def fake_vault_write(state: CompassState) -> dict: + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(state: CompassState) -> dict: + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +def _job(url: str = "https://jobs.example.com/sierra-1") -> RawJob: + return RawJob( + company="Sierra", + title="SWE, Agent", + url=url, + source="ashby", + description="An agent engineering role at Sierra.", + date_posted=_dt.date(2026, 5, 18), + ) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_above_threshold_job_pauses_and_registers_in_state_store(monkeypatch): + """run_pipeline with a single above-threshold job: paused=1, written=0, + state_store has a row.""" + from compass.pipeline.graph import run_pipeline + + # No real scraping — pass the job in directly. + result = await run_pipeline(raw_jobs=[_job()]) + + assert result["jobs_processed"] == 1 + assert result["jobs_written"] == 0 # paused before vault_write + assert result["jobs_paused"] == 1 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["company"] == "Sierra" + assert rows[0]["score"] == pytest.approx(4.2) + assert rows[0]["job_url"] == "https://jobs.example.com/sierra-1" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_below_threshold_job_runs_to_completion_no_state_store_row(monkeypatch): + """Below-threshold path is unchanged from 1.A — no interrupt, vault_write fires.""" + + async def low_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=2.0, + reasoning="Mismatched.", + matched_skills=[], + missing_skills=["A", "B"], + tailoring_notes="", + ) + } + + monkeypatch.setattr("compass.pipeline.graph.score_node", low_score) + from compass.pipeline.graph import run_pipeline + + result = await run_pipeline(raw_jobs=[_job()]) + assert result["jobs_written"] == 1 + assert result["jobs_paused"] == 0 + assert await state_store.list_pending() == [] + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_thread_id_is_deterministic_for_same_url_and_batch(): + """Re-running the SAME batch (same start_wall) reuses the same thread_id — + second run hits the idempotent INSERT OR IGNORE path.""" + from compass.pipeline.graph import _thread_id_for + + tid_a = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_b = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_c = _thread_id_for("https://jobs.example.com/y", _dt.datetime(2026, 5, 19, 9, 0, 0)) + assert tid_a == tid_b + assert tid_a != tid_c + assert len(tid_a) == 16 From fb484018ba551d1d088db18355b7c7ccff1cf168 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:36:14 -0500 Subject: [PATCH 07/46] feat(hitl): resume_pending entrypoint via Command(resume=...) --- compass/hitl/resume.py | 60 +++++++++++++++ tests/hitl/test_resume.py | 155 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 compass/hitl/resume.py create mode 100644 tests/hitl/test_resume.py diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py new file mode 100644 index 0000000..c94cea7 --- /dev/null +++ b/compass/hitl/resume.py @@ -0,0 +1,60 @@ +"""Resume a paused LangGraph thread by re-opening the checkpointer and +recompiling the graph. Single source of truth for resumes — the timeout +checker and MCP `approve` tool both go through here.""" + +from __future__ import annotations + +import logging +from typing import Any + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver +from langgraph.types import Command + +from compass.hitl import state_store + +logger = logging.getLogger(__name__) + + +async def resume_pending( + thread_id: str, + *, + decision: dict[str, Any], + status_override: str | None = None, +) -> dict: + """Drive a paused thread to completion. + + decision: passed as `Command(resume=decision)`. Expected shape + {"approved": bool, "feedback": str | None}. + status_override: optional explicit status to write to the state store + (timeout_checker uses "timed_out"). If None, derived from + decision["approved"]: True -> "approved", False -> "rejected". + Raises LookupError if the thread isn't in the state store. + Raises ValueError if the thread is already resolved. + """ + from compass.config import HITL_CHECKPOINT_DB + + row = await state_store.get_pending(thread_id) + if row is None: + raise LookupError(f"no pending thread {thread_id!r}") + if row["status"] != "pending": + raise ValueError(f"thread {thread_id!r} already resolved ({row['status']!r})") + + # Late import to avoid potential circular import: graph.py imports state_store + from compass.pipeline.graph import build_graph + + config = {"configurable": {"thread_id": thread_id}} + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + resolved_status = "approved" if decision.get("approved") else "rejected" + await state_store.mark_resolved( + thread_id, + status=resolved_status, + feedback=decision.get("feedback"), + ) + logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) + return final diff --git a/tests/hitl/test_resume.py b/tests/hitl/test_resume.py new file mode 100644 index 0000000..92907eb --- /dev/null +++ b/tests/hitl/test_resume.py @@ -0,0 +1,155 @@ +"""resume_pending re-opens the checkpointer, recompiles the graph, and drives +the resume via Command(resume=...). Verifies vault_write fires on approve +and tailor is skipped on reject.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="agent role", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ) + } + + async def fake_tailor(_): + return {"tailored_paragraph": "lead with MCP."} + + written = {"calls": 0, "with_tailor": 0} + + async def fake_vault_write(state): + written["calls"] += 1 + if state.get("tailored_paragraph"): + written["with_tailor"] += 1 + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + return written + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_approve_runs_tailor_then_vault_write(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + + pending = await state_store.list_pending() + tid = pending[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": True, "feedback": "LGTM"}) + assert final["vault_written"] is True + assert final["human_approved"] is True + assert stub_llm_nodes["with_tailor"] == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_reject_skips_tailor_writes_to_vault(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/2", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": False}) + assert final["human_approved"] is False + assert stub_llm_nodes["with_tailor"] == 0 + assert stub_llm_nodes["calls"] == 1 # vault_write still fired (rejected branch) + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_unknown_thread_raises(): + from compass.hitl.resume import resume_pending + + with pytest.raises(LookupError, match="thread"): + await resume_pending("nope-1234", decision={"approved": True}) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_already_resolved_raises(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/3", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + await resume_pending(tid, decision={"approved": True}) + + # Second resume call should not crash and should not re-fire vault_write + before = stub_llm_nodes["calls"] + with pytest.raises(ValueError, match="already resolved"): + await resume_pending(tid, decision={"approved": True}) + assert stub_llm_nodes["calls"] == before From 4e2832264c0f2d8c5836f278239e9d54b297e7fb Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:41:07 -0500 Subject: [PATCH 08/46] feat(hitl): timeout_checker auto-cancels stale approvals; agent-log audit --- compass/hitl/timeout_checker.py | 75 ++++++++++++ tests/hitl/test_timeout_checker.py | 181 +++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 compass/hitl/timeout_checker.py create mode 100644 tests/hitl/test_timeout_checker.py diff --git a/compass/hitl/timeout_checker.py b/compass/hitl/timeout_checker.py new file mode 100644 index 0000000..24213b7 --- /dev/null +++ b/compass/hitl/timeout_checker.py @@ -0,0 +1,75 @@ +"""Auto-cancel pending approvals older than HITL_TIMEOUT_HOURS. + +Module entrypoint (CLI): python -m compass.hitl.timeout_checker +Async API: check_and_resume_timeouts(timeout_hours: int | None = None) + +In Phase 1.B.1 this is human-run. In Phase 1.B.3 a Modal cron will import and +schedule it — see modal_app.py.""" + +from __future__ import annotations + +import asyncio +import logging + +from compass.hitl import state_store +from compass.hitl.resume import resume_pending + +logger = logging.getLogger(__name__) + + +async def check_and_resume_timeouts(*, timeout_hours: int | None = None) -> int: + """Resume every stale pending row with {"approved": False}. Returns the + count successfully timed-out (i.e. exclude rows that errored).""" + from compass.config import HITL_TIMEOUT_HOURS + + hrs = timeout_hours if timeout_hours is not None else HITL_TIMEOUT_HOURS + stale = await state_store.list_timed_out(timeout_hours=hrs) + if not stale: + return 0 + + logger.info("hitl: %d pending approval(s) past %dh timeout", len(stale), hrs) + timed_out = 0 + for row in stale: + tid = row["thread_id"] + try: + await resume_pending( + tid, + decision={ + "approved": False, + "feedback": f"auto-cancelled after {hrs}h timeout", + }, + status_override="timed_out", + ) + # Spec § DoD line 230: HiTL timeout must log to _meta/agent-log.md + _log_to_agent_log( + f"hitl-timeout: auto-cancelled {tid} ({row['company']} / {row['title']}, " + f"score={row['score']:.2f}) after {hrs}h" + ) + timed_out += 1 + except Exception as e: + logger.exception("hitl: timeout resume failed for %s — marking as error", tid) + try: + await state_store.mark_resolved(tid, status="error", feedback=f"resume error: {e}") + _log_to_agent_log(f"hitl-timeout-error: {tid} resume failed: {e}") + except Exception: + logger.exception("hitl: also failed to mark %s as error", tid) + return timed_out + + +def _log_to_agent_log(line: str) -> None: + """Append a timestamped row to compass-vault/_meta/agent-log.md. + + Late-binds via compass.vault.writer.append_agent_log so the temp_vault + fixture works in tests. + """ + from compass.vault.writer import append_agent_log + + try: + append_agent_log(line) + except Exception: + logger.exception("hitl: failed to write to agent-log.md") + + +if __name__ == "__main__": + n = asyncio.run(check_and_resume_timeouts()) + print(f"Timed out: {n}") diff --git a/tests/hitl/test_timeout_checker.py b/tests/hitl/test_timeout_checker.py new file mode 100644 index 0000000..82989b8 --- /dev/null +++ b/tests/hitl/test_timeout_checker.py @@ -0,0 +1,181 @@ +"""timeout_checker resumes pending rows older than HITL_TIMEOUT_HOURS with +{"approved": False}, marks them as 'timed_out' in the state store, appends +a row to _meta/agent-log.md (spec § DoD line 230), and leaves young +pending rows untouched.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="x", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ) + } + + async def fake_vault_write(_): + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + async def fake_tailor(_): + return {"tailored_paragraph": "..."} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_resumes_old_pending_as_rejected(monkeypatch): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the row's created_at to look 5h old + import aiosqlite + + from compass.config import HITL_STATE_DB + + five_h_ago = (_dt.datetime.now(_dt.UTC) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute( + "UPDATE pending_approvals SET created_at = ? WHERE thread_id = ?", + (five_h_ago, tid), + ) + await conn.commit() + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "timed_out" + # Spec DoD line 230 — timeout must be logged to _meta/agent-log.md + from compass.config import AGENT_LOG_PATH + + log_text = AGENT_LOG_PATH.read_text(encoding="utf-8") + assert "hitl-timeout" in log_text + assert tid in log_text + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_leaves_young_pending_alone(): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/2", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 0 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["status"] == "pending" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_continues_after_one_resume_failure(monkeypatch): + """A single broken thread (e.g. checkpoint corrupted) must not block the + rest of the queue from timing out.""" + from compass.hitl import timeout_checker + + # Two rows, both stale + await state_store.add_pending( + thread_id="tid-broken", + job_url="x://1", + company="C", + title="T", + score=4.0, + score_reasoning="r", + matched_skills=[], + missing_skills=[], + ) + await state_store.add_pending( + thread_id="tid-ok", + job_url="x://2", + company="C", + title="T", + score=4.0, + score_reasoning="r", + matched_skills=[], + missing_skills=[], + ) + # Backdate both + import aiosqlite + + from compass.config import HITL_STATE_DB + + five_h_ago = (_dt.datetime.now(_dt.UTC) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute("UPDATE pending_approvals SET created_at = ?", (five_h_ago,)) + await conn.commit() + + # Make resume_pending raise for tid-broken + async def flaky(thread_id, **kw): + if thread_id == "tid-broken": + raise RuntimeError("checkpoint missing") + from compass.hitl.state_store import mark_resolved + + await mark_resolved(thread_id, status="timed_out") + + monkeypatch.setattr("compass.hitl.timeout_checker.resume_pending", flaky) + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 # one succeeded, one failed + row_broken = await state_store.get_pending("tid-broken") + assert row_broken["status"] == "error" From 634a98d263086df2af72001707a18dc0aa621613 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:44:49 -0500 Subject: [PATCH 09/46] refactor(hitl): extract HITL_TIMEOUT_FEEDBACK_PREFIX shared constant --- compass/hitl/__init__.py | 14 ++++++++++++++ compass/hitl/timeout_checker.py | 4 ++-- compass/pipeline/nodes/vault_write.py | 3 ++- tests/hitl/test_timeout_checker.py | 10 ++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/compass/hitl/__init__.py b/compass/hitl/__init__.py index e69de29..e222fa2 100644 --- a/compass/hitl/__init__.py +++ b/compass/hitl/__init__.py @@ -0,0 +1,14 @@ +"""HiTL pause/resume infrastructure for Compass. + +The audit-trail mapping in vault_write._derive_hitl_decision matches on this +prefix; both sites must import HITL_TIMEOUT_FEEDBACK_PREFIX to stay in sync. +""" + +HITL_TIMEOUT_FEEDBACK_PREFIX = "auto-cancelled after" +"""Leading substring of the feedback string used to mark a timeout-driven resume. + +Used by: + - compass.hitl.timeout_checker — writes f"{HITL_TIMEOUT_FEEDBACK_PREFIX} {hrs}h timeout" + - compass.pipeline.nodes.vault_write._derive_hitl_decision — matches + feedback.startswith(HITL_TIMEOUT_FEEDBACK_PREFIX) to set hitl_decision="timed_out" +""" diff --git a/compass/hitl/timeout_checker.py b/compass/hitl/timeout_checker.py index 24213b7..7e81867 100644 --- a/compass/hitl/timeout_checker.py +++ b/compass/hitl/timeout_checker.py @@ -11,7 +11,7 @@ import asyncio import logging -from compass.hitl import state_store +from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX, state_store from compass.hitl.resume import resume_pending logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ async def check_and_resume_timeouts(*, timeout_hours: int | None = None) -> int: tid, decision={ "approved": False, - "feedback": f"auto-cancelled after {hrs}h timeout", + "feedback": f"{HITL_TIMEOUT_FEEDBACK_PREFIX} {hrs}h timeout", }, status_override="timed_out", ) diff --git a/compass/pipeline/nodes/vault_write.py b/compass/pipeline/nodes/vault_write.py index fb93350..eb45377 100644 --- a/compass/pipeline/nodes/vault_write.py +++ b/compass/pipeline/nodes/vault_write.py @@ -36,6 +36,7 @@ def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | None]: """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" from compass.config import SCORE_THRESHOLD + from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX approved = state.get("human_approved") if approved is None: @@ -48,7 +49,7 @@ def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | N if approved is True: decision = "approved" - elif feedback.startswith("auto-cancelled after"): + elif feedback.startswith(HITL_TIMEOUT_FEEDBACK_PREFIX): decision = "timed_out" elif score_value < SCORE_THRESHOLD: decision = "auto_rejected" diff --git a/tests/hitl/test_timeout_checker.py b/tests/hitl/test_timeout_checker.py index 82989b8..df833b4 100644 --- a/tests/hitl/test_timeout_checker.py +++ b/tests/hitl/test_timeout_checker.py @@ -179,3 +179,13 @@ async def flaky(thread_id, **kw): assert n == 1 # one succeeded, one failed row_broken = await state_store.get_pending("tid-broken") assert row_broken["status"] == "error" + + +async def test_timeout_feedback_string_matches_vault_write_prefix(): + """Lock the contract: timeout_checker's feedback string MUST be matchable by + vault_write._derive_hitl_decision's startswith() check. Otherwise the + JobNote.hitl_decision='timed_out' mapping silently breaks.""" + from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX + + feedback = f"{HITL_TIMEOUT_FEEDBACK_PREFIX} 4h timeout" + assert feedback.lower().startswith(HITL_TIMEOUT_FEEDBACK_PREFIX.lower()) From 8af30b75c681356e6fb831f7a9d7e7c1c15103dc Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 00:47:30 -0500 Subject: [PATCH 10/46] feat(mcp): pending_approvals + approve tools --- compass/mcp_server/server.py | 47 ++++++++++++++ tests/mcp_server/__init__.py | 0 tests/mcp_server/test_pending_approvals.py | 75 ++++++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tests/mcp_server/__init__.py create mode 100644 tests/mcp_server/test_pending_approvals.py diff --git a/compass/mcp_server/server.py b/compass/mcp_server/server.py index 1760130..9c7e0ab 100644 --- a/compass/mcp_server/server.py +++ b/compass/mcp_server/server.py @@ -38,6 +38,10 @@ update_application_status(app_id, ...) -> transition status with optional next-action fields list_pending_actions(through_date) -> ApplicationNotes with due next_action_date tailor_resume(job_id) -> tailoring suggestions from JobNote frontmatter + + HiTL approvals: + pending_approvals() -> paused threads awaiting approval (oldest first) + approve(thread_id, approved, feedback) -> resume a paused thread; runs tailor on approve """ from __future__ import annotations @@ -46,6 +50,8 @@ from compass.analysis import gap_aggregator, skill_assessor from compass.config import VAULT_PATH +from compass.hitl import state_store as _state_store +from compass.hitl.resume import resume_pending as _resume_pending from compass.vault.learning_bridge import path_to_uri, resolve, scan_evidence from compass.vault.taxonomy import all_canonicals @@ -330,5 +336,46 @@ async def tailor_resume(job_id: str) -> dict: } +# ── HiTL approvals ─────────────────────────────────────────────────────────── + + +@mcp.tool() +async def pending_approvals() -> list[dict]: + """List jobs paused at hitl awaiting human approval. Oldest first. + + Returned rows include: thread_id, job_url, company, title, score, + score_reasoning, matched_skills, missing_skills, created_at (ISO8601 UTC), + status, resolved_at, feedback. All values are JSON-safe primitives. + """ + rows = await _state_store.list_pending() + # rows are already plain dicts of JSON-safe primitives + list[str] + return rows + + +@mcp.tool() +async def approve(thread_id: str, approved: bool, feedback: str | None = None) -> dict: + """Resume a paused thread. approved=True runs tailor + vault_write; + approved=False skips tailor and writes the rejected JobNote. + + Returns {"vault_written": bool, "human_approved": bool, "human_feedback": str | None} + on success, or {"error": "..."} if the thread is unknown or already resolved. + """ + try: + final = await _resume_pending( + thread_id, + decision={"approved": approved, "feedback": feedback}, + ) + except (LookupError, ValueError) as e: + return {"error": str(e)} + # Strip non-JSON-safe state from the return (current_job is a Pydantic model, + # extracted_requirements / score_result are Pydantic). Phase 1.A bug #11 + # pattern: FastMCP cannot serialize Pydantic instances or datetime objects. + return { + "vault_written": bool(final.get("vault_written")), + "human_approved": bool(final.get("human_approved")), + "human_feedback": final.get("human_feedback"), + } + + if __name__ == "__main__": mcp.run() diff --git a/tests/mcp_server/__init__.py b/tests/mcp_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/mcp_server/test_pending_approvals.py b/tests/mcp_server/test_pending_approvals.py new file mode 100644 index 0000000..0c51efb --- /dev/null +++ b/tests/mcp_server/test_pending_approvals.py @@ -0,0 +1,75 @@ +"""MCP wrappers around state_store + resume_pending — JSON-serializable +return shapes (no datetime objects, no Pydantic instances).""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from compass.hitl import state_store + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_pending_approvals_tool_returns_jsonable_rows(): + from compass.mcp_server.server import pending_approvals + + await state_store.add_pending( + thread_id="tid-1", + job_url="https://x/1", + company="Sierra", + title="SWE Agent", + score=4.2, + score_reasoning="solid", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + ) + rows = await pending_approvals() + assert len(rows) == 1 + assert rows[0]["thread_id"] == "tid-1" + # Must be plain JSON-serialisable + json.dumps(rows) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_invokes_resume(monkeypatch): + from compass.mcp_server import server as mcp_server + + captured = {} + + async def fake_resume(thread_id, *, decision, status_override=None): + captured["thread_id"] = thread_id + captured["decision"] = decision + captured["status_override"] = status_override + return {"vault_written": True, "human_approved": decision["approved"]} + + monkeypatch.setattr(mcp_server, "_resume_pending", fake_resume) + + result = await mcp_server.approve(thread_id="tid-x", approved=True, feedback="LGTM") + assert result == {"vault_written": True, "human_approved": True, "human_feedback": None} + assert captured == { + "thread_id": "tid-x", + "decision": {"approved": True, "feedback": "LGTM"}, + "status_override": None, + } + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_handles_unknown_thread_as_error_dict(): + from compass.mcp_server.server import approve + + result = await approve(thread_id="missing", approved=False) + assert "error" in result and "missing" in result["error"] From 244140616cf41c39ace183692eba09dd1cc4076b Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 01:27:46 -0500 Subject: [PATCH 11/46] fix(hitl): close audit-trail divergence; migrate run log; PID in thread_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1: resume_pending now derives state_store status from the FINAL graph state's human_approved, not the input decision — a graph that auto-rejects on resume (SCORE_THRESHOLD edited between pause and resume, hitl_node short-circuits before consuming interrupt()) no longer leaves state_store='approved' while the JobNote says hitl_decision='auto_rejected'. C1 sticky-threshold: score_node now captures SCORE_THRESHOLD into CompassState at score time. hitl_node and vault_write_node prefer the captured value over the live config, so threshold edits between pause and resume can no longer silently flip the decision. C2: _append_run_log self-migrates 5-col pre-1.B.1 pipeline-runs.md to 6-col on next append (Paused=0 inserted into historical rows). I2: _thread_id_for includes os.getpid() to disambiguate cross-process same- microsecond collisions (defensive for 1.B.3 Modal cron + local CLI race). --- compass/hitl/resume.py | 7 ++- compass/mcp_server/server.py | 1 + compass/pipeline/graph.py | 59 ++++++++++++++++++++-- compass/pipeline/nodes/hitl.py | 11 +++- compass/pipeline/nodes/score.py | 7 ++- compass/pipeline/nodes/vault_write.py | 7 ++- compass/pipeline/state.py | 2 + tests/hitl/test_resume.py | 32 ++++++++++++ tests/pipeline/test_graph_checkpointing.py | 9 ++-- tests/pipeline/test_graph_integration.py | 42 +++++++++++++++ tests/pipeline/test_hitl_node_interrupt.py | 42 +++++++++++++++ 11 files changed, 206 insertions(+), 13 deletions(-) diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py index c94cea7..5b7d382 100644 --- a/compass/hitl/resume.py +++ b/compass/hitl/resume.py @@ -50,7 +50,12 @@ async def resume_pending( if status_override is not None: resolved_status = status_override else: - resolved_status = "approved" if decision.get("approved") else "rejected" + # Derive from the FINAL graph state, not the input decision. A graph that + # auto-rejected on resume (e.g. SCORE_THRESHOLD edited between pause and + # resume, hitl_node short-circuited before consuming interrupt()) would + # otherwise be recorded as "approved" in state_store while the JobNote + # carries hitl_decision="auto_rejected" — audit-trail divergence. + resolved_status = "approved" if final.get("human_approved") is True else "rejected" await state_store.mark_resolved( thread_id, status=resolved_status, diff --git a/compass/mcp_server/server.py b/compass/mcp_server/server.py index 9c7e0ab..c379dc6 100644 --- a/compass/mcp_server/server.py +++ b/compass/mcp_server/server.py @@ -97,6 +97,7 @@ async def score_jd(jd_text: str) -> dict: "jobs_written": 0, "errors": [], "thread_id": None, + "score_threshold": None, } extract_result = await extract_node(state) if extract_result.get("errors"): diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index cd845ce..1b009db 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -119,8 +119,14 @@ def _vault_url_set() -> set[str]: def _thread_id_for(job_url: str, batch_started_at: datetime) -> str: - """Deterministic 16-char SHA-1 of (url, batch start) — same batch + same job = same thread.""" - raw = f"{job_url}|{batch_started_at.isoformat()}" + """Deterministic 16-char SHA-1 of (url, batch start, pid) — same batch + same job = same thread. + + PID is included to disambiguate cross-process collisions (e.g. local CLI + and Modal cron both starting at the same wall-clock microsecond in 1.B.3). + """ + import os + + raw = f"{job_url}|{batch_started_at.isoformat()}|{os.getpid()}" return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] @@ -140,6 +146,7 @@ def _initial_state(job: RawJob, thread_id: str | None = None) -> CompassState: "jobs_written": 0, "errors": [], "thread_id": thread_id, + "score_threshold": None, } @@ -279,6 +286,7 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: "in_scope": None, "role_family": None, "thread_id": None, + "score_threshold": None, } # `jobs_paused` is informational only — not in CompassState TypedDict. aggregate["jobs_paused"] = paused_count # type: ignore[typeddict-unknown-key] @@ -305,13 +313,21 @@ def _append_run_log(state: CompassState, duration_s: float) -> None: """Append one row per run to `_meta/pipeline-runs.md` for debugging cron failures.""" log_path = VAULT_PATH / "_meta" / "pipeline-runs.md" log_path.parent.mkdir(parents=True, exist_ok=True) + expected_header = "| Timestamp | Processed | Written | Paused | Errors | Duration |" + expected_separator = "|---|---|---|---|---|---|" + if not log_path.exists(): log_path.write_text( - "# Pipeline Run Log\n\n" - "| Timestamp | Processed | Written | Paused | Errors | Duration |\n" - "|---|---|---|---|---|---|\n", + f"# Pipeline Run Log\n\n{expected_header}\n{expected_separator}\n", encoding="utf-8", ) + else: + # One-time migration: pre-1.B.1 logs are 5-column. Insert a Paused=0 + # column into existing rows so Dataview parses the table correctly. + text = log_path.read_text(encoding="utf-8") + if expected_header not in text: + _migrate_run_log(log_path, text, expected_header, expected_separator) + ts = datetime.now().isoformat(timespec="seconds") paused = state.get("jobs_paused", 0) # type: ignore[typeddict-item] row = ( @@ -322,6 +338,39 @@ def _append_run_log(state: CompassState, duration_s: float) -> None: f.write(row) +def _migrate_run_log(log_path, text: str, expected_header: str, expected_separator: str) -> None: + """One-time migration: insert Paused=0 column into existing 5-col rows. + + Pre-1.B.1 rows: | ts | processed | written | errors | duration | + Post-1.B.1 rows: | ts | processed | written | paused | errors | duration | + """ + new_lines = [] + in_table = False + migrated_header = False + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("| Timestamp"): + new_lines.append(expected_header) + in_table = True + migrated_header = True + continue + if in_table and stripped.startswith("|---"): + new_lines.append(expected_separator) + continue + if in_table and stripped.startswith("|") and not stripped.startswith("| Timestamp"): + cells = [c.strip() for c in stripped.strip("|").split("|")] + if len(cells) == 5: + ts, processed, written, errors, duration = cells + new_lines.append(f"| {ts} | {processed} | {written} | 0 | {errors} | {duration} |") + continue + elif len(cells) == 6: + new_lines.append(line) + continue + new_lines.append(line) + if migrated_header: + log_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8") + + def _count_unknown_skills_seen_this_run(start_wall: datetime) -> int: """Count rows appended to the unknown-skills log since `start_wall`. diff --git a/compass/pipeline/nodes/hitl.py b/compass/pipeline/nodes/hitl.py index 30ae342..743ec68 100644 --- a/compass/pipeline/nodes/hitl.py +++ b/compass/pipeline/nodes/hitl.py @@ -26,12 +26,19 @@ async def hitl_node(state: CompassState) -> dict: score = state.get("score_result") job = state.get("current_job") - if score is None or score.score < SCORE_THRESHOLD: + # Prefer the threshold captured at score time (sticky across pause/resume). + # Falls back to live config only for backward-compat with paused threads + # checkpointed before this field existed. + threshold = state.get("score_threshold") + if threshold is None: + threshold = SCORE_THRESHOLD + + if score is None or score.score < threshold: logger.info( "hitl: auto-reject %s (score=%s, threshold=%.2f)", job.url if job else "(unknown)", getattr(score, "score", None), - SCORE_THRESHOLD, + threshold, ) return {"human_approved": False} diff --git a/compass/pipeline/nodes/score.py b/compass/pipeline/nodes/score.py index 817c383..b282ef6 100644 --- a/compass/pipeline/nodes/score.py +++ b/compass/pipeline/nodes/score.py @@ -11,6 +11,7 @@ import logging +from compass.config import SCORE_THRESHOLD from compass.llm import make_agent from compass.pipeline.state import CompassState, JobRequirements, JobScore from compass.vault.reader import read_resume, read_skill_inventory @@ -113,9 +114,13 @@ async def score_node(state: CompassState) -> dict: return { "score_result": None, "errors": [*state.get("errors", []), f"score_node: {type(e).__name__}: {e}"], + "score_threshold": SCORE_THRESHOLD, } - return {"score_result": _constrain_to_jd_skills(result, req)} + return { + "score_result": _constrain_to_jd_skills(result, req), + "score_threshold": SCORE_THRESHOLD, + } def _constrain_to_jd_skills(score: JobScore, req: JobRequirements) -> JobScore: diff --git a/compass/pipeline/nodes/vault_write.py b/compass/pipeline/nodes/vault_write.py index eb45377..ca4c08f 100644 --- a/compass/pipeline/nodes/vault_write.py +++ b/compass/pipeline/nodes/vault_write.py @@ -47,11 +47,16 @@ def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | N score = state.get("score_result") score_value = score.score if score is not None else 0.0 + # Prefer state-captured threshold (sticky); fall back to config for old checkpoints. + threshold = state.get("score_threshold") + if threshold is None: + threshold = SCORE_THRESHOLD + if approved is True: decision = "approved" elif feedback.startswith(HITL_TIMEOUT_FEEDBACK_PREFIX): decision = "timed_out" - elif score_value < SCORE_THRESHOLD: + elif score_value < threshold: decision = "auto_rejected" else: decision = "rejected" diff --git a/compass/pipeline/state.py b/compass/pipeline/state.py index e71978c..6158838 100644 --- a/compass/pipeline/state.py +++ b/compass/pipeline/state.py @@ -74,3 +74,5 @@ class CompassState(TypedDict): errors: list[str] thread_id: str | None + + score_threshold: float | None diff --git a/tests/hitl/test_resume.py b/tests/hitl/test_resume.py index 92907eb..438bce0 100644 --- a/tests/hitl/test_resume.py +++ b/tests/hitl/test_resume.py @@ -131,6 +131,38 @@ async def test_resume_unknown_thread_raises(): await resume_pending("nope-1234", decision={"approved": True}) +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_status_derives_from_final_state_not_input(stub_llm_nodes, monkeypatch): + """If the graph auto-rejects on resume (e.g. threshold edited mid-flight), + state_store must record 'rejected', not the input decision's 'approved'.""" + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/divergence-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the resume to auto-reject regardless of the input decision — + # simulates hitl_node short-circuiting on resume (e.g. SCORE_THRESHOLD + # edited between pause and resume). + async def auto_reject(state): + return {"human_approved": False} + + monkeypatch.setattr("compass.pipeline.graph.hitl_node", auto_reject) + + await resume_pending(tid, decision={"approved": True, "feedback": "human said yes"}) + + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" # NOT "approved" — graph really auto-rejected + + @pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") async def test_resume_already_resolved_raises(stub_llm_nodes): from compass.hitl.resume import resume_pending diff --git a/tests/pipeline/test_graph_checkpointing.py b/tests/pipeline/test_graph_checkpointing.py index f69eba3..c50d93e 100644 --- a/tests/pipeline/test_graph_checkpointing.py +++ b/tests/pipeline/test_graph_checkpointing.py @@ -125,11 +125,14 @@ async def low_score(state: CompassState) -> dict: @pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") -async def test_thread_id_is_deterministic_for_same_url_and_batch(): - """Re-running the SAME batch (same start_wall) reuses the same thread_id — - second run hits the idempotent INSERT OR IGNORE path.""" +async def test_thread_id_is_deterministic_for_same_url_and_batch_same_process(monkeypatch): + """Re-running the SAME batch (same start_wall) within the same process + reuses the same thread_id — second run hits the idempotent INSERT OR IGNORE path.""" + import os + from compass.pipeline.graph import _thread_id_for + monkeypatch.setattr(os, "getpid", lambda: 12345) tid_a = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) tid_b = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) tid_c = _thread_id_for("https://jobs.example.com/y", _dt.datetime(2026, 5, 19, 9, 0, 0)) diff --git a/tests/pipeline/test_graph_integration.py b/tests/pipeline/test_graph_integration.py index aa7533b..4f050e8 100644 --- a/tests/pipeline/test_graph_integration.py +++ b/tests/pipeline/test_graph_integration.py @@ -209,3 +209,45 @@ async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms, auto_app log_text = log_path.read_text() assert "| Timestamp |" in log_text # header was written assert "| 1 |" in log_text # one job processed + + +async def test_run_log_migrates_5col_to_6col_on_first_post_1b1_append(temp_vault): + """A pre-1.B.1 pipeline-runs.md with 5-col rows gets migrated to 6-col on + the next append. Existing data preserved with Paused=0.""" + log_path = temp_vault / "_meta" / "pipeline-runs.md" + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text( + "# Pipeline Run Log\n\n" + "| Timestamp | Processed | Written | Errors | Duration |\n" + "|---|---|---|---|---|\n" + "| 2026-05-17T10:00:00 | 20 | 5 | 0 | 7.2s |\n" + "| 2026-05-18T10:00:00 | 15 | 3 | 1 | 5.4s |\n", + encoding="utf-8", + ) + + from compass.pipeline.graph import _append_run_log + + state = { + "raw_jobs": [], + "current_job": None, + "extracted_requirements": None, + "score_result": None, + "in_scope": None, + "role_family": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 10, + "jobs_written": 2, + "errors": [], + "thread_id": None, + "score_threshold": None, + } + state["jobs_paused"] = 1 # type: ignore[typeddict-unknown-key] + _append_run_log(state, 4.1) + + text = log_path.read_text(encoding="utf-8") + assert "| Timestamp | Processed | Written | Paused | Errors | Duration |" in text + assert "| 2026-05-17T10:00:00 | 20 | 5 | 0 | 0 | 7.2s |" in text # migrated + assert "| 2026-05-18T10:00:00 | 15 | 3 | 0 | 1 | 5.4s |" in text diff --git a/tests/pipeline/test_hitl_node_interrupt.py b/tests/pipeline/test_hitl_node_interrupt.py index b6d0506..f8ca5ae 100644 --- a/tests/pipeline/test_hitl_node_interrupt.py +++ b/tests/pipeline/test_hitl_node_interrupt.py @@ -98,3 +98,45 @@ async def test_malformed_resume_value_defaults_to_rejected(monkeypatch): monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", lambda _p: "bogus") result = await hitl_node(_state(score=4.2)) assert result == {"human_approved": False} + + +async def test_below_threshold_uses_state_score_threshold_when_present(monkeypatch): + """If state['score_threshold'] is set (captured at score time), hitl uses + it — not the live config value. Prevents resume-time SCORE_THRESHOLD + edits from silently auto-rejecting a pre-approved thread.""" + captured = {} + + def fake_interrupt(_p): + captured["called"] = True + return {"approved": True} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + + state = _state(score=3.0) + state["score_threshold"] = 2.0 # captured at score time; less than 3.0 + # Live config: SCORE_THRESHOLD=3.5 (default) + import compass.pipeline.nodes.hitl as hitl_mod + + monkeypatch.setattr(hitl_mod, "SCORE_THRESHOLD", 3.5) + + result = await hitl_node(state) + assert captured.get("called") is True # interrupt fired despite config threshold > score + assert result["human_approved"] is True + + +async def test_below_threshold_falls_back_to_config_when_state_threshold_missing(monkeypatch): + """Backward-compat: paused threads from before this fix have no + state['score_threshold']; hitl falls back to config.""" + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda p: pytest.fail("interrupt should not fire when score < config threshold"), + ) + + state = _state(score=2.0) + # state['score_threshold'] not set — defaults to None via .get() + import compass.pipeline.nodes.hitl as hitl_mod + + monkeypatch.setattr(hitl_mod, "SCORE_THRESHOLD", 3.5) + + result = await hitl_node(state) + assert result == {"human_approved": False} From 25736b0bd3bd4af19bd0e8dd453790393f623467 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 01:43:54 -0500 Subject: [PATCH 12/46] fix(hitl): regenerate gap plan after resume to keep counters in sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C3: gap_aggregator.regenerate() was only called at end of run_pipeline(). Resume paths (MCP approve + timeout_checker) bypassed it, so CompanyNote roles_seen and SkillNote appears_in_jobs drifted from JobNote ground truth. Adding the call inside resume_pending closes the single source of truth. Defers I4 (double-resume race claim_pending) to Phase 1.B.3 alongside Modal cron — the real-world race trigger. --- compass/hitl/resume.py | 15 ++++++++ .../2026-05-18-compass-phase-1b1-hitl.md | 1 + tests/hitl/test_resume.py | 35 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py index 5b7d382..0bf65a7 100644 --- a/compass/hitl/resume.py +++ b/compass/hitl/resume.py @@ -61,5 +61,20 @@ async def resume_pending( status=resolved_status, feedback=decision.get("feedback"), ) + + if final.get("vault_written"): + # Resumed thread wrote a JobNote — derived counters (skills/appears_in_jobs, + # companies/roles_seen) now drift unless we resync. Phase 0 bug #12 + Phase + # 1.A bug #1 family: run_pipeline() calls regenerate() at end of batch, but + # resume paths (MCP approve + timeout_checker) bypass that. Call it here so + # every resume keeps the vault internally consistent. + from compass.analysis import gap_aggregator + + try: + gap_aggregator.regenerate(write=True) + except Exception: + # Counter sync failure must not block the resume — log and continue. + logger.exception("hitl: gap_aggregator.regenerate failed after resume") + logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) return final diff --git a/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md index b63e928..b95ed27 100644 --- a/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md +++ b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md @@ -2197,6 +2197,7 @@ All of the following must hold before declaring Phase 1.B.1 complete: | `approve(thread_id, approved=None)` "request changes" branch | UX | post-1.B | Today binary; if needed, ship as third decision dict key | | Per-thread auth (anyone with MCP access can approve) | Security | post-1.B | Solo-user system; revisit if hosted | | Surface `pending_approvals` in the Dataview dashboard | UX polish | **2.B** | Dataview can't read SQLite; needs a tiny exporter | +| Atomic `claim_pending()` to prevent double-resume race (I4: MCP approve vs. `timeout_checker` cron, or MCP double-click) | Race / dup writes | **1.B.3** | Real-world trigger is the Modal cron; ship the claim alongside cron in 1.B.3. Today's MCP-only flow is single-user, no concurrent caller. | --- diff --git a/tests/hitl/test_resume.py b/tests/hitl/test_resume.py index 438bce0..357102a 100644 --- a/tests/hitl/test_resume.py +++ b/tests/hitl/test_resume.py @@ -163,6 +163,41 @@ async def auto_reject(state): assert row["status"] == "rejected" # NOT "approved" — graph really auto-rejected +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_regenerates_counters(stub_llm_nodes, monkeypatch): + """A resumed thread writes a JobNote — gap_aggregator.regenerate() MUST run + so derived counters (CompanyNote.roles_seen, SkillNote.appears_in_jobs) + reflect the new JobNote. Phase 0 #12 / Phase 1.A #1 drift family.""" + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + regen_called = {"count": 0} + + def fake_regenerate(write: bool = False): + regen_called["count"] += 1 + return ([], {}) + + monkeypatch.setattr("compass.analysis.gap_aggregator.regenerate", fake_regenerate) + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/regen-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Baseline: run_pipeline did NOT regenerate (jobs_written==0 because the + # job paused before vault_write). Reset counter to isolate the resume call. + regen_called["count"] = 0 + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + assert regen_called["count"] == 1, "resume_pending must call gap_aggregator.regenerate" + + @pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") async def test_resume_already_resolved_raises(stub_llm_nodes): from compass.hitl.resume import resume_pending From 230799db42478ba1285fec48b5d228eb72524270 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 02:40:03 -0500 Subject: [PATCH 13/46] docs: phase 1.b.2 RAG implementation plan --- .../plans/2026-05-19-compass-phase-1b2-rag.md | 1324 +++++++++++++++++ 1 file changed, 1324 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md diff --git a/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md b/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md new file mode 100644 index 0000000..5af25f0 --- /dev/null +++ b/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md @@ -0,0 +1,1324 @@ +# Compass Phase 1.B.2 — RAG via Chroma + Phase 1.B.1 carryover cleanup (Implementation Plan) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace `score_node`'s wholesale-inject of `_profile/skill-inventory.md` with semantic retrieval against a Chroma vector index (closes the spec's RAG portfolio claim). Also close two Phase 1.B.1 carryover defects flagged in the post-smoke adversarial reviews: LangGraph msgpack deprecation warnings on `RawJob` / `JobRequirements` / `JobScore` (will break paused threads on the next LangGraph major) and `checkpoints.db` bloat from never-deleted resolved-thread checkpoint blobs. + +**Architecture:** New `compass/rag/` package with two modules. `indexer.py` parses `_profile/skill-inventory.md` into one Chroma document per `## SkillName` section, embeds via `sentence-transformers` (`all-MiniLM-L6-v2`), persists to `CHROMA_PATH`. Idempotent on re-run (upsert by stable id = skill heading). `retriever.py` exposes `retrieve(query, k=8) -> list[Chunk]` with lazy index init (build-on-miss). `score_node._profile_text()` is rewritten to retrieve top-k chunks given the JD's `required_skills + nice_to_have + summary` joined as a query, instead of injecting the whole inventory. Resume markdown still ships in full (it's small and load-bearing for context). Token-cost savings: ~2,500 tokens → ~750 tokens per scored JD. Two small infrastructure cleanups land alongside: a serde registration for the three Pydantic state types via `JsonPlusSerializer.register_pydantic_class()` (closes msgpack warnings) and a `_purge_thread_checkpoints(thread_id)` step at the end of `resume_pending` (closes the bloat). + +**Module-level discipline (carried forward from 1.A + 1.B.1):** every module that touches `compass.config.CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH`, `HITL_CHECKPOINT_DB`, or `VAULT_PATH` must reference it via `import compass.config as cfg; cfg.` *inside function bodies*, never as module-level captured constants. The `temp_*` test fixtures monkeypatch these attributes. + +**Tech Stack:** Python 3.12 · chromadb 1.5 (`PersistentClient`) · sentence-transformers 5.5 (`all-MiniLM-L6-v2`, ~90MB local model cache) · langgraph 1.2 (`from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer`) · pytest + pytest-asyncio (`auto` mode, no `asyncio.run` in tests). + +**Authoritative spec:** `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` § Phase 1.B +**Previous-phase handoff:** Phase 1.B.1 retrospective lives in this branch's commits between `phase-1a-application-tracking` and `phase-1b1-hitl`. See `docs/PHASE_1A_COMPLETE.md` for the broader handoff and adversarial-review pattern. + +**Closes these deferred items from Phase 1.B.1:** +1. **I-2 (Phase 1.B.1 post-smoke pass 6)** — `~/.compass/checkpoints.db` bloat from never-purged resolved-thread blobs. Fix lands in Task 2. +2. **I1 (Phase 1.B.1 post-smoke pass 4)** — LangGraph msgpack `Deserializing unregistered type compass.pipeline.state.RawJob` warnings. Will become hard errors when `LANGGRAPH_STRICT_MSGPACK=true` defaults on (currently False, verified by `langgraph.checkpoint.serde.jsonplus`). Fix lands in Task 1. + +**Does NOT touch in this phase:** +- HiTL flow (Phase 1.B.1 — done; only carryover cleanup is in scope) +- I4 race claim_pending (deferred to Phase 1.B.3; documented in 1.B.1 plan's deferred table) +- Modal cron + Langfuse callback fix + URL-dedup for filtered-jobs → **Phase 1.B.3** +- 30-JD eval harness → **Phase 2.A** +- Do not modify Phase 1.B.1 HiTL code outside the explicit carryover fixes in Tasks 1–2. + +--- + +## File Structure + +### New +- `compass/rag/__init__.py` — empty +- `compass/rag/indexer.py` — parse skill-inventory.md → embed → persist to Chroma; `build_index(force_rebuild=False)`; `__main__` CLI entrypoint +- `compass/rag/retriever.py` — `retrieve(query, k=8) -> list[RetrievedChunk]`; lazy index init (build if missing) +- `tests/rag/__init__.py` +- `tests/rag/conftest.py` — `temp_chroma_path` fixture (monkeypatches `compass.config.CHROMA_PATH`); `tiny_inventory` fixture (writes a small 3-section inventory to the temp vault); model-cache reuse fixture so the 90MB sentence-transformers model is only fetched once per test session +- `tests/rag/test_indexer.py` +- `tests/rag/test_retriever.py` +- `tests/pipeline/test_score_node_rag.py` — verifies score_node uses retrieved chunks instead of full inventory + +### Modify +- `compass/pipeline/nodes/score.py` — `_profile_text()` rewritten to call retriever; keep resume inline; signature compatibility preserved +- `compass/pipeline/graph.py` — pass `serde=JsonPlusSerializer(...)` to `AsyncSqliteSaver.from_conn_string(...)` AND register the three Pydantic state types +- `compass/hitl/resume.py` — call `_purge_thread_checkpoints(thread_id)` after `mark_resolved` succeeds (only on `vault_written=True`, mirrors the regen pattern) + +### Untouched +- All Phase 1.B.1 HiTL infrastructure (`compass/hitl/{state_store,timeout_checker,__init__}.py`) +- All other pipeline nodes (`intake`, `intake_filter`, `extract`, `reflect`, `hitl`, `tailor`, `vault_write`) +- Vault schemas, vault writer, scrapers, applications lifecycle +- MCP server tools + +### Decomposition rationale +The two HiTL cleanups (msgpack + checkpoint purge) are intentionally Tasks 1 and 2 — small, isolated, and lower-risk than the RAG work. They land FIRST so the new tests pass against a stable baseline before any RAG changes. RAG splits across three files: `indexer.py` (write path), `retriever.py` (read path), and the `score_node` integration. The indexer/retriever split lets tests exercise each side in isolation (build an index; query an index) without needing the full pipeline. The retriever is the only one `score_node` depends on, keeping the dependency arrow one-way. `temp_chroma_path` is per-test to avoid sentence-transformer-cache leakage across tests, but the model itself (≈90MB) is loaded once per session via a module-cached factory. + +--- + +## Task 0: Pre-flight + +**Files:** none + +- [ ] **Step 1: Verify clean tree on `phase-1b1-hitl` tag** + +```bash +cd /Users/akmini/Documents/compass +git status # expected: clean +git describe --tags --abbrev=0 # expected: phase-1b1-hitl +uv run pytest -q # expected: 239 passed +uv run ruff check # expected: All checks passed +``` + +If working tree is dirty, STOP and ask the user before proceeding. + +- [ ] **Step 2: Confirm Chroma + sentence-transformers + langgraph serde availability AND verify the construction form actually suppresses warnings** + +```bash +uv run python -c " +import chromadb, sentence_transformers, logging, datetime +print('chromadb', chromadb.__version__) # expected: 1.x +print('sentence_transformers', sentence_transformers.__version__) # expected: 5.x +from chromadb import PersistentClient +import langgraph.checkpoint.serde.jsonplus as _jp +_jp._warned_unregistered_types.clear() # reset module-level once-only set +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer +from compass.pipeline.state import RawJob, JobRequirements, JobScore + +# Pass the allowlist at construction — `with_msgpack_allowlist` is a no-op +# when `allowed_msgpack_modules=True` (langgraph's non-STRICT default). +captured = [] +class H(logging.Handler): + def emit(self, r): + msg = r.getMessage() + if 'unregistered' in msg or 'fingerprint' in msg.lower(): + captured.append(msg) +logging.getLogger('langgraph').addHandler(H()) + +serde = JsonPlusSerializer(allowed_msgpack_modules=[ + ('compass.pipeline.state', 'RawJob'), + ('compass.pipeline.state', 'JobRequirements'), + ('compass.pipeline.state', 'JobScore'), +]) +rj = RawJob(company='C', title='T', url='u://x', source='ashby', description='d', date_posted=datetime.date(2026,5,19)) +ser = serde.dumps_typed(rj) +restored = serde.loads_typed(ser) +assert isinstance(restored, RawJob), f'expected RawJob, got {type(restored).__name__}' +assert not captured, f'allowlist did not suppress warnings: {captured}' +print('serde tuple-list allowlist verified — no warnings, types preserved') +" +``` + +Expected: prints `serde tuple-list allowlist verified — no warnings, types preserved`. If it fails with warnings still captured OR `restored` is not a RawJob, the allowlist API has shifted; STOP and inspect `inspect.signature(JsonPlusSerializer)` for the current constructor signature before continuing. + +**Lesson:** the deprecation warning text is misleading — it says "add to allowed_msgpack_modules" suggesting you can call a method after construction. You cannot. The allowlist MUST be passed at construction time as `(module, classname)` tuples. + +- [ ] **Step 2.5: Confirm checkpoint DB table names match what Task 2 will DELETE from** + +```bash +sqlite3 ~/.compass/checkpoints.db "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;" +``` + +Expected: exactly `checkpoints` and `writes`. If the table names differ, Task 2's DELETE statements must be updated to match — there's no try/except wrapper to mask a mismatch. + +- [ ] **Step 3: Verify `_profile/skill-inventory.md` structure (drives chunking)** + +```bash +grep -c '^## ' ~/Documents/compass-vault/_profile/skill-inventory.md +``` + +Expected: ≥ 15 level-2 headings. If the structure has materially changed, the chunking strategy (`## SkillName` per chunk) needs revisiting first. + +- [ ] **Step 4: Create branch** + +```bash +git checkout -b phase-1b2-rag +``` + +--- + +## Task 1: Register Pydantic state types with LangGraph checkpointer serde (carryover I1) + +**Why first:** Smallest, lowest-risk fix. Closes the msgpack deprecation warnings logged on every resume in Phase 1.B.1's smoke. Lands at the top of the phase so all subsequent tests benefit from a quieter log. + +**Files:** +- Modify: `compass/pipeline/graph.py` +- Create: `tests/pipeline/test_checkpoint_serde.py` + +**Background:** LangGraph 1.2 uses msgpack for checkpoint serialization. When it encounters an unregistered class (our `RawJob`, `JobRequirements`, `JobScore` Pydantic models), it logs `Deserializing unregistered type ... This will be blocked in a future version`. The clean fix is to construct a `JsonPlusSerializer` with explicit type registration and pass it to `AsyncSqliteSaver(serde=...)`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/pipeline/test_checkpoint_serde.py`: + +```python +"""Verify LangGraph checkpoint serde explicitly allowlists the + compass.pipeline.state module so the 'Deserializing unregistered type' + warnings (logged in 1.B.1 smoke) don't recur, and so a future + LANGGRAPH_STRICT_MSGPACK=true default doesn't break paused-thread + deserialization.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +async def test_build_checkpoint_serde_allowlists_state_module(): + """_build_checkpoint_serde must return a JsonPlusSerializer whose + allowlist suppresses the 'unregistered type' warning AND preserves + the Pydantic class on round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer + from compass.pipeline.graph import _build_checkpoint_serde + + # langgraph caches "already warned for this type" in a module-level set; + # clear it so this test sees a clean slate regardless of prior imports. + _jp._warned_unregistered_types.clear() + + serde = _build_checkpoint_serde() + assert isinstance(serde, JsonPlusSerializer) + + from compass.pipeline.state import RawJob + import logging + captured = [] + class _H(logging.Handler): + def emit(self, r): + if "unregistered type" in r.getMessage() or "fingerprint" in r.getMessage().lower(): + captured.append(r.getMessage()) + h = _H() + logging.getLogger("langgraph").addHandler(h) + try: + import datetime + rj = RawJob(company="C", title="T", url="u://x", source="ashby", + description="d", date_posted=datetime.date(2026, 5, 19)) + ser = serde.dumps_typed(rj) + restored = serde.loads_typed(ser) + assert isinstance(restored, RawJob) + finally: + logging.getLogger("langgraph").removeHandler(h) + assert not captured, f"serde still emitted unregistered-type warnings: {captured}" + + +async def test_run_pipeline_emits_no_unregistered_warnings(checkpoint_db, monkeypatch, temp_vault): + """Smoke: run_pipeline mounts AsyncSqliteSaver with our serde, not the + default. No 'unregistered type' warnings should be emitted during + a checkpoint round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from compass.pipeline.state import JobRequirements, JobScore, RawJob + import datetime as _dt + import logging + + # See note in test_build_checkpoint_serde_allowlists_state_module — clear + # the once-only warned-types set so this test isn't fooled by warnings + # already emitted in earlier imports. + _jp._warned_unregistered_types.clear() + + captured = [] + class _Handler(logging.Handler): + def emit(self, record): + if "unregistered type" in record.getMessage(): + captured.append(record.getMessage()) + handler = _Handler() + logging.getLogger("langgraph").addHandler(handler) + + # Stub the LLM-touching nodes (existing pattern from test_graph_checkpointing.py) + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="x")} + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes=""), "score_threshold": 3.5} + async def fake_tailor(_): return {"tailored_paragraph": "..."} + async def fake_vault_write(_): return {"vault_written": True, "jobs_written": 1} + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + # Auto-approve interrupt so the graph completes + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", + lambda _: {"approved": True, "feedback": None}) + + from compass.pipeline.graph import run_pipeline + await run_pipeline(raw_jobs=[RawJob( + company="Sierra", title="SWE", url="u://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 19), + )]) + + logging.getLogger("langgraph").removeHandler(handler) + assert not captured, f"Expected no 'unregistered type' warnings, got: {captured}" +``` + +Run: `uv run pytest tests/pipeline/test_checkpoint_serde.py -v` — expect both to fail (`_build_checkpoint_serde` doesn't exist). + +- [ ] **Step 2: Implement `_build_checkpoint_serde` in `compass/pipeline/graph.py`** + +Add near the existing checkpointer imports: + +```python +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer +``` + +Add as a module-level helper just above `build_graph`: + +```python +def _build_checkpoint_serde() -> JsonPlusSerializer: + """Allow our Pydantic state classes on the msgpack allowlist. + + Must be set at construction — `with_msgpack_allowlist` is a no-op when the + default `allowed_msgpack_modules=True` (langgraph's non-STRICT default). + """ + return JsonPlusSerializer( + allowed_msgpack_modules=[ + ("compass.pipeline.state", "RawJob"), + ("compass.pipeline.state", "JobRequirements"), + ("compass.pipeline.state", "JobScore"), + ] + ) +``` + +- [ ] **Step 3: Wire the serde into `AsyncSqliteSaver.from_conn_string`** + +In `run_pipeline`, change: + +```python +async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: +``` + +to: + +```python +async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() +``` + +(`from_conn_string(conn_string: str)` in checkpoint-sqlite 3.1 doesn't accept `serde=`. Setting `checkpointer.serde` after entering the context manager is the supported pattern.) + +Apply the same change in `compass/hitl/resume.py`'s `async with AsyncSqliteSaver.from_conn_string(...)` block — same pattern, same one line. Otherwise resume reuses the default serde and the warnings recur. + +- [ ] **Step 4: Run the tests** + +```bash +uv run pytest tests/pipeline/test_checkpoint_serde.py -v +``` + +Expected: 2 passed. If the second test still captures warnings, the serde isn't being applied to the active checkpointer — re-inspect `run_pipeline` and `resume.py`. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 241 passed (239 + 2 new). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/graph.py compass/hitl/resume.py tests/pipeline/test_checkpoint_serde.py +git commit -m "feat(hitl): register Pydantic state types with checkpoint serde" +``` + +--- + +## Task 2: Purge resolved-thread checkpoint blobs (carryover I-2) + +**Background:** Phase 1.B.1 left ~7.6MB in `~/.compass/checkpoints.db` after the smoke. Every paused thread accumulates ~10 checkpoint blobs and is never deleted post-resolution. At 50 jobs/day ÷ 5 paused, the file grows ~1MB/day forever. + +**Files:** +- Modify: `compass/hitl/resume.py` — `_purge_thread_checkpoints(thread_id)` called inside the `async with checkpointer` block after `mark_resolved` succeeds +- Modify: `tests/hitl/test_resume.py` — add a regression test + +- [ ] **Step 1: Write the failing test** + +Append to `tests/hitl/test_resume.py`: + +```python +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_purges_thread_checkpoint_blobs(stub_llm_nodes): + """After a thread is resolved (approved OR rejected), its checkpoint rows + in HITL_CHECKPOINT_DB must be deleted — otherwise the DB grows + unboundedly. Phase 1.B.1 post-smoke I-2 finding.""" + import aiosqlite + from compass.config import HITL_CHECKPOINT_DB + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="u://purge-probe", + source="ashby", description="...", date_posted=_dt.date(2026, 5, 19)) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + tid = (await state_store.list_pending())[0]["thread_id"] + + # Verify checkpoint rows exist before resume + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (pre_count,) = await cur.fetchone() + assert pre_count > 0, f"setup error: no checkpoint rows for {tid}" + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + + # Verify checkpoint rows for this thread are gone (state_store row is + # what stays — that's the audit trail, not the LangGraph internals). + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (post_count,) = await cur.fetchone() + assert post_count == 0, f"checkpoint rows for resolved thread {tid} were not purged" +``` + +Run: `uv run pytest tests/hitl/test_resume.py::test_resume_purges_thread_checkpoint_blobs -v` — expect failure. + +- [ ] **Step 2: Implement `_purge_thread_checkpoints` in `compass/hitl/resume.py`** + +Add a private helper at the bottom of the file: + +```python +async def _purge_thread_checkpoints(thread_id: str) -> None: + """Drop LangGraph per-step history for a resolved thread; state_store is the audit trail.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + await conn.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,)) + await conn.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,)) + await conn.commit() +``` + +Wire the call inside `resume_pending` AFTER `mark_resolved` succeeds AND AFTER the `async with AsyncSqliteSaver(...)` block exits — `_purge_thread_checkpoints` opens its own connection, so it doesn't need the saver's context: + +```python + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() # carried from Task 1 + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + resolved_status = "approved" if final.get("human_approved") is True else "rejected" + await state_store.mark_resolved( + thread_id, status=resolved_status, feedback=decision.get("feedback"), + ) + await _purge_thread_checkpoints(thread_id) + + if final.get("vault_written"): + from compass.analysis import gap_aggregator + try: + gap_aggregator.regenerate(write=True) + except Exception: + logger.exception("hitl: gap_aggregator.regenerate failed after resume") +``` + +- [ ] **Step 3: Run the test** + +```bash +uv run pytest tests/hitl/test_resume.py -v +``` + +Expected: all 6 (5 prior + 1 new) pass. + +- [ ] **Step 4: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 242 passed (241 + 1). Ruff clean. + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/resume.py tests/hitl/test_resume.py +git commit -m "fix(hitl): purge thread checkpoint blobs on resume (bound checkpoints.db growth)" +``` + +--- + +## Task 3: RAG indexer (`compass/rag/indexer.py`) + +**Files:** +- Create: `compass/rag/__init__.py` (empty) +- Create: `compass/rag/indexer.py` +- Create: `tests/rag/__init__.py` +- Create: `tests/rag/conftest.py` +- Create: `tests/rag/test_indexer.py` + +**Chunking strategy:** one chunk per `## SkillName` heading in `_profile/skill-inventory.md`. The chunk's `id` is the kebab-cased skill name (deterministic across rebuilds). The chunk's `metadata` carries `skill` (raw heading text), `section` (parent grouping if applicable; for now just "skill"), `source` ("skill-inventory.md"). The chunk's `document` is the section body text (heading + everything until the next `## ` or EOF). + +**Embedding model:** `all-MiniLM-L6-v2` via sentence-transformers. The model downloads on first use (~90MB to `~/.cache/torch/sentence_transformers/`); for tests we use a session-scoped fixture so it loads once. + +- [ ] **Step 1: Write the conftest** + +Create `tests/rag/conftest.py`: + +```python +"""Shared fixtures for RAG tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def temp_chroma_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Per-test Chroma index path — avoids cross-test pollution.""" + p = tmp_path / "chroma" + import compass.config as cfg + monkeypatch.setattr(cfg, "CHROMA_PATH", p) + return p + + +@pytest.fixture +def tiny_inventory(temp_vault, monkeypatch: pytest.MonkeyPatch) -> Path: + """Write a small skill-inventory.md to the temp vault — 3 distinct sections + so retrieval tests can verify which section gets surfaced.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "# Skill Inventory\n\n" + "## Python\n\n" + "Level 4. 5 years building backends and agents in Python. " + "Cisco MCP server, LangGraph pipelines, FastAPI services.\n\n" + "## LangGraph\n\n" + "Level 3. Built Compass pipeline with stateful graph, conditional " + "edges, interrupt()+AsyncSqliteSaver checkpointing for HITL.\n\n" + "## React\n\n" + "Level 1. Touched in school projects. Not a primary skill.\n", + encoding="utf-8", + ) + return inv + + +@pytest.fixture(scope="session") +def embedding_model_cached(): + """Pre-load the sentence-transformers model once per test session. + Subsequent fixtures + tests reuse the in-process cache.""" + from sentence_transformers import SentenceTransformer + # Touch it to force download/cache. Use the same name as compass.config. + SentenceTransformer("all-MiniLM-L6-v2") +``` + +- [ ] **Step 2: Write the failing indexer tests** + +Create `tests/rag/test_indexer.py`: + +```python +"""indexer parses skill-inventory.md by ## SkillName, embeds, persists. + Idempotent on rebuild (upsert by stable id = kebab-cased skill name). + + These tests use real sentence-transformers (the model is cached at session + scope; ~90MB on first run). They do NOT touch the network beyond the + initial model download.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_indexer_builds_one_chunk_per_section(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + count = await indexer.build_index() + assert count == 3, f"expected 3 chunks (Python/LangGraph/React), got {count}" + + +async def test_indexer_uses_kebab_case_ids(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + client = indexer._client() + col = client.get_collection("skill_inventory") + got_ids = set(col.get()["ids"]) + assert got_ids == {"python", "langgraph", "react"} + + +async def test_indexer_metadata_carries_skill_and_source(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + client = indexer._client() + col = client.get_collection("skill_inventory") + res = col.get(ids=["python"]) + assert res["metadatas"][0]["skill"] == "Python" + assert res["metadatas"][0]["source"] == "skill-inventory.md" + + +async def test_indexer_is_idempotent_on_rebuild(tiny_inventory, temp_chroma_path): + """A second build_index() must not duplicate rows. Upsert by id.""" + from compass.rag import indexer + + await indexer.build_index() + await indexer.build_index() # second call + client = indexer._client() + col = client.get_collection("skill_inventory") + assert col.count() == 3 + + +async def test_indexer_force_rebuild_clears_stale_chunks(tiny_inventory, temp_chroma_path): + """If the inventory shrinks (skill removed), force_rebuild=True must remove + the stale chunk; otherwise the old embedding lingers and pollutes retrieval.""" + from compass.rag import indexer + + await indexer.build_index() + tiny_inventory.write_text( + "# Skill Inventory\n\n## Python\n\nLevel 4.\n", + encoding="utf-8", + ) + await indexer.build_index(force_rebuild=True) + client = indexer._client() + col = client.get_collection("skill_inventory") + got_ids = set(col.get()["ids"]) + assert got_ids == {"python"} + + +async def test_indexer_repairs_stale_l2_collection(temp_vault, temp_chroma_path): + """An existing L2 collection from a pre-1.B.2 install must be dropped+recreated + with the cosine metric — otherwise the retriever score formula breaks silently.""" + from chromadb import PersistentClient + + from compass.rag import indexer + + # Simulate pre-1.B.2 state: collection exists with default (L2) metric. + client = PersistentClient(path=str(temp_chroma_path)) + client.create_collection(name="skill_inventory") # no cosine metadata + + col = indexer._collection(client) + assert (col.metadata or {}).get("hnsw:space") == "cosine" + + +async def test_indexer_excludes_assessor_grades_heading(temp_vault, temp_chroma_path): + """Auto-generated `## Assessor-current grades` is metadata, not a skill — + must NOT land in the index or it pollutes retrieval with every skill name.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "## Python\n\nLevel 4.\n\n" + "## Assessor-current grades\n\nPython: 4 / LangGraph: 3 / MCP: 5\n", + encoding="utf-8", + ) + from compass.rag import indexer + + n = await indexer.build_index() + assert n == 1 + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_handles_empty_inventory_gracefully(temp_vault, temp_chroma_path): + """No ## sections — count is 0, no crash, no collection bloat.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Skill Inventory\n\nNo skills yet.\n", encoding="utf-8") + from compass.rag import indexer + + count = await indexer.build_index() + assert count == 0 +``` + +Run: `uv run pytest tests/rag/test_indexer.py -v` — expect all to fail with `ModuleNotFoundError: compass.rag.indexer`. + +- [ ] **Step 3: Implement `compass/rag/indexer.py`** + +```python +"""Chroma index of _profile/skill-inventory.md. + +One chunk per `## SkillName` section. Collection metric is cosine — the +retriever's similarity score formula depends on this. +""" + +from __future__ import annotations + +import asyncio +import logging +import re + +logger = logging.getLogger(__name__) + +_COLLECTION_NAME = "skill_inventory" +_SECTION_RE = re.compile(r"^## +(.+)$", re.M) +# Auto-generated assessor output; not a skill — exclude from the index. +_EXCLUDED_HEADINGS = {"Assessor-current grades"} + + +def _client(): + import compass.config as cfg + from chromadb import PersistentClient + + cfg.CHROMA_PATH.mkdir(parents=True, exist_ok=True) + return PersistentClient(path=str(cfg.CHROMA_PATH)) + + +def _collection(client): + """Return the skill_inventory collection pinned to cosine distance. + + `get_or_create_collection` does NOT update an existing collection's + metadata — so a pre-existing L2 collection would silently keep the wrong + metric and break the retriever's score formula. Drop and recreate if so. + """ + existing = {c.name for c in client.list_collections()} + if _COLLECTION_NAME in existing: + col = client.get_collection(_COLLECTION_NAME) + if (col.metadata or {}).get("hnsw:space") == "cosine": + return col + client.delete_collection(_COLLECTION_NAME) + return client.create_collection( + name=_COLLECTION_NAME, + metadata={"hnsw:space": "cosine"}, + ) + + +def _kebab(name: str) -> str: + # Drop parenthetical asides and em-dash commentary — they're not identity. + # `Fine-Tuning (awareness only — ...)` -> `fine-tuning` + # `MCP — your strongest cluster` -> `mcp` + primary = re.sub(r"\s*(?:\(|[—–]\s).*$", "", name).strip() + return re.sub(r"[^a-z0-9]+", "-", primary.lower()).strip("-") or "untitled" + + +def _parse_inventory(text: str) -> list[tuple[str, str]]: + """Return [(skill_name, section_text), ...] — one entry per ## heading. + + Skips headings in _EXCLUDED_HEADINGS (auto-generated metadata that isn't + a real skill and would otherwise pollute retrieval). + """ + matches = list(_SECTION_RE.finditer(text)) + sections = [] + for i, m in enumerate(matches): + name = m.group(1).strip() + if any(name.startswith(prefix) for prefix in _EXCLUDED_HEADINGS): + continue + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + sections.append((name, text[m.start():end].strip())) + return sections + + +def _embed(documents: list[str]) -> list[list[float]]: + import compass.config as cfg + from sentence_transformers import SentenceTransformer + + model = SentenceTransformer(cfg.EMBEDDING_MODEL) + arr = model.encode(documents, convert_to_numpy=True, show_progress_bar=False) + return [vec.tolist() for vec in arr] + + +async def build_index(force_rebuild: bool = False) -> int: + """Embed each `## Section` of skill-inventory.md and upsert into Chroma. + + Idempotent — repeat calls upsert by stable id. Pass `force_rebuild=True` + to drop the collection first (use when the inventory shrinks; otherwise + deleted skills' stale chunks linger). + """ + import compass.config as cfg + + if not cfg.SKILL_INVENTORY_PATH.exists(): + logger.warning("rag: skill-inventory.md not found at %s", cfg.SKILL_INVENTORY_PATH) + return 0 + + sections = _parse_inventory(cfg.SKILL_INVENTORY_PATH.read_text(encoding="utf-8")) + if not sections: + logger.info("rag: 0 ## sections in skill-inventory.md; nothing to index") + return 0 + + client = _client() + if force_rebuild and _COLLECTION_NAME in {c.name for c in client.list_collections()}: + client.delete_collection(_COLLECTION_NAME) + collection = _collection(client) + + documents = [body for _, body in sections] + embeddings = await asyncio.to_thread(_embed, documents) + + collection.upsert( + ids=[_kebab(name) for name, _ in sections], + documents=documents, + embeddings=embeddings, + metadatas=[ + {"skill": name, "source": "skill-inventory.md"} for name, _ in sections + ], + ) + logger.info("rag: indexed %d sections at %s", len(sections), cfg.CHROMA_PATH) + return len(sections) + + +def _main() -> None: + import argparse + + import compass.config as cfg + + parser = argparse.ArgumentParser(description="Rebuild Chroma index of skill-inventory.md") + parser.add_argument("--force", action="store_true", help="Drop+rebuild the collection") + n = asyncio.run(build_index(force_rebuild=parser.parse_args().force)) + print(f"Indexed {n} sections from {cfg.SKILL_INVENTORY_PATH}") + + +if __name__ == "__main__": + _main() +``` + +- [ ] **Step 4: Run the tests** + +```bash +uv run pytest tests/rag/test_indexer.py -v +``` + +Expected: 8 passed. First run downloads the model (~90MB, 30-60s on first install). Subsequent runs are fast. + +If a test fails because Chroma's `get_or_create_collection` API differs in your installed version, check the actual `chromadb` 1.5 API and adjust the call. The public surface has been stable in 0.5+. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 250 passed (242 + 8). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/rag/__init__.py compass/rag/indexer.py tests/rag/__init__.py \ + tests/rag/conftest.py tests/rag/test_indexer.py +git commit -m "feat(rag): chroma index of skill-inventory.md (one chunk per ## section)" +``` + +--- + +## Task 4: RAG retriever (`compass/rag/retriever.py`) + +**Files:** +- Create: `compass/rag/retriever.py` +- Create: `tests/rag/test_retriever.py` + +- [ ] **Step 1: Write the failing retriever tests** + +Create `tests/rag/test_retriever.py`: + +```python +"""retriever.retrieve(query, k) returns top-k chunks from the Chroma index. + Lazy-init: builds the index on miss so first-run usage doesn't crash.""" + +from __future__ import annotations + +import pytest + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_retrieve_returns_top_k_relevant_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + + hits = await retriever.retrieve("Python backend agent work", k=2) + assert len(hits) == 2 + # Soft assertion — embedding-model top-1 can flap on a 3-doc corpus. + # The real contract is "Python ranks in the top-2", not "exactly first". + skills = [h.skill for h in hits] + assert "Python" in skills + assert any("Cisco MCP" in h.document for h in hits) # body content present + assert all(0.0 <= h.score <= 1.0 for h in hits) # normalized similarity scores + + +async def test_retrieve_lazy_inits_index_on_miss(tiny_inventory, temp_chroma_path): + """If no index exists yet, retrieve() builds one before querying.""" + from compass.rag import retriever + + # No build_index() call beforehand + hits = await retriever.retrieve("LangGraph stateful pipeline", k=1) + assert len(hits) == 1 + assert hits[0].skill == "LangGraph" + + +async def test_retrieve_returns_empty_when_inventory_empty(temp_vault, temp_chroma_path): + """No sections in inventory → retrieve returns [] gracefully (no crash).""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Empty\n", encoding="utf-8") + from compass.rag import retriever + + hits = await retriever.retrieve("anything", k=5) + assert hits == [] + + +async def test_retrieve_k_larger_than_corpus_returns_all(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + hits = await retriever.retrieve("python or react", k=99) + assert 1 <= len(hits) <= 3 # corpus has 3 chunks; can't exceed +``` + +Run: `uv run pytest tests/rag/test_retriever.py -v` — expect failures (`compass.rag.retriever` doesn't exist). + +- [ ] **Step 2: Implement `compass/rag/retriever.py`** + +```python +"""Top-k retrieval over the skill-inventory Chroma index. + +Lazy-init on first call so scoring works before any explicit indexer run. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from compass.rag import indexer + + +@dataclass +class RetrievedChunk: + skill: str + document: str + score: float # cosine similarity in [0, 1]; 1.0 = identical + + +async def retrieve(query: str, k: int = 8) -> list[RetrievedChunk]: + if not (query or "").strip(): + return [] + + collection = indexer._collection(indexer._client()) + if collection.count() == 0 and await indexer.build_index() == 0: + return [] + + [query_emb] = await asyncio.to_thread(indexer._embed, [query]) + res = collection.query( + query_embeddings=[query_emb], + n_results=min(k, collection.count()), + ) + + # Collection is pinned to cosine (see indexer._collection); distance ∈ [0, 2]. + return [ + RetrievedChunk(skill=m["skill"], document=d, score=max(0.0, 1.0 - dist / 2.0)) + for m, d, dist in zip( + res["metadatas"][0], res["documents"][0], res["distances"][0], strict=True + ) + ] +``` + +- [ ] **Step 3: Run the tests** + +```bash +uv run pytest tests/rag/test_retriever.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 254 passed (250 + 4). Ruff clean. + +- [ ] **Step 5: Commit** + +```bash +git add compass/rag/retriever.py tests/rag/test_retriever.py +git commit -m "feat(rag): semantic retriever with lazy index init" +``` + +--- + +## Task 5: Wire retriever into `score_node` + +**Files:** +- Modify: `compass/pipeline/nodes/score.py` +- Create: `tests/pipeline/test_score_node_rag.py` + +**Strategy:** the existing `_profile_text()` returns `## RESUME\n{full resume}\n\n## SKILL INVENTORY\n{full inventory}`. The new version returns `## RESUME\n{full resume}\n\n## RELEVANT SKILLS\n{top-k chunks joined}`. Resume stays inline (it's small and contextual); inventory is replaced by retrieval. + +**Query construction:** join the JD's `required_skills + nice_to_have_skills + summary` into a single string. This embeds the candidate-relevant query in one shot. K=8 covers most expected coverage. + +- [ ] **Step 1: Write the failing test** + +Create `tests/pipeline/test_score_node_rag.py`: + +```python +"""score_node uses RAG retrieval to build profile context, not full inventory.""" + +from __future__ import annotations + +import pytest + +from compass.pipeline.state import JobRequirements + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +def _stub_req(**overrides) -> JobRequirements: + base = dict( + required_skills=["Python", "LangGraph"], + nice_to_have_skills=["MCP"], + seniority="senior", + remote_policy="remote", + summary="Build agentic systems.", + ) + base.update(overrides) + return JobRequirements(**base) + + +async def test_profile_text_includes_resume_and_retrieved_chunks(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME TEXT") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME TEXT" in text + assert "## Python\nLevel 4." in text + + +async def test_retrieval_query_carries_jd_skills_and_summary(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + captured: dict[str, object] = {} + + async def fake_retrieve(query, k=8): + captured["query"] = query + return [RetrievedChunk(skill="Python", document="## Python", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "") + + await score_mod._profile_text(_stub_req()) + q = captured["query"] + assert "Python" in q and "LangGraph" in q and "MCP" in q + assert "Build agentic systems" in q + + +async def test_score_node_handles_empty_jd_skills(monkeypatch): + """A JD with no required/nice-to-have skills falls back to summary only.""" + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query: str, k: int = 8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME") + + req = JobRequirements( + required_skills=[], nice_to_have_skills=[], + seniority="mid", remote_policy="remote", summary="Just a summary.", + ) + text = await score_mod._profile_text(req) + assert "RESUME" in text + assert "Python" in text # still got a retrieved chunk + + +async def test_score_node_handles_empty_retrieval(monkeypatch): + """If retriever returns [] (e.g. empty corpus), the profile context still + includes the resume — score_node doesn't crash.""" + from compass.pipeline.nodes import score as score_mod + + async def fake_retrieve(query: str, k: int = 8): + return [] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME ONLY") + + req = JobRequirements( + required_skills=["X"], nice_to_have_skills=[], + seniority="mid", remote_policy="remote", summary="x", + ) + text = await score_mod._profile_text(req) + assert "RESUME ONLY" in text + # No "## RELEVANT SKILLS" block when retrieval is empty +``` + +Run: `uv run pytest tests/pipeline/test_score_node_rag.py -v` — expect failures (the score module doesn't import rag_retrieve and _profile_text is currently sync). + +- [ ] **Step 2: Rewrite `_profile_text` in `compass/pipeline/nodes/score.py`** + +Two edits to the file: + +**2a. Add the import near the top (alongside `read_resume` / `read_skill_inventory`):** + +```python +from compass.rag.retriever import retrieve as rag_retrieve +``` + +You can REMOVE the `read_skill_inventory` import from `score.py` only — `_profile_text` no longer uses it. **Do NOT** delete `read_skill_inventory` from `compass/vault/reader.py` — `tests/vault/test_reader.py` still exercises it, and the function may be useful for future tooling outside the pipeline. + +**2b. Rewrite `_profile_text` from sync to async, accepting `req: JobRequirements`:** + +```python +async def _profile_text(req: JobRequirements) -> str: + """Build the candidate-profile context for the score prompt. + + RAG via Chroma replaces the prior wholesale-inject of skill-inventory.md. + Resume stays inline (small, load-bearing); inventory is now top-k chunks + retrieved against the JD's skill + summary query. + """ + query_parts = [*req.required_skills, *req.nice_to_have_skills] + if req.summary: + query_parts.append(req.summary) + query = " ".join(query_parts).strip() + + chunks = await rag_retrieve(query, k=8) if query else [] + + profile = f"## RESUME\n{read_resume()}" + if chunks: + ranked = "\n\n".join(c.document for c in chunks) + profile += f"\n\n## RELEVANT SKILLS (top-{len(chunks)} by similarity)\n{ranked}" + return profile +``` + +**2c. Update the caller in `score_node`:** + +```python + profile = await _profile_text(req) + result = await _score_with_retry(req, profile) +``` + +(Currently it's `await _score_with_retry(req, _profile_text())` — split into two lines and pass `req` to the new async helper. **Do not modify the `return {...}` block at the end of `score_node` — the `"score_threshold": SCORE_THRESHOLD` key was added in Phase 1.B.1 and must stay.**) + +- [ ] **Step 3: Run the new tests** + +```bash +uv run pytest tests/pipeline/test_score_node_rag.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Confirm existing score_node tests still pass** + +```bash +uv run pytest tests/pipeline/ -v +``` + +Expected: all green. If existing tests fail because they assumed `_profile_text()` is sync, the test file needs minor updates — usually wrapping a sync `_profile_text()` call site with `asyncio.run()` or making the test async. Inspect the test names that fail and patch them; do NOT change the production code to compensate. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 258 passed (254 + 4). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/nodes/score.py tests/pipeline/test_score_node_rag.py +git commit -m "feat(rag): score_node uses Chroma retrieval instead of full skill-inventory inject" +``` + +--- + +## Task 6: Live smoke + retro + +> **PAUSE HERE before running this task.** Step 3 hits real OpenRouter on at least one JD with the new RAG-injected context. Confirm with the user before running. Cost: ~$0.01 for a single re-score, ~$0.05 for a full small-board pipeline run. + +**Goal:** Prove three things tests cannot prove: +1. The Chroma index actually builds against the real `~/Documents/compass-vault/_profile/skill-inventory.md` (21 sections, not the test fixture's 3) +2. Retrieval against a real JD's skills returns sensible chunks +3. score_node with RAG produces a score consistent with the pre-RAG score (no major drift) + +- [ ] **Step 1: Snapshot the vault and DBs** + +```bash +cp -r ~/Documents/compass-vault/jobs ~/Documents/compass-vault/jobs.preB2.bak 2>/dev/null || true +cp ~/.compass/hitl.db ~/.compass/hitl.db.preB2.bak 2>/dev/null || true +cp ~/.compass/checkpoints.db ~/.compass/checkpoints.db.preB2.bak 2>/dev/null || true +``` + +- [ ] **Step 2: Build the index against the real inventory** + +```bash +uv run python -m compass.rag.indexer --force +``` + +Expected output: `Indexed: N chunks` where N matches `grep -c '^## ' ~/Documents/compass-vault/_profile/skill-inventory.md` (currently 21). + +Verify the index exists: + +```bash +ls -la ~/.compass/chroma/ +``` + +Expected: at least one `.sqlite3` file present. First run may also populate `~/.cache/torch/sentence_transformers/` with the ~90MB model. + +- [ ] **Step 3: Probe retrieval against a real JD profile** + +```bash +uv run python -c " +import asyncio +from compass.rag.retriever import retrieve +async def main(): + hits = await retrieve('agentic AI engineer with LangGraph MCP Python production systems', k=5) + for h in hits: + print(f'{h.score:.3f} {h.skill}') +asyncio.run(main()) +" +``` + +Expected: top-5 results include skills like Python, MCP, LangGraph, Agent Frameworks, RAG (or similar — the inventory's strongest sections). Scores should range roughly 0.4–0.7. If top-5 has irrelevant skills (e.g. "Voice", "Fine-Tuning"), the embedding model isn't separating the inventory well — flag as a quality concern but don't block. + +- [ ] **Step 4: Re-score one existing JobNote with the new pipeline** + +Pick any previously-approved JobNote (e.g. `~/Documents/compass-vault/jobs/2024-10-05-cognition-Special_Projects_Engineer-5343fadf.md` from the Phase 1.B.1 smoke, if it still exists; otherwise pick any JobNote with a known pre-RAG match_score). Note the pre-RAG score; expect the post-RAG score to land within ±0.5. + +```bash +# Re-score via the MCP score_jd tool — bypasses vault writes +uv run python -c " +import asyncio +from compass.mcp_server.server import score_jd +async def main(): + jd_text = open('/Users/akmini/Documents/compass-vault/jobs/2024-10-05-cognition-Special_Projects_Engineer-5343fadf.md').read() + result = await score_jd(jd_text) + print(f'RAG score: {result[\"score\"][\"score\"]}') + print(f'matched: {result[\"score\"][\"matched_skills\"]}') + print(f'missing: {result[\"score\"][\"missing_skills\"]}') +asyncio.run(main()) +" +``` + +Expected: score within 2.5–3.5 (pre-RAG was 3.0). If the new score is wildly different (e.g. 0.5 or 5.0), the retrieval context is missing something critical — investigate the retrieved chunks vs the full inventory before proceeding. + +- [ ] **Step 5: Run a small live pipeline** + +```bash +MAX_JOBS_PER_RUN=5 \ + GREENHOUSE_BOARDS=anthropic \ + LEVER_COMPANIES= \ + ASHBY_BOARDS=decagon \ + uv run python -m compass.pipeline.graph +``` + +Expected output: pipeline completes with no `Deserializing unregistered type` warnings (Task 1 closes them) and no abnormal score drift. Inspect `_meta/agent-log.md` for any errors related to RAG. + +- [ ] **Step 6: Verify checkpoint DB stays bounded** + +After Step 5 + any resume action: + +```bash +sqlite3 ~/.compass/checkpoints.db "SELECT COUNT(DISTINCT thread_id) FROM checkpoints;" +``` + +Expected: count equals the number of still-pending threads (not the total number of threads ever paused). Task 2's purge keeps it bounded. + +- [ ] **Step 7: Tag the phase** + +```bash +git tag phase-1b2-rag +``` + +- [ ] **Step 8: Cleanup or keep backups** + +If everything looks healthy: + +```bash +rm -rf ~/Documents/compass-vault/jobs.preB2.bak ~/.compass/hitl.db.preB2.bak ~/.compass/checkpoints.db.preB2.bak +``` + +If anything looks wrong, restore: + +```bash +rm -rf ~/Documents/compass-vault/jobs && mv ~/Documents/compass-vault/jobs.preB2.bak ~/Documents/compass-vault/jobs +mv ~/.compass/hitl.db.preB2.bak ~/.compass/hitl.db +mv ~/.compass/checkpoints.db.preB2.bak ~/.compass/checkpoints.db +``` + +--- + +## Definition of Done + +**Code & tests** +- 258 tests passing (`uv run pytest -q`) +- Ruff clean (`uv run ruff check && uv run ruff format --check`) +- No module-level captures of `CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH` — every reference is inside a function body +- `_build_checkpoint_serde` is called inside the `async with AsyncSqliteSaver(...)` block in BOTH `run_pipeline` and `resume_pending` (carry forward the Phase 1.A discipline) +- `_purge_thread_checkpoints` runs only after `mark_resolved` succeeds, and inside the open checkpointer context + +**Behaviour (verified empirically in Task 6)** +- `python -m compass.rag.indexer --force` indexes all 21 sections of the real inventory +- Retrieval against a real JD-skills query surfaces sensible top-5 chunks +- A re-score of one existing JobNote produces a score within ±0.5 of the pre-RAG score +- A full small-board pipeline run completes with no `Deserializing unregistered type` warnings +- `checkpoints.db` size stays bounded — resolved threads' rows are deleted + +**Docs** +- Plan file's "What's deferred" table updated if any new items surface +- Phase 1.B.2 retrospective (optional but recommended) saved to `docs/PHASE_1B2_COMPLETE.md` + +--- + +## What's deferred (and to which sub-phase) + +| Concern | Severity | Phase | Why deferred | +|---|---|---|---| +| 30-JD eval harness comparing pre-RAG vs post-RAG score MAE | Portfolio claim | **2.A** | Needs labeled dataset; out of 1.B.2 scope | +| Modal cron + Langfuse callback fix + URL-dedup for filtered-jobs | Required for "no babysitting" | **1.B.3** | Spec phase ordering | +| I4 double-resume race claim_pending (Phase 1.B.1 deferred item) | Important | **1.B.3** | Cron is the real-world race trigger | +| Embedding-model upgrade (e.g. `nomic-embed-text` for higher quality) | Quality polish | **2.B** | `all-MiniLM-L6-v2` is good enough for v1; cost-free swap later | +| Skill-inventory re-indexing on Obsidian write (file-watch) | UX | **3+** | Manual `python -m compass.rag.indexer` is fine for solo use | +| Per-chunk metadata enrichment (level, category, calibration) | Quality | **2.B** | Current metadata = {skill, source} suffices for top-k retrieval | + +--- + +## Critical lessons to carry forward (from Phases 0 + 1.A + 1.B.1) + +1. **Tests check shape; only adversarial real-data inspection catches data-correctness bugs.** Task 6 is non-optional — do not declare 1.B.2 done from green CI alone. +2. **Module-level config captures freeze at import time** and silently break test fixtures. Always late-bind inside function bodies (`CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH`, `HITL_CHECKPOINT_DB`). +3. **A graph compiled without a checkpointer silently breaks `interrupt()`** — apply the same discipline to the new serde: `_build_checkpoint_serde` must be called INSIDE the `async with` block, NEVER stashed in a module-level constant. +4. **JSON-serializability of MCP return shapes.** Phase 1.A bug #11: `date` objects don't serialize through FastMCP. Same applies to RAG retrieval — if any future MCP tool returns `RetrievedChunk`, convert to a plain dict first. +5. **`or`-fallback against falsy values is a recurring trap.** Phase 1.B.1 caught `state.get("score_threshold") or SCORE_THRESHOLD` would mis-fallback on 0.0. Same pattern applies anywhere we read state with a defensible-falsy default — use `is None` checks instead. +6. **The audit-trail/state-store/JobNote consistency triad is load-bearing.** Phase 1.B.1's C1 silent divergence (state_store said "approved", JobNote said "auto_rejected") was caught only by adversarial cross-checking. Apply the same rigor here: after any pipeline change, sample real JobNotes and confirm `match_score`, `matched_skills`, `missing_skills` are consistent with what RAG retrieved. + +--- + +## Plan Review Loop + +Before executing this plan, dispatch a single plan-document-reviewer subagent with: +- Path to this plan: `docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md` +- Path to spec: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- Phase scope reference: this plan's "Closes these deferred items" section + +If the reviewer flags issues, fix in place and re-dispatch. Cap at 3 iterations. + +--- + +## Execution handoff + +Once approved, two options: + +1. **Subagent-Driven (recommended)** — Use `superpowers:subagent-driven-development`. Fresh implementer per Task 1–5, combined spec+quality reviewer between tasks, hard pause before Task 6 (live LLM + real vault). Same pattern that worked in Phase 1.B.1. + +2. **Inline Execution** — Use `superpowers:executing-plans`. Single session, batch through Tasks 1–5 with checkpoints, hard pause before Task 6. + +Phase 1.B.1 used Subagent-Driven and it caught 5+ real bugs across tasks. Keep the pattern. From d41a0d537b8f4b77420dac703f5659b069ec8d48 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 02:42:29 -0500 Subject: [PATCH 14/46] feat(hitl): register Pydantic state types with checkpoint serde --- compass/hitl/resume.py | 3 +- compass/pipeline/graph.py | 17 +++ tests/pipeline/test_checkpoint_serde.py | 147 ++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 tests/pipeline/test_checkpoint_serde.py diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py index 0bf65a7..2efcdf1 100644 --- a/compass/hitl/resume.py +++ b/compass/hitl/resume.py @@ -40,10 +40,11 @@ async def resume_pending( raise ValueError(f"thread {thread_id!r} already resolved ({row['status']!r})") # Late import to avoid potential circular import: graph.py imports state_store - from compass.pipeline.graph import build_graph + from compass.pipeline.graph import _build_checkpoint_serde, build_graph config = {"configurable": {"thread_id": thread_id}} async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() graph = build_graph(checkpointer=checkpointer) final = await graph.ainvoke(Command(resume=decision), config=config) diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index 1b009db..c9ae612 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -23,6 +23,7 @@ from datetime import datetime import frontmatter +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from langgraph.graph import END, START, StateGraph @@ -64,6 +65,21 @@ def _route_after_hitl(state: CompassState) -> str: return "vault_write" +def _build_checkpoint_serde() -> JsonPlusSerializer: + """Allow our Pydantic state classes on the msgpack allowlist. + + Must be set at construction — `with_msgpack_allowlist` is a no-op when the + default `allowed_msgpack_modules=True` (langgraph's non-STRICT default). + """ + return JsonPlusSerializer( + allowed_msgpack_modules=[ + ("compass.pipeline.state", "RawJob"), + ("compass.pipeline.state", "JobRequirements"), + ("compass.pipeline.state", "JobScore"), + ] + ) + + def build_graph(checkpointer=None): builder = StateGraph(CompassState) builder.add_node("intake", intake_node) @@ -254,6 +270,7 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: HITL_CHECKPOINT_DB.parent.mkdir(parents=True, exist_ok=True) async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() # Enable WAL on the checkpoint DB once per process. Cheap if already set. # See state_store._connect for the rationale. try: diff --git a/tests/pipeline/test_checkpoint_serde.py b/tests/pipeline/test_checkpoint_serde.py new file mode 100644 index 0000000..14bc012 --- /dev/null +++ b/tests/pipeline/test_checkpoint_serde.py @@ -0,0 +1,147 @@ +"""LangGraph checkpoint serde — Pydantic state classes on the msgpack allowlist +so checkpoint round-trip neither warns nor blocks under future STRICT mode.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +async def test_build_checkpoint_serde_allowlists_state_module(): + """_build_checkpoint_serde must return a serde whose allowlist suppresses + the 'unregistered type' warning AND preserves Pydantic classes on round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer + + from compass.pipeline.graph import _build_checkpoint_serde + + _jp._warned_unregistered_types.clear() + + serde = _build_checkpoint_serde() + assert isinstance(serde, JsonPlusSerializer) + + import datetime + import logging + + from compass.pipeline.state import RawJob + + captured = [] + + class _H(logging.Handler): + def emit(self, r): + msg = r.getMessage() + if "unregistered type" in msg or "fingerprint" in msg.lower(): + captured.append(msg) + + h = _H() + logging.getLogger("langgraph").addHandler(h) + try: + rj = RawJob( + company="C", + title="T", + url="u://x", + source="ashby", + description="d", + date_posted=datetime.date(2026, 5, 19), + ) + ser = serde.dumps_typed(rj) + restored = serde.loads_typed(ser) + assert isinstance(restored, RawJob) + finally: + logging.getLogger("langgraph").removeHandler(h) + assert not captured, f"serde still emitted unregistered-type warnings: {captured}" + + +async def test_run_pipeline_emits_no_unregistered_warnings(checkpoint_db, monkeypatch, temp_vault): + """End-to-end smoke: run_pipeline mounts AsyncSqliteSaver with our serde. + No 'unregistered type' warnings during a full graph round-trip.""" + import datetime as _dt + import logging + + import langgraph.checkpoint.serde.jsonplus as _jp + + from compass.pipeline.state import JobRequirements, JobScore, RawJob + + _jp._warned_unregistered_types.clear() + + captured = [] + + class _Handler(logging.Handler): + def emit(self, record): + if "unregistered type" in record.getMessage(): + captured.append(record.getMessage()) + + handler = _Handler() + logging.getLogger("langgraph").addHandler(handler) + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="x", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ), + "score_threshold": 3.5, + } + + async def fake_tailor(_): + return {"tailored_paragraph": "..."} + + async def fake_vault_write(_): + return {"vault_written": True, "jobs_written": 1} + + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", lambda _: {"approved": True, "feedback": None} + ) + + from compass.pipeline.graph import run_pipeline + + await run_pipeline( + raw_jobs=[ + RawJob( + company="Sierra", + title="SWE", + url="u://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 19), + ) + ] + ) + + logging.getLogger("langgraph").removeHandler(handler) + assert not captured, f"Expected no 'unregistered type' warnings, got: {captured}" From 1a4313274ecc41f9388a93314508737222349d58 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 02:45:51 -0500 Subject: [PATCH 15/46] fix(hitl): purge thread checkpoint blobs on resume --- compass/hitl/resume.py | 13 +++++++++++++ tests/hitl/test_resume.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py index 2efcdf1..996c76f 100644 --- a/compass/hitl/resume.py +++ b/compass/hitl/resume.py @@ -62,6 +62,7 @@ async def resume_pending( status=resolved_status, feedback=decision.get("feedback"), ) + await _purge_thread_checkpoints(thread_id) if final.get("vault_written"): # Resumed thread wrote a JobNote — derived counters (skills/appears_in_jobs, @@ -79,3 +80,15 @@ async def resume_pending( logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) return final + + +async def _purge_thread_checkpoints(thread_id: str) -> None: + """Drop LangGraph per-step history for a resolved thread; state_store is the audit trail.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + await conn.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,)) + await conn.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,)) + await conn.commit() diff --git a/tests/hitl/test_resume.py b/tests/hitl/test_resume.py index 357102a..0f05e5c 100644 --- a/tests/hitl/test_resume.py +++ b/tests/hitl/test_resume.py @@ -220,3 +220,42 @@ async def test_resume_already_resolved_raises(stub_llm_nodes): with pytest.raises(ValueError, match="already resolved"): await resume_pending(tid, decision={"approved": True}) assert stub_llm_nodes["calls"] == before + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_purges_thread_checkpoint_blobs(stub_llm_nodes): + """After a thread resolves, its checkpoint rows in HITL_CHECKPOINT_DB must + be deleted — otherwise the DB grows unboundedly. Phase 1.B.1 I-2.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="u://purge-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 19), + ) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + tid = (await state_store.list_pending())[0]["thread_id"] + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (pre_count,) = await cur.fetchone() + assert pre_count > 0, f"setup: no checkpoint rows for {tid}" + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (post_count,) = await cur.fetchone() + assert post_count == 0, f"checkpoint rows for resolved {tid} were not purged" From c0a8d4ecca78867294cc687c786cb12e020d83f6 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 02:51:03 -0500 Subject: [PATCH 16/46] feat(rag): chroma index of skill-inventory.md --- compass/rag/indexer.py | 132 ++++++++++++++++++++++++++++++++++++++ tests/rag/__init__.py | 0 tests/rag/conftest.py | 48 ++++++++++++++ tests/rag/test_indexer.py | 96 +++++++++++++++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 compass/rag/indexer.py create mode 100644 tests/rag/__init__.py create mode 100644 tests/rag/conftest.py create mode 100644 tests/rag/test_indexer.py diff --git a/compass/rag/indexer.py b/compass/rag/indexer.py new file mode 100644 index 0000000..7e3f1b2 --- /dev/null +++ b/compass/rag/indexer.py @@ -0,0 +1,132 @@ +"""Chroma index of _profile/skill-inventory.md. + +One chunk per `## SkillName` section. Collection metric is cosine — the +retriever's similarity score formula depends on this. +""" + +from __future__ import annotations + +import asyncio +import logging +import re + +logger = logging.getLogger(__name__) + +_COLLECTION_NAME = "skill_inventory" +_SECTION_RE = re.compile(r"^## +(.+)$", re.M) +# Auto-generated assessor output; not a skill — exclude from the index. +_EXCLUDED_HEADINGS = {"Assessor-current grades"} + + +def _client(): + from chromadb import PersistentClient + + import compass.config as cfg + + cfg.CHROMA_PATH.mkdir(parents=True, exist_ok=True) + return PersistentClient(path=str(cfg.CHROMA_PATH)) + + +def _collection(client): + """Return the skill_inventory collection pinned to cosine distance. + + `get_or_create_collection` does NOT update an existing collection's + metadata — so a pre-existing L2 collection would silently keep the wrong + metric and break the retriever's score formula. Drop and recreate if so. + """ + existing = {c.name for c in client.list_collections()} + if _COLLECTION_NAME in existing: + col = client.get_collection(_COLLECTION_NAME) + if (col.metadata or {}).get("hnsw:space") == "cosine": + return col + client.delete_collection(_COLLECTION_NAME) + return client.create_collection( + name=_COLLECTION_NAME, + metadata={"hnsw:space": "cosine"}, + ) + + +def _kebab(name: str) -> str: + # Drop parenthetical asides and em-dash commentary — they're not identity. + # `Fine-Tuning (awareness only — ...)` -> `fine-tuning` + # `MCP — your strongest cluster` -> `mcp` + primary = re.sub(r"\s*(?:\(|[—–]\s).*$", "", name).strip() + return re.sub(r"[^a-z0-9]+", "-", primary.lower()).strip("-") or "untitled" + + +def _parse_inventory(text: str) -> list[tuple[str, str]]: + """Return [(skill_name, section_text), ...] — one entry per ## heading. + + Skips headings in _EXCLUDED_HEADINGS (auto-generated metadata that isn't + a real skill and would otherwise pollute retrieval). + """ + matches = list(_SECTION_RE.finditer(text)) + sections = [] + for i, m in enumerate(matches): + name = m.group(1).strip() + if any(name.startswith(prefix) for prefix in _EXCLUDED_HEADINGS): + continue + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + sections.append((name, text[m.start() : end].strip())) + return sections + + +def _embed(documents: list[str]) -> list[list[float]]: + from sentence_transformers import SentenceTransformer + + import compass.config as cfg + + model = SentenceTransformer(cfg.EMBEDDING_MODEL) + arr = model.encode(documents, convert_to_numpy=True, show_progress_bar=False) + return [vec.tolist() for vec in arr] + + +async def build_index(force_rebuild: bool = False) -> int: + """Embed each `## Section` of skill-inventory.md and upsert into Chroma. + + Idempotent — repeat calls upsert by stable id. Pass `force_rebuild=True` + to drop the collection first (use when the inventory shrinks; otherwise + deleted skills' stale chunks linger). + """ + import compass.config as cfg + + if not cfg.SKILL_INVENTORY_PATH.exists(): + logger.warning("rag: skill-inventory.md not found at %s", cfg.SKILL_INVENTORY_PATH) + return 0 + + sections = _parse_inventory(cfg.SKILL_INVENTORY_PATH.read_text(encoding="utf-8")) + if not sections: + logger.info("rag: 0 ## sections in skill-inventory.md; nothing to index") + return 0 + + client = _client() + if force_rebuild and _COLLECTION_NAME in {c.name for c in client.list_collections()}: + client.delete_collection(_COLLECTION_NAME) + collection = _collection(client) + + documents = [body for _, body in sections] + embeddings = await asyncio.to_thread(_embed, documents) + + collection.upsert( + ids=[_kebab(name) for name, _ in sections], + documents=documents, + embeddings=embeddings, + metadatas=[{"skill": name, "source": "skill-inventory.md"} for name, _ in sections], + ) + logger.info("rag: indexed %d sections at %s", len(sections), cfg.CHROMA_PATH) + return len(sections) + + +def _main() -> None: + import argparse + + import compass.config as cfg + + parser = argparse.ArgumentParser(description="Rebuild Chroma index of skill-inventory.md") + parser.add_argument("--force", action="store_true", help="Drop+rebuild the collection") + n = asyncio.run(build_index(force_rebuild=parser.parse_args().force)) + print(f"Indexed {n} sections from {cfg.SKILL_INVENTORY_PATH}") + + +if __name__ == "__main__": + _main() diff --git a/tests/rag/__init__.py b/tests/rag/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/rag/conftest.py b/tests/rag/conftest.py new file mode 100644 index 0000000..18e9300 --- /dev/null +++ b/tests/rag/conftest.py @@ -0,0 +1,48 @@ +"""Shared fixtures for RAG tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def temp_chroma_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Per-test Chroma path so the persistent client doesn't leak between tests.""" + p = tmp_path / "chroma" + import compass.config as cfg + + monkeypatch.setattr(cfg, "CHROMA_PATH", p) + return p + + +@pytest.fixture +def tiny_inventory(temp_vault) -> Path: + """Small 3-section inventory for fast retrieval tests.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "# Skill Inventory\n\n" + "## Python\n\n" + "Level 4. 5 years building backends and agents in Python. " + "Cisco MCP server, LangGraph pipelines, FastAPI services.\n\n" + "## LangGraph\n\n" + "Level 3. Built Compass pipeline with stateful graph, conditional " + "edges, interrupt()+AsyncSqliteSaver checkpointing for HITL.\n\n" + "## React\n\n" + "Level 1. Touched in school projects. Not a primary skill.\n", + encoding="utf-8", + ) + return inv + + +@pytest.fixture(scope="session") +def embedding_model_cached(): + """Pre-load sentence-transformers once per test session. ~90MB on first run.""" + from sentence_transformers import SentenceTransformer + + SentenceTransformer("all-MiniLM-L6-v2") diff --git a/tests/rag/test_indexer.py b/tests/rag/test_indexer.py new file mode 100644 index 0000000..d59c748 --- /dev/null +++ b/tests/rag/test_indexer.py @@ -0,0 +1,96 @@ +"""indexer parses skill-inventory.md by ## SkillName, embeds, persists. + +Idempotent on rebuild (upsert by stable kebab id). Uses real sentence- +transformers (model cached at session scope — ~90MB on first run only).""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_indexer_builds_one_chunk_per_section(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + assert await indexer.build_index() == 3 + + +async def test_indexer_uses_kebab_case_ids(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python", "langgraph", "react"} + + +async def test_indexer_metadata_carries_skill_and_source(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + col = indexer._collection(indexer._client()) + res = col.get(ids=["python"]) + assert res["metadatas"][0]["skill"] == "Python" + assert res["metadatas"][0]["source"] == "skill-inventory.md" + + +async def test_indexer_is_idempotent_on_rebuild(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + await indexer.build_index() + col = indexer._collection(indexer._client()) + assert col.count() == 3 + + +async def test_indexer_force_rebuild_clears_stale_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + tiny_inventory.write_text( + "# Skill Inventory\n\n## Python\n\nLevel 4.\n", + encoding="utf-8", + ) + await indexer.build_index(force_rebuild=True) + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_repairs_stale_l2_collection(temp_vault, temp_chroma_path): + """Pre-1.B.2 installs may have an L2 collection — must be dropped+recreated + so the retriever's cosine-score formula stays valid.""" + from chromadb import PersistentClient + + from compass.rag import indexer + + client = PersistentClient(path=str(temp_chroma_path)) + client.create_collection(name="skill_inventory") # default = L2 + + col = indexer._collection(client) + assert (col.metadata or {}).get("hnsw:space") == "cosine" + + +async def test_indexer_excludes_assessor_grades_heading(temp_vault, temp_chroma_path): + """Auto-generated `## Assessor-current grades` is metadata, not a skill — + must NOT land in the index or it pollutes retrieval with every skill name.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "## Python\n\nLevel 4.\n\n" + "## Assessor-current grades\n\nPython: 4 / LangGraph: 3 / MCP: 5\n", + encoding="utf-8", + ) + from compass.rag import indexer + + assert await indexer.build_index() == 1 + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_handles_empty_inventory_gracefully(temp_vault, temp_chroma_path): + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Skill Inventory\n\nNo skills yet.\n", encoding="utf-8") + from compass.rag import indexer + + assert await indexer.build_index() == 0 From 092954d8848c054234fa1bdad8edf65993a06148 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 02:55:38 -0500 Subject: [PATCH 17/46] feat(rag): semantic retriever with lazy index init --- compass/rag/retriever.py | 41 +++++++++++++++++++++++++++++ tests/rag/test_retriever.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 compass/rag/retriever.py create mode 100644 tests/rag/test_retriever.py diff --git a/compass/rag/retriever.py b/compass/rag/retriever.py new file mode 100644 index 0000000..8b51a50 --- /dev/null +++ b/compass/rag/retriever.py @@ -0,0 +1,41 @@ +"""Top-k retrieval over the skill-inventory Chroma index. + +Lazy-init on first call so scoring works before any explicit indexer run. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from compass.rag import indexer + + +@dataclass +class RetrievedChunk: + skill: str + document: str + score: float # cosine similarity in [0, 1]; 1.0 = identical + + +async def retrieve(query: str, k: int = 8) -> list[RetrievedChunk]: + if not (query or "").strip(): + return [] + + collection = indexer._collection(indexer._client()) + if collection.count() == 0 and await indexer.build_index() == 0: + return [] + + [query_emb] = await asyncio.to_thread(indexer._embed, [query]) + res = collection.query( + query_embeddings=[query_emb], + n_results=min(k, collection.count()), + ) + + # Collection is pinned to cosine (see indexer._collection); distance ∈ [0, 2]. + return [ + RetrievedChunk(skill=m["skill"], document=d, score=max(0.0, 1.0 - dist / 2.0)) + for m, d, dist in zip( + res["metadatas"][0], res["documents"][0], res["distances"][0], strict=True + ) + ] diff --git a/tests/rag/test_retriever.py b/tests/rag/test_retriever.py new file mode 100644 index 0000000..b2bf87d --- /dev/null +++ b/tests/rag/test_retriever.py @@ -0,0 +1,51 @@ +"""retriever.retrieve(query, k) — top-k chunks by cosine similarity. + +Lazy-init builds the index on first call so scoring works before any +explicit indexer invocation.""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_retrieve_returns_top_k_relevant_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + + hits = await retriever.retrieve("Python backend agent work", k=2) + assert len(hits) == 2 + skills = [h.skill for h in hits] + assert "Python" in skills + assert any("Cisco MCP" in h.document for h in hits) + assert all(0.0 <= h.score <= 1.0 for h in hits) + + +async def test_retrieve_lazy_inits_index_on_miss(tiny_inventory, temp_chroma_path): + """If no index exists yet, retrieve() builds one before querying.""" + from compass.rag import retriever + + hits = await retriever.retrieve("LangGraph stateful pipeline", k=1) + assert len(hits) == 1 + assert hits[0].skill == "LangGraph" + + +async def test_retrieve_returns_empty_when_inventory_empty(temp_vault, temp_chroma_path): + """No sections in inventory → retrieve returns [] without crashing.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Empty\n", encoding="utf-8") + from compass.rag import retriever + + hits = await retriever.retrieve("anything", k=5) + assert hits == [] + + +async def test_retrieve_k_larger_than_corpus_returns_all(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + hits = await retriever.retrieve("python or react", k=99) + assert 1 <= len(hits) <= 3 From c8f324d71089be2d1f1a70e594c90f251f3099d2 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 03:02:03 -0500 Subject: [PATCH 18/46] feat(rag): score_node uses chroma retrieval instead of full skill-inventory inject --- compass/pipeline/nodes/score.py | 26 ++++++-- tests/conftest.py | 8 +++ tests/pipeline/test_score.py | 11 +++- tests/pipeline/test_score_node_rag.py | 87 +++++++++++++++++++++++++++ tests/rag/conftest.py | 8 --- 5 files changed, 125 insertions(+), 15 deletions(-) create mode 100644 tests/pipeline/test_score_node_rag.py diff --git a/compass/pipeline/nodes/score.py b/compass/pipeline/nodes/score.py index b282ef6..5082599 100644 --- a/compass/pipeline/nodes/score.py +++ b/compass/pipeline/nodes/score.py @@ -14,7 +14,8 @@ from compass.config import SCORE_THRESHOLD from compass.llm import make_agent from compass.pipeline.state import CompassState, JobRequirements, JobScore -from compass.vault.reader import read_resume, read_skill_inventory +from compass.rag.retriever import retrieve as rag_retrieve +from compass.vault.reader import read_resume logger = logging.getLogger(__name__) @@ -95,8 +96,24 @@ async def _score_with_retry(req: JobRequirements, profile_text: str) -> JobScore return retry -def _profile_text() -> str: - return f"## RESUME\n{read_resume()}\n\n## SKILL INVENTORY\n{read_skill_inventory()}" +async def _profile_text(req: JobRequirements) -> str: + """Build candidate-profile context for the score prompt. + + Resume stays inline; the prior full skill-inventory inject is now top-k + chunks retrieved against the JD's skills + summary. + """ + query_parts = [*req.required_skills, *req.nice_to_have_skills] + if req.summary: + query_parts.append(req.summary) + query = " ".join(query_parts).strip() + + chunks = await rag_retrieve(query, k=8) if query else [] + + profile = f"## RESUME\n{read_resume()}" + if chunks: + ranked = "\n\n".join(c.document for c in chunks) + profile += f"\n\n## RELEVANT SKILLS (top-{len(chunks)} by similarity)\n{ranked}" + return profile async def score_node(state: CompassState) -> dict: @@ -108,7 +125,8 @@ async def score_node(state: CompassState) -> dict: } try: - result = await _score_with_retry(req, _profile_text()) + profile = await _profile_text(req) + result = await _score_with_retry(req, profile) except Exception as e: logger.exception("score_node: LLM call failed") return { diff --git a/tests/conftest.py b/tests/conftest.py index e7f2d49..340727e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,14 @@ import pytest +@pytest.fixture(scope="session") +def embedding_model_cached(): + """Pre-load sentence-transformers once per test session. ~90MB on first run.""" + from sentence_transformers import SentenceTransformer + + SentenceTransformer("all-MiniLM-L6-v2") + + @pytest.fixture def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: """Point HITL_STATE_DB at a fresh per-test SQLite file. diff --git a/tests/pipeline/test_score.py b/tests/pipeline/test_score.py index 94de527..c72a61b 100644 --- a/tests/pipeline/test_score.py +++ b/tests/pipeline/test_score.py @@ -64,8 +64,9 @@ async def test_score_node_missing_requirements_errors(monkeypatch, temp_vault): async def test_score_node_passes_profile_to_llm(monkeypatch, temp_vault): - """The profile_text passed to _score must include resume + skill-inventory content.""" + """The profile_text passed to _score includes resume + RAG-retrieved skill chunks.""" from compass.pipeline.nodes import score + from compass.rag.retriever import RetrievedChunk captured = {} @@ -75,9 +76,13 @@ async def fake_score(req, profile_text: str) -> JobScore: score=3.0, reasoning="", matched_skills=[], missing_skills=[], tailoring_notes="" ) + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 3.", score=0.9)] + monkeypatch.setattr(score, "_score", fake_score) + monkeypatch.setattr(score, "rag_retrieve", fake_retrieve) req = JobRequirements( - required_skills=[], + required_skills=["Python"], nice_to_have_skills=[], years_experience=None, seniority="mid", @@ -86,7 +91,7 @@ async def fake_score(req, profile_text: str) -> JobScore: ) await score.score_node(_state(req)) assert "Fake resume body" in captured["profile_text"] - assert "Python: 3" in captured["profile_text"] + assert "## Python\nLevel 3." in captured["profile_text"] async def test_score_node_drops_skills_outside_jd_universe(monkeypatch, temp_vault): diff --git a/tests/pipeline/test_score_node_rag.py b/tests/pipeline/test_score_node_rag.py new file mode 100644 index 0000000..4b18db0 --- /dev/null +++ b/tests/pipeline/test_score_node_rag.py @@ -0,0 +1,87 @@ +"""score_node uses RAG retrieval to build profile context, not full inventory.""" + +from __future__ import annotations + +import pytest + +from compass.pipeline.state import JobRequirements + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +def _stub_req(**overrides) -> JobRequirements: + base = dict( + required_skills=["Python", "LangGraph"], + nice_to_have_skills=["MCP"], + seniority="senior", + remote_policy="remote", + summary="Build agentic systems.", + ) + base.update(overrides) + return JobRequirements(**base) + + +async def test_profile_text_includes_resume_and_retrieved_chunks(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME TEXT") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME TEXT" in text + assert "## Python\nLevel 4." in text + + +async def test_retrieval_query_carries_jd_skills_and_summary(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + captured: dict[str, object] = {} + + async def fake_retrieve(query, k=8): + captured["query"] = query + return [RetrievedChunk(skill="Python", document="## Python", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "") + + await score_mod._profile_text(_stub_req()) + q = captured["query"] + assert "Python" in q and "LangGraph" in q and "MCP" in q + assert "Build agentic systems" in q + + +async def test_score_node_handles_empty_jd_skills(monkeypatch): + """JD with no required/nice-to-have skills falls back to summary-only query.""" + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME") + + text = await score_mod._profile_text( + _stub_req(required_skills=[], nice_to_have_skills=[], summary="Just a summary.") + ) + assert "RESUME" in text + assert "## Python\nLevel 4." in text + + +async def test_score_node_handles_empty_retrieval(monkeypatch): + """If retriever returns [], profile context still includes the resume.""" + from compass.pipeline.nodes import score as score_mod + + async def fake_retrieve(query, k=8): + return [] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME ONLY") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME ONLY" in text diff --git a/tests/rag/conftest.py b/tests/rag/conftest.py index 18e9300..e263540 100644 --- a/tests/rag/conftest.py +++ b/tests/rag/conftest.py @@ -38,11 +38,3 @@ def tiny_inventory(temp_vault) -> Path: encoding="utf-8", ) return inv - - -@pytest.fixture(scope="session") -def embedding_model_cached(): - """Pre-load sentence-transformers once per test session. ~90MB on first run.""" - from sentence_transformers import SentenceTransformer - - SentenceTransformer("all-MiniLM-L6-v2") From 6f9a879c61110ec13281ef9d0c5ca651da5dd8ac Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 03:06:37 -0500 Subject: [PATCH 19/46] docs(score): refresh module docstring for RAG retrieval --- compass/pipeline/nodes/score.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compass/pipeline/nodes/score.py b/compass/pipeline/nodes/score.py index 5082599..b2f1bb5 100644 --- a/compass/pipeline/nodes/score.py +++ b/compass/pipeline/nodes/score.py @@ -1,8 +1,9 @@ """ score_node — score a job against the candidate profile. -Reads resume.md + skill-inventory.md from the vault and passes them as context -to the LLM. Returns a JobScore (0.0–5.0) with matched/missing/tailoring breakdown. +Profile context is the full resume plus top-k chunks retrieved from the +skill-inventory Chroma index, queried with the JD's skills + summary. +Returns a JobScore (0.0–5.0) with matched/missing/tailoring breakdown. Model: SCORE_MODEL (default google/gemini-2.5-flash). """ From c4c7fe8ae7fa07b757b510a81fe2d6f4911b7a6d Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 11:29:44 -0500 Subject: [PATCH 20/46] fix(tests): use monkeypatch.setattr for ss._now to prevent test isolation leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw 'ss._now = lambda: ...' assignments in test_state_store.py bypassed monkeypatch's tracking, leaking the frozen time into subsequent tests in the suite. Caused timeout_checker tests to see a stale _now() value and mis-classify backdated rows as not-yet-stale. Pre-existing bug from Phase 1.B.1 — passed the full suite by ordering luck; exposed when 1.B.2's new tests shifted the test interaction. --- tests/hitl/test_state_store.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/hitl/test_state_store.py b/tests/hitl/test_state_store.py index b7dc47a..355db43 100644 --- a/tests/hitl/test_state_store.py +++ b/tests/hitl/test_state_store.py @@ -59,15 +59,14 @@ async def test_list_pending_only_returns_pending_status(): assert [r["thread_id"] for r in rows] == ["tid-pending"] -async def test_list_pending_orders_oldest_first(): +async def test_list_pending_orders_oldest_first(monkeypatch): """The MCP UI shows the queue oldest-first so things don't get lost.""" import compass.hitl.state_store as ss - # First insertion at frozen_now fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) - ss._now = lambda: fixed + monkeypatch.setattr(ss, "_now", lambda: fixed) await _add_one(thread_id="tid-old") - ss._now = lambda: fixed + _dt.timedelta(minutes=5) + monkeypatch.setattr(ss, "_now", lambda: fixed + _dt.timedelta(minutes=5)) await _add_one(thread_id="tid-new") rows = await state_store.list_pending() assert [r["thread_id"] for r in rows] == ["tid-old", "tid-new"] @@ -93,17 +92,16 @@ async def test_mark_resolved_unknown_thread_raises(): await state_store.mark_resolved("tid-missing", status="approved") -async def test_list_timed_out_returns_only_old_pending(frozen_now): +async def test_list_timed_out_returns_only_old_pending(frozen_now, monkeypatch): import compass.hitl.state_store as ss - # Insert one row "5 hours ago", one "1 hour ago" old = frozen_now - _dt.timedelta(hours=5) young = frozen_now - _dt.timedelta(hours=1) - ss._now = lambda: old + monkeypatch.setattr(ss, "_now", lambda: old) await _add_one(thread_id="tid-old") - ss._now = lambda: young + monkeypatch.setattr(ss, "_now", lambda: young) await _add_one(thread_id="tid-young") - ss._now = lambda: frozen_now + monkeypatch.setattr(ss, "_now", lambda: frozen_now) rows = await state_store.list_timed_out(timeout_hours=4) assert [r["thread_id"] for r in rows] == ["tid-old"] From 1d663f75a8b1af92b2356e4bf930f2879b6d5600 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 11:29:44 -0500 Subject: [PATCH 21/46] style(scripts): apply ruff format to clean pre-existing drift --- scripts/seed_vault.py | 1 + scripts/test_scrape.py | 1 + scripts/test_vault_roundtrip.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/seed_vault.py b/scripts/seed_vault.py index cd37467..b0e9cfc 100644 --- a/scripts/seed_vault.py +++ b/scripts/seed_vault.py @@ -6,6 +6,7 @@ Creates all required folders and placeholder files. Does NOT overwrite existing files. """ + import sys from pathlib import Path diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py index 1068792..87c71a6 100644 --- a/scripts/test_scrape.py +++ b/scripts/test_scrape.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/test_scrape.py """ + from __future__ import annotations import asyncio diff --git a/scripts/test_vault_roundtrip.py b/scripts/test_vault_roundtrip.py index 977f378..165882d 100644 --- a/scripts/test_vault_roundtrip.py +++ b/scripts/test_vault_roundtrip.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/test_vault_roundtrip.py """ + from __future__ import annotations import sys @@ -48,7 +49,9 @@ def main() -> int: job_path = write_job_note(note) print(f" OK wrote job note: {job_path.name}") - company_path = write_company_note(CompanyNote(company=SENTINEL_COMPANY, tier="unknown", roles_seen=1)) + company_path = write_company_note( + CompanyNote(company=SENTINEL_COMPANY, tier="unknown", roles_seen=1) + ) print(f" OK wrote company note: {company_path.name}") skill_path = update_skill_note("Python", SENTINEL_URL) From 3828d8feef0350dc4670b2022e8c838840880de7 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 12:32:36 -0500 Subject: [PATCH 22/46] fix(intake): tighten OUT keyword list + document deferred LLM-behavior bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live behavior review surfaced 4 false-negatives at intake_filter — added: - engineering manager / lead / director / head / vp of engineering - solutions engineer (the bare title — 'sales engineer' already caught) - security engineer / application security / infrastructure security - operations specialist / product operations / program manager 'Solutions architect' intentionally NOT added — borderline per Phase 1.A spec. Three deeper LLM-behavior bugs (extract under-extraction, OR-as-AND, and candidate-strong-skill-marked-missing) documented in docs/KNOWN_DATA_QUALITY_ISSUES.md for Phase 2.A eval-harness work. --- compass/pipeline/role_family.py | 23 ++++++- docs/KNOWN_DATA_QUALITY_ISSUES.md | 104 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 docs/KNOWN_DATA_QUALITY_ISSUES.md diff --git a/compass/pipeline/role_family.py b/compass/pipeline/role_family.py index 7fac4a3..079d38d 100644 --- a/compass/pipeline/role_family.py +++ b/compass/pipeline/role_family.py @@ -82,17 +82,36 @@ # pre-sales / solutions "presales", "pre-sales", + "solutions engineer", + "solution engineer", + # NOTE: "solutions architect" intentionally NOT in this list — that title + # straddles pre-sales and hands-on infra depending on company; the LLM + # body-check stage decides per-JD. # CS "customer success", "customer experience", "customer support", "technical csm", - # PM + # PM / product ops "product manager", "product management", "group pm", "agent pm", - # design (note: "designer", "motion graphics", etc. — substring is safe because they're distinctive) + "product operations", + "operations specialist", + "program manager", + # management track (Akash isn't pursuing management — per role-clarifications) + "engineering manager", + "engineering lead", + "director of engineering", + "head of engineering", + "vp of engineering", + "vp engineering", + # security (separate engineering discipline, not agentic-AI) + "security engineer", + "application security", + "infrastructure security", + # design (note: "designer" is distinctive enough for substring match) "designer", "motion graphics", "web designer", diff --git a/docs/KNOWN_DATA_QUALITY_ISSUES.md b/docs/KNOWN_DATA_QUALITY_ISSUES.md new file mode 100644 index 0000000..fd790e3 --- /dev/null +++ b/docs/KNOWN_DATA_QUALITY_ISSUES.md @@ -0,0 +1,104 @@ +# Known Data-Quality Issues — Pipeline Behavior + +> Surfaced during the Phase 1.B.2 deep adversarial review against real vault data. These are pre-existing Phase 0.B / 1.A LLM-behavior defects that distort scoring + skill matching. Fix scope: **Phase 2.A's eval harness work** — each needs a regression test against labeled JDs before prompt tuning, otherwise we whack-a-mole. + +**Tag context:** discovered at `phase-1b2-rag`. Vault snapshot: 23 JobNotes across Anthropic / Decagon / Cognition / Sierra / Ramp. + +--- + +## B1. `extract_node` under-extracts `skills_required` on best-fit roles + +**Symptom:** roles that are obvious matches for the candidate end up with empty or near-empty `skills_required` lists, which artificially caps the score because the scorer has nothing to match against. + +**Evidence:** +- `jobs/2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f.md` → `skills_required: []`, `skills_matched: []`, `match_score: 3.0`. The JD describes building an agent SDK + runtime + evals — a near-perfect fit for the candidate's MCP/agent cluster (level 3-4). The score reasoning even acknowledges the alignment, but the score caps at 3.0 because there are no required-skill entries to flip into matched. **A fair human grade is ~4.0.** +- `jobs/2026-05-06-cognition-Software_Engineer-7c26fdaf.md` → `skills_required: [Python]` only, despite the JD emphasizing sub-agent orchestration, tool use, agent infrastructure. + +**Hypothesis:** the extract prompt is being too literal about "explicit requirements only" and missing skills implied by responsibility statements. JDs at agentic startups frequently lead with mission text ("build the orchestration engine"), with the canonical skill list buried or implied. + +**Phase 2.A fix surface:** +- Tighten extract prompt to also pull canonical skills from responsibility statements (`build orchestration engine` → LangGraph, `subagent coordination` → Multi-Agent). +- Add a labeled JD eval where the ground-truth skill list includes implied skills; assert extract recall ≥ 0.7. + +--- + +## B2. `extract_node` mis-reads OR-lists as AND-lists + +**Symptom:** "languages such as A, B, C, or similar" is interpreted as four ANDed requirements. Candidate gets dinged on missing alternatives the JD didn't actually require. + +**Evidence:** +- `jobs/2026-04-15-cognition-AI_Support_Engineer-25133cbd.md` → JD: "Strong ability to read and reason about code in **multiple languages, such as Python, TypeScript, Java, Go, or similar**". Extracted: `skills_required: [Python, TypeScript, Go, Docker]`. Scorer marked TS+Go as missing → 2.0 → auto-rejected. **A fair human assigns ~3.0-3.5.** The role was incorrectly auto-rejected by extraction artefact. + +**Phase 2.A fix surface:** +- Extract prompt addendum: explicit OR-list detection ("languages such as X, Y, or Z" → store as `nice_to_have_skills` or a separate `language_alternatives` field, not as `skills_required`). +- Eval coverage: at least 3 OR-list JDs in the labeled set with ground-truth scores. + +--- + +## B3. `score_node` occasionally marks candidate-strong skills as missing + +**Symptom:** scorer LLM lists a skill in `missing_skills` despite the candidate having a non-trivial level in the profile. + +**Evidence:** +- `jobs/2026-04-07-ramp-Security_Engineer_Cloud-a25113ae.md` → `skills_required: [Python]`, `skills_missing: [Python]`, `skills_matched: []`. The candidate has Python at level 3 (skill-inventory.md). RAG would have surfaced the Python chunk on any query containing "Python". The scorer LLM didn't consult the retrieved profile faithfully. +- Low-fit branch failure mode: when the overall role is wildly off (security, in this case), the scorer appears to short-circuit and declare every required skill missing without verifying the candidate's level. + +**Phase 2.A fix surface:** +- Score prompt assertion: "Before adding a skill to missing_skills, verify it is NOT in the candidate's profile at level ≥ 2." +- Defensive post-filter: in `score.py`, augment `_constrain_to_jd_skills` to also check `skills_missing` against the candidate's profile and downgrade false-misses to nothing. (Requires access to canonical skill levels at score time — currently only the chunks are passed.) +- Eval regression test: "if candidate has skill X at level ≥ 3 AND skill X is in `skills_required`, then X MUST appear in `skills_matched`, never in `skills_missing`." + +--- + +## B4. ✅ FIXED in `phase-1b2-rag` — `intake_filter` title-keyword stage was leaky + +Resolved at this tag: added `engineering manager`, `engineering lead`, `director/head/vp of engineering`, `solutions engineer`, `solution engineer`, `security engineer`, `application security`, `infrastructure security`, `operations specialist`, `product operations`, `program manager` to `OUT_SUBSTRING_KEYWORDS`. The four false-negatives observed (Decagon EM, Ramp AI Ops Specialist, Decagon Senior Solutions Eng, Sierra Security Eng) now correctly drop at intake. + +--- + +## B5. Anti-claim gating is inconsistent on post-training/research roles + +**Symptom:** the candidate's `role-clarifications.md` declares SFT/LoRA/RLHF/DPO as level 0 (explicit anti-claim). Sometimes the scorer catches this, sometimes it doesn't. + +**Evidence:** +- Cognition Research Post-Training role → correctly scored 0.0, reasoning cited RLHF=0. +- `jobs/2025-08-22-decagon-Senior_Research_Engineer-5030d3e2.md` → JD requires "Prior experience post-training and deploying LLMs in production". Scored 3.0. The disqualifier was missed. + +**Phase 2.A fix surface:** +- The anti-claim signal should be a hard rule, not a contextual hint. Add an explicit pre-score check: if JD `required_skills` contains any of {fine-tuning, RLHF, post-training, SFT, LoRA, DPO} AND candidate role-clarification anti-claims include them, then score is capped at 1.0 regardless of LLM output. + +--- + +## What's working correctly (positive findings) + +These were spot-checked and are NOT a problem: + +- **`jd_summary`** extraction is faithful across 8 spot-checks. No hallucinated facts. ATS boilerplate correctly excluded. +- **`_constrain_to_jd_skills`** filter (Phase 0 fix #5) is intact. Would catch over-extraction; current bug class is under-extraction. +- **LLM-stage `intake_filter`** is well-calibrated. Catches Japanese pre-sales architect, fellowship roles, GTM partnerships, HR coordinator — all correct. No false positives observed. +- **Pre-RAG vs post-RAG (Phase 1.B.1 vs 1.B.2)** comparison shows no quality regression. Top-k=8 chunks produce score reasoning at comparable depth to the prior full-inventory inject. +- **Audit-trail integrity** (`hitl_decision`, `hitl_at`, `tailored_paragraph`, `score_threshold`) is correct across the 23 JobNotes inspected. + +--- + +## Why these are deferred to Phase 2.A + +The spec's Phase 2.A is the **eval harness** phase: 30+ hand-labeled JDs with ground-truth scores + skill lists, run nightly, alert on MAE drift. That's the right place to attack B1/B2/B3/B5 because: + +1. Each bug class needs a regression test, not a one-off prompt tweak. +2. Prompt tuning without measurement is whack-a-mole — fixing B1 might break B2 silently. +3. The labeled dataset doubles as the portfolio artifact ("eval-driven LLM development"). + +Phase 1.B.2 closed the architectural RAG portfolio claim. Phase 1.B.3 closes the cron+observability claim. Phase 2.A makes the LLM behavior measurably correct. + +--- + +## Stop-gap mitigation pre-2.A + +If a particular roll-out is needed before the eval harness: + +1. **B3 has the highest cost-to-value for a manual fix** — adding a one-line "verify candidate level ≥ 2 before declaring missing" assertion in the score prompt is low-risk. +2. **B2 has high impact when it fires** but is rare. Manual JD-by-JD review until 2.A is cheap (only ~20 jobs/week reach scoring). +3. **B1 affects every best-fit JD.** This is the single most consequential prompt-tuning target. + +**Recommendation:** do not attempt B1-B3 ad-hoc. Build 2.A first. From 7ebd4a11744d645886e36f6d36c7753a4fd41a76 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 12:37:41 -0500 Subject: [PATCH 23/46] =?UTF-8?q?docs:=20expand=20data-quality=20issues=20?= =?UTF-8?q?=E2=80=94=20derived-field=20staleness,=20gap=20filter,=20tailor?= =?UTF-8?q?=20hallucination,=20portfolio-claim=20risks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second adversarial pass surfaced two new bug classes (derived-field staleness asymmetry on role_family; gap_aggregator includes auto_rejected/ timed_out jobs at full weight) plus three operational portfolio-claim risks (skill assessor never exercised, applications never created, concurrent runs not protected). --- docs/KNOWN_DATA_QUALITY_ISSUES.md | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/docs/KNOWN_DATA_QUALITY_ISSUES.md b/docs/KNOWN_DATA_QUALITY_ISSUES.md index fd790e3..90d6526 100644 --- a/docs/KNOWN_DATA_QUALITY_ISSUES.md +++ b/docs/KNOWN_DATA_QUALITY_ISSUES.md @@ -69,6 +69,81 @@ Resolved at this tag: added `engineering manager`, `engineering lead`, `director --- +--- + +## B6. Derived-field staleness asymmetry — `role_family` is set once and never reconciled + +**Symptom:** when the keyword OUT list expands (as in commit `3828d8f`), existing JobNotes that should now be classified out-of-scope keep their old `role_family`. `gap_aggregator.regenerate()` reads ALL JobNotes regardless of `role_family`, so stale entries keep contributing to the master gap plan. + +**Evidence:** +- `jobs/2026-03-17-ramp-AI_Operations_Specialist_Agentic_Workflows-f489c32b.md` has `role_family: agent-engineer` but `keyword_classify()` now correctly returns `out-of-scope` for that title. The JobNote was written before "operations specialist" was in the OUT list. Its skills still feed gap-plan weights. + +**Pattern:** counters (`appears_in_jobs`, `roles_seen`) are derived at every gap_aggregator run — Phase 0 bug #12 / Phase 1.A bug #1 enforced this discipline. But `role_family` is treated as ground truth once written. **Same asymmetry could repeat for any future "set-once classification" field** (e.g. `tier`, `seniority`). + +**Phase 2.A fix surface:** +- Either: `gap_aggregator.load_jobs()` re-runs `keyword_classify` on each JobNote and skips out-of-scope. +- Or: a one-time migration script that re-classifies and updates `role_family` in place (similar to the Phase 1.A `cleanup_stale_jobnotes.py` pattern). +- Either way: add a regression test that asserts "no JobNote in the vault has a `role_family` that current code would classify as `out-of-scope`." + +--- + +## B7. `gap_aggregator` includes `auto_rejected` / `timed_out` / `null hitl_decision` jobs at full weight + +**Symptom:** the gap plan is computed from all 23 JobNotes — currently 1 approved, 10 auto_rejected, 8 null (pre-1.B.1), 4 timed_out — with no `hitl_decision` filter. A `timed_out` job with score 4.0 contributes the same gap signal as an `approved` job. + +**Trade-off:** the original spec rationale was "JD-market signal is independent of personal action" — even rejected jobs reveal what skills the market wants. Defensible. But it conflates: +- "Skills the market demands generally" (all in-scope JDs) +- "Skills I should study to convert near-misses to applies" (high-score auto_rejected = stretch signals) +- "Skills the human is committed to pursuing" (approved only) + +**Phase 2.A fix surface:** +- Add an optional filter mode to `gap_aggregator.regenerate(filter_decision=None | "approved_or_above_threshold")`. +- Document the chosen weighting decision in `_profile/preferences.md` so the user can tune. + +--- + +## B8. `tailor_node` has no programmatic constraint against hallucination + +**Symptom:** unlike `score_node`'s `_constrain_to_jd_skills`, `tailor_node`'s prompt says "Mention real projects and concrete numbers when the profile provides them" but enforces nothing. The agent could invent project names and they'd ship to the JobNote. + +**Evidence:** the one tailored paragraph in the vault (Cognition Special Projects) was spot-checked — every concrete claim verified against `_profile/resume.md`. **No hallucination in this sample**, but n=1 with no code-level defense. + +**Risk:** Sonnet on a sparse JD ("Founder mindset, 2+ years exp") with a longer profile could easily invent specifics. + +**Phase 2.A fix surface:** +- Post-generation regex pass that flags proper-noun project mentions not present in `resume.md` (similar to extract's JD-substring validation in Phase 0). +- Eval coverage: hand-label 5 generated tailoring paragraphs against ground-truth resume facts; assert zero unsupported claims. + +--- + +## Portfolio-claim risks (operational, not code bugs) + +These are NOT data-quality bugs but portfolio narrative risks: the spec advertises features whose code is built but real data is empty. + +### PR1. Skill assessor loop is theoretical — zero evidence URIs + +`grep '^evidence:' ~/Documents/compass-vault/skills/*.md` returns 95/95 `evidence: []`. The "unique angle" — the agent that grades candidate skills against `learning-vault://` evidence — has never operated on real input. The MCP `assess_skills` tool returns immediately with no work. + +**The README narrative says:** "an agent inside it that grades my skills against the live job market and tells me what to study next." Currently it can't, because no evidence is wired up. The infrastructure exists; the demo isn't recorded. + +**Mitigation paths:** +- Manually wire 3-5 `learning-vault://` URIs to a sample of skills (e.g. MCP, LangGraph, RAG) — proves the loop end-to-end. +- Record one assessor run with grade-change output as a portfolio screenshot. + +### PR2. Application lifecycle has never been used + +`~/Documents/compass-vault/applications/` is empty. All 23 JobNotes have `applied_at: null`. The Dashboard panels "In-flight applications" and "Today's next actions" will both always be empty until someone runs `add_application(job_id)`. + +**Mitigation:** apply to one job for real, exercise the full lifecycle (`add_application` → `update_application_status` → next-action reminder). Provides a portfolio screenshot. + +### PR3. Concurrent pipeline runs not protected + +Two parallel `run_pipeline()` invocations share `HITL_CHECKPOINT_DB`. SQLite raises `database is locked` and one process may leave a thread without a saved checkpoint. Also two parallel `gap_aggregator.regenerate()` calls race on `MASTER_GAP_PLAN_PATH.write_text()` — atomic per call but last-writer-wins across processes. + +**Phase 1.B.3 fix surface:** Modal cron + human-MCP race is the real-world trigger. Use SQLite advisory lock or `flock` on a sentinel file. + +--- + ## What's working correctly (positive findings) These were spot-checked and are NOT a problem: From db3b1322cb44ab131f3159ffac3d09d52ec47363 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 12:48:32 -0500 Subject: [PATCH 24/46] fix(gap): skip out-of-scope JobNotes; one-time role_family migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B6 from KNOWN_DATA_QUALITY_ISSUES.md: role_family is set once at intake; when the OUT keyword list expanded (commit 3828d8f), stale JobNotes kept contributing to the gap plan. Two-part fix: 1. scripts/migrate_role_family.py — one-time migration. Re-runs keyword_classify on each JobNote title; rewrites role_family to 'out-of-scope' for titles the current classifier rejects. In-scope re-classifications (e.g. body-signal upgrades) are left alone since this script doesn't re-evaluate the JD body. 2. gap_aggregator.load_jobs() now skips JobNotes with role_family= 'out-of-scope'. Without this, the migration alone would only rename the field but the gap math would still include them. Migrated 5 stale entries (Sierra/Ramp Security Engineer x2, Decagon Senior Solutions Engineer, Decagon Engineering Manager, Ramp AI Operations Specialist) — all were leaking through the pre-3828d8f filter. --- compass/analysis/gap_aggregator.py | 5 ++ scripts/migrate_role_family.py | 89 +++++++++++++++++++ tests/analysis/__init__.py | 0 .../test_gap_aggregator_role_filter.py | 39 ++++++++ 4 files changed, 133 insertions(+) create mode 100644 scripts/migrate_role_family.py create mode 100644 tests/analysis/__init__.py create mode 100644 tests/analysis/test_gap_aggregator_role_filter.py diff --git a/compass/analysis/gap_aggregator.py b/compass/analysis/gap_aggregator.py index 48a3902..193c960 100644 --- a/compass/analysis/gap_aggregator.py +++ b/compass/analysis/gap_aggregator.py @@ -68,6 +68,11 @@ def load_jobs() -> list[JobSummary]: fm = _parse_frontmatter(f) if not fm: continue + # Skip out-of-scope JobNotes — they shouldn't influence the gap plan. + # role_family is set once at intake; if a stale entry's title would + # now classify as out-of-scope, the migration script handles that. + if fm.get("role_family") == "out-of-scope": + continue out.append( JobSummary( file=f, diff --git a/scripts/migrate_role_family.py b/scripts/migrate_role_family.py new file mode 100644 index 0000000..6840b1a --- /dev/null +++ b/scripts/migrate_role_family.py @@ -0,0 +1,89 @@ +"""One-time migration: re-classify existing JobNote role_family fields. + +`role_family` is stored once at intake and never reconciled. When the +intake_filter OUT keyword list expands (e.g. commit 3828d8f added +"engineering manager", "solutions engineer", "security engineer", +"operations specialist"), existing JobNotes keep their old role_family +and continue to feed the gap plan. This script walks every JobNote +and rewrites role_family to whatever current `keyword_classify` decides. + +Behavior: +- title resolves to out-of-scope → set role_family="out-of-scope" +- title resolves to a specific family → set role_family=that family +- title is ambiguous (None — LLM stage decides) → leave role_family alone + +Dry-run by default; --apply to commit. + +Usage: + uv run python -m scripts.migrate_role_family # dry-run + uv run python -m scripts.migrate_role_family --apply # commit changes +""" + +from __future__ import annotations + +import argparse +import sys +from typing import TYPE_CHECKING + +import frontmatter + +from compass.config import VAULT_PATH +from compass.pipeline.role_family import keyword_classify + +if TYPE_CHECKING: + from pathlib import Path + + +def find_changes() -> list[tuple[Path, str, str]]: + """Return [(file, old, new), ...] for JobNotes whose title now classifies as + out-of-scope. + + Only out-of-scope transitions are migrated. In-scope re-classifications + (e.g. keyword says swe-fullstack but body-signal upgrader originally set + agent-engineer) are left alone — the original write had access to the JD + body that this script doesn't re-evaluate. + """ + changes: list[tuple[Path, str, str]] = [] + for path in sorted((VAULT_PATH / "jobs").glob("*.md")): + post = frontmatter.load(path) + title = post.metadata.get("title", "") + old = post.metadata.get("role_family", "") + in_scope, new = keyword_classify(title) + if in_scope is False and old != "out-of-scope": + changes.append((path, old, new)) + return changes + + +def apply_changes(changes: list[tuple[Path, str, str]]) -> None: + for path, _, new in changes: + post = frontmatter.load(path) + post.metadata["role_family"] = new + path.write_text(frontmatter.dumps(post), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Actually rewrite the JobNotes") + args = parser.parse_args() + + changes = find_changes() + if not changes: + print("No JobNotes need role_family migration.") + return 0 + + print(f"Found {len(changes)} JobNote(s) whose role_family disagrees with current classifier:\n") + for path, old, new in changes: + print(f" {path.name}") + print(f" {old!r} -> {new!r}") + + if not args.apply: + print(f"\nDry-run. Pass --apply to rewrite {len(changes)} files.") + return 0 + + apply_changes(changes) + print(f"\nMigrated {len(changes)} JobNote(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/analysis/__init__.py b/tests/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/analysis/test_gap_aggregator_role_filter.py b/tests/analysis/test_gap_aggregator_role_filter.py new file mode 100644 index 0000000..3d4dec7 --- /dev/null +++ b/tests/analysis/test_gap_aggregator_role_filter.py @@ -0,0 +1,39 @@ +"""Regression test for B6 fix: out-of-scope JobNotes must not contribute to gap plan.""" +from __future__ import annotations +from pathlib import Path +import frontmatter +import pytest + + +def _write_job(vault: Path, name: str, role_family: str, required: list[str], score: float) -> None: + (vault / "jobs").mkdir(parents=True, exist_ok=True) + post = frontmatter.Post( + "# Test job\n", + company="Test", + title=f"Test Title {name}", + url=f"https://x/{name}", + source="manual", + date_found="2026-05-19", + match_score=score, + score_reasoning="t", + seniority="mid", + role_family=role_family, + tier="apply-now", + skills_required=required, + skills_nice_to_have=[], + skills_matched=[], + skills_missing=required, + jd_summary="t", + ) + (vault / "jobs" / f"{name}.md").write_text(frontmatter.dumps(post), encoding="utf-8") + + +def test_gap_aggregator_skips_out_of_scope_jobs(temp_vault): + from compass.analysis import gap_aggregator + + _write_job(temp_vault, "inscope", "agent-engineer", ["Python"], score=4.0) + _write_job(temp_vault, "outofscope", "out-of-scope", ["TypeScript"], score=4.0) + + jobs = gap_aggregator.load_jobs() + titles = sorted(j.title for j in jobs) + assert titles == ["Test Title inscope"] # out-of-scope job excluded From 5b39d58b1750ceea3e17d0e19055edec977e877e Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 12:56:35 -0500 Subject: [PATCH 25/46] docs: comprehensive project overview from phase 0 through 1.b.2 --- docs/PROJECT_OVERVIEW.md | 510 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 docs/PROJECT_OVERVIEW.md diff --git a/docs/PROJECT_OVERVIEW.md b/docs/PROJECT_OVERVIEW.md new file mode 100644 index 0000000..819f955 --- /dev/null +++ b/docs/PROJECT_OVERVIEW.md @@ -0,0 +1,510 @@ +# Compass — Complete Project Overview + +> A plain-language walkthrough of what Compass is, how it works, what we've built phase by phase, and why each design decision was made. Read this when you want to see the whole project in one place. + +**Current tag:** `phase-1b2-rag` · **Tests:** 259 passing · **Branch lineage:** `phase-0a-foundation` → `phase-0b-pipeline-mvp` → `phase-1a-application-tracking` → `phase-1b1-hitl` → `phase-1b2-rag` + +--- + +## 1. What is Compass? + +### The problem + +Job search at the senior-engineer level is two simultaneous problems: + +1. **Application work** — find roles you fit, decide which ones to apply to, tailor your story for each +2. **Skill-gap work** — figure out what the JD market is actually asking for that you don't yet have, and study toward those gaps + +Most job-search tools solve only the first problem. They surface roles. They don't help you decide what to study to be a stronger candidate next quarter. + +### The solution + +Compass is a tool that does both at once. It: + +- Scrapes job boards (Greenhouse, Lever, Ashby) +- Drops the noise (sales, PM, design roles) +- Scores each engineering role against your profile (0.0 - 5.0) +- Identifies skills the JDs want that you don't yet have, ranked by demand +- Pauses the high-scoring ones so you (the human) can approve before any LLM-generated tailoring is wasted +- Writes everything to an Obsidian markdown vault so you can read/edit it like a regular notes app +- Has an **agent inside it that grades your own skills against evidence you provide**, then re-prioritizes the gap plan based on which skills moved + +It's a research-and-preparation tool, not an auto-apply bot. The human is in the loop for every application decision. + +### The three "sides" of the system + +Compass has three storage layers, each with a different owner: + +``` +┌─────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐ +│ THE PIPELINE │ │ PRODUCT VAULT │ │ LEARNING VAULT │ +│ (the code) │─────▶│ agent-written │◀─────│ human-written │ +│ │ │ markdown │ │ markdown │ +│ scrapes JDs │ │ │ │ │ +│ scores them │ │ jobs/*.md │ │ projects/*/*.md │ +│ writes results │ │ skills/*.md │ │ canon/*.md │ +│ runs gap analysis │ │ applications/*.md │ │ daily/*.md │ +│ │ │ companies/*.md │ │ roadmap/*.md │ +└─────────────────────┘ │ dashboard.md │ └──────────────────────┘ + └──────────────────────┘ + ~/Documents/compass-vault ~/Documents/learning-vault +``` + +**Why three?** Separation of concerns. The agent owns what it writes (jobs, skills, scores) and never touches your personal notes. Your learning vault is private; the agent only *reads* it (via `learning-vault://` URIs) to find evidence that you've actually done the work you claim. + +This is the system's "unique angle": the agent doesn't take your word for what you know. It grades your skills against the evidence in your learning vault, using a deliberately skeptical hiring-manager persona. + +--- + +## 2. The Big Picture: How a Job Flows Through Compass + +A single job posting goes through 8 steps. Each step has a Python module in `compass/pipeline/nodes/`. The whole thing is a LangGraph state machine — meaning each step writes to a shared `state` dictionary, and the graph routes between steps based on what's in that state. + +``` + ┌──────────────────────────────────────────────────────┐ + │ │ + raw JD ───▶ intake ──▶ intake_filter ─out-of-scope──▶ filtered-jobs.md + │ + │ in-scope + ▼ + extract ──▶ score ──▶ reflect + │ + ▼ + hitl + ┌────────┼─────────┐ + below-thresh above-thresh + │ (PAUSE here) + │ ┌─────────────────┐ + │ │ human approves │ + │ │ via MCP tool │ + │ └─────────────────┘ + │ │ + ▼ ▼ + vault_write ◀──── tailor + │ + ▼ + JobNote.md +``` + +Let me walk through each step. + +### Step 1: `intake_node` + +Just packages the raw job data into the shared state. No LLM call. No skill needed. This is the entry point. + +### Step 2: `intake_filter_node` + +The first cost-control gate. Decides if this JD is even worth showing to the LLM. Three stages in order: + +1. **Title keyword filter** — if the title contains `"account executive"`, `"product manager"`, `"sales engineer"`, etc., it's out-of-scope. Cheap, no LLM call. +2. **Body-signal upgrader** — if the title is something generic like "Software Engineer" but the JD body mentions "agent", "MCP", "LangGraph" enough times, promote the role to `agent-engineer`. +3. **LLM fallback** — for genuinely borderline titles (e.g. "Solutions Architect"), call Gemini Flash once to read the first 500 chars of the JD and decide. + +Out-of-scope JDs get logged to `_meta/filtered-jobs.md` and short-circuit to END. They never reach the score step. + +**Why this matters:** without intake_filter, every JD costs ~$0.003 for extract + ~$0.003 for score, even the sales JDs. With it, ~70% of JDs drop here for free. + +### Step 3: `extract_node` + +Calls Gemini 2.5 Flash with a structured-output schema (Pydantic AI). The LLM reads the JD and produces: + +- `required_skills: list[str]` — canonical skill names from the JD +- `nice_to_have_skills: list[str]` +- `seniority: "junior" | "mid" | "senior" | "staff" | "unknown"` +- `years_experience: int | None` +- `remote_policy: "remote" | "hybrid" | "onsite" | "unknown"` +- `summary: str` — a 2-3 sentence summary of the role + +The extracted skills are normalized through the canonical taxonomy (`compass/vault/taxonomy.py`) — so JD-mentions like "Pydantic" and "pydantic-ai" both map to `Pydantic AI`. + +**Defense in depth:** the schema doesn't catch the LLM inventing skills that aren't in the JD. So there's a substring-validation step: every extracted canonical (or synonym) must appear in the JD text. Phase 0 caught a case where the LLM read an Anthropic sales JD and invented "Federated Learning" — schema-valid, completely fabricated. The validation step now drops these silently. + +### Step 4: `score_node` + +The other expensive LLM call. Gemini 2.5 Flash again, structured output. It receives: + +- The job requirements from step 3 +- The candidate's profile context — resume + relevant skill-inventory chunks (retrieved via RAG, see Phase 1.B.2 below) + +And returns: + +- `score: float` (0.0 - 5.0) +- `reasoning: str` (2-3 sentences explaining) +- `matched_skills: list[str]` — skills from the JD the candidate has +- `missing_skills: list[str]` — skills from the JD the candidate doesn't have +- `tailoring_notes: str` (only if score ≥ 3.0) + +**Defense in depth:** the LLM is constrained twice — once via prompt ("matched/missing MUST be subset of JD's required + nice-to-have"), once via post-LLM filter (`_constrain_to_jd_skills`). Phase 0 caught the LLM listing every profile skill as "matched" when the JD had empty required_skills. + +### Step 5: `reflect_node` + +A no-op pass-through for now. Reserved for Phase 2.A where we'll have a second LLM read borderline scores and dissent. + +### Step 6: `hitl_node` — Human-in-the-Loop gate + +The interesting one. Three branches: + +- `score < SCORE_THRESHOLD` (default 3.5) → **auto-reject** with no LLM call. Sets `human_approved=False`. The JD still gets written to the vault (so the gap aggregator sees it), but the expensive Sonnet tailoring step is skipped. +- `score >= SCORE_THRESHOLD` → **`interrupt()`**. LangGraph pauses the graph mid-execution, saves a checkpoint to a SQLite DB, and returns control to the caller. The thread sits idle until a human approves via MCP tool. +- On resume → the LLM's `interrupt()` call returns the human's decision (`{"approved": True/False, "feedback": str}`) and the graph continues to either tailor or vault_write. + +**Why pause instead of always running tailor?** Tailor uses Sonnet (~$0.05/call). Cheap for one job, expensive at 50/day if most of them are mid-fit. Pausing lets the human bulk-approve only the apply-now candidates. + +### Step 7: `tailor_node` + +Only runs if the human approved. Uses Sonnet 4.6 to write a custom paragraph for that JD, referencing the candidate's real projects. The output is recruiter-grade prose, not LLM slop. Example from a real run: + +> "Position yourself as the ideal founder-mindset candidate who has already built and deployed real-world AI agents at scale. Lead with your MCP server portfolio at Cisco where you created natural language interfaces..." + +### Step 8: `vault_write_node` + +Writes the final JobNote markdown file to `~/Documents/compass-vault/jobs/YYYY-MM-DD-Company-Role-.md`. Updates the company's CompanyNote (`companies/Company.md`). Populates the audit trail (`hitl_decision: approved | rejected | auto_rejected | timed_out`, `hitl_at: `). + +After all jobs in the batch finish, `gap_aggregator.regenerate()` reads every JobNote and produces `study-plans/master-gap-plan.md` — the ranked list of "skills the JD market wants that you don't have, ordered by demand × score × tier-weight." + +--- + +## 3. Phase 0: Building the Foundation + +### Phase 0.A — Foundation (`phase-0a-foundation`) + +The bones. No LLM calls yet. + +- 3 ATS scrapers (Greenhouse, Lever, Ashby) — using each ATS's public unauthenticated API +- Vault reader/writer with Pydantic schemas +- Canonical skill taxonomy (95 skills, with synonyms) +- `learning-vault://` URI bridge + +### Phase 0.B — Pipeline MVP (`phase-0b-pipeline-mvp`) + +The LangGraph pipeline. All 7 nodes implemented. Per-node OpenRouter model routing (Gemini Flash for cheap extract/score, Sonnet for tailor). Batch-level URL dedup. Gap aggregator. Master gap plan auto-regeneration. + +### What Phase 0 taught us — the 16 silent bugs + +This phase is where the project's testing discipline took shape. **Every "ready" claim got rolled back by the next adversarial probe.** Sample bugs: + +| # | What broke | Lesson | +|---|---|---| +| 1 | Greenhouse scraper returned empty `content` field | API list endpoint omits content by default — append `?content=true` | +| 4 | LLM invented "Federated Learning" from an Anthropic sales JD | Validate every extracted skill appears in JD text | +| 6 | Taxonomy parser misread 3-column tables (Voice section had no Synonyms column) | Per-section header column detection | +| 7 | Substring fallback false positives: `"Pythonist" → Python`, `"Goblet" → Go` | Drop the substring fallback; strict synonym match only | +| 8 | `React` vs `ReAct` case collision — JDs mentioning React.js silently became the agentic ReAct pattern | Case-sensitive canonicals set | +| 11 | Filename collisions on similar titles (`Engineer/Backend` and `Engineer (Backend)` both became `Engineer_Backend.md` — silent overwrite) | Append 8-char SHA-1 of URL to filename | +| 12 | Skill counter drift — `appears_in_jobs` accumulated on every overwrite, inflated by URL-dedup reruns | Remove counter increments from pipeline; derive from JobNotes at gap-plan time | + +**The pattern:** tests pass, smoke tests pass, code looks correct — but real data is wrong. Schema-valid, semantically meaningless. Only adversarial probing on real outputs catches this class of bug. + +This pattern became the project's most expensive lesson and is repeated through every subsequent phase. + +--- + +## 4. Phase 1.A: Making It Daily-Usable + +### What shipped (`phase-1a-application-tracking`) + +Two big themes: + +**1. Role-family gating + removing the score-write gate.** Phase 0.B had `SCORE_THRESHOLD=3.5` as a vault-write gate — only jobs scoring above that got written. This was the wrong filter. Akash wants the gap plan to include skills from stretch roles (jobs he'd score 2.0 on), because *those gaps are the most informative for study planning*. Filtering on match score hid exactly the right signal. + +The fix: +- **Role-family gate at intake** (the `intake_filter_node` above) drops out-of-scope JDs cheaply +- **All in-scope JDs reach the vault** regardless of score +- The expensive Sonnet tailor step is still gated on score (so cost stays bounded) + +**2. Application lifecycle.** MCP tools so you can mark a job applied and track status transitions: `add_application(job_id)`, `update_application_status(app_id, status, next_action, next_action_date)`, `list_pending_actions()`. + +### The bugs Phase 1.A caught (20 of them) + +Same pattern as Phase 0. A few highlights: + +- **Bug #1**: parallel writes to `companies/Sierra.md` (5 concurrent jobs reading `roles_seen=0`, incrementing to 1, last writer wins) — fixed by deriving `roles_seen` at gap-plan time, never incrementing +- **Bug #2**: invalid manual `tier` value in Obsidian (user typing `tier: applynow` instead of `apply-now`) crashed the next pipeline run — fixed by tolerating invalid Literal values +- **Bug #B**: body-signal upgrader triple-counted `"agent" ⊂ "agentic" ⊂ "agentic ai"` — fixed by longest-first word-bounded matching +- **Bug #F**: `add_application` silently overwrote in-flight ApplicationNotes — fixed with `force=True` default-off + +--- + +## 5. Phase 1.B.1: Real Human-in-the-Loop + +### What shipped (`phase-1b1-hitl`) + +Phase 0/1.A's `hitl_node` was a placeholder — it auto-approved everything above threshold. Phase 1.B.1 made it real: + +- **`langgraph.types.interrupt()`** — when a high-scoring job hits `hitl_node`, the graph pauses. Mid-execution. The Python coroutine returns, but the graph state is checkpointed. +- **`AsyncSqliteSaver`** — the checkpoint goes to `~/.compass/checkpoints.db`. Survives process restarts. +- **`compass/hitl/state_store.py`** — a separate SQLite table tracking which threads are paused, with company/title/score for display. Powers the `pending_approvals()` MCP tool. +- **`compass/hitl/resume.py`** — re-opens the checkpointer, recompiles the graph, drives `graph.ainvoke(Command(resume={"approved": True, "feedback": "LGTM"}))`. The single entry point for both human approves AND timeout auto-cancels. +- **`compass/hitl/timeout_checker.py`** — auto-cancels pending approvals older than `HITL_TIMEOUT_HOURS` (default 4). Phase 1.B.3 will wire this into a Modal cron. +- **MCP tools** — `pending_approvals()` and `approve(thread_id, approved, feedback)` so the human can see and resolve from Claude Code. +- **Audit trail** — `JobNote.hitl_decision` is now a Literal of `approved | rejected | auto_rejected | timed_out`, populated from state by `vault_write_node`. `JobNote.hitl_at` is the timestamp. + +### Why this is load-bearing for the portfolio claim + +The spec says "real LangGraph `interrupt()` + `AsyncSqliteSaver` checkpointing." This is one of the headline differentiators when interviewing for tier-2 agentic-engineering roles. Phase 1.B.1 delivers it as actual working code, not a slideware claim. + +### The bugs adversarial review caught (8 more) + +The plan went through **6 adversarial review iterations** before execution. Found defects included: + +- **C1 (silent data divergence)**: state_store said "approved" but JobNote said "auto_rejected" for the same thread — because the resume process used the default `SCORE_THRESHOLD=3.5` while the original run used `1.5`, so `hitl_node` short-circuited on resume without consuming the `interrupt()` value. Two-part fix: derive resume status from the FINAL graph state, AND make the threshold check sticky by capturing it in state at score time. +- **C2 (run-log column drift)**: pre-1.B.1 `pipeline-runs.md` header was 5-col, new code wrote 6-col rows. Dataview rendered misaligned. Self-healing migration added. +- **C3 (counter drift on resume paths)**: `gap_aggregator.regenerate()` was only called at end of `run_pipeline()`. Resume paths bypassed it. Cognition CompanyNote showed `roles_seen: 3` while 9 cognition JobNotes existed on disk. Fixed by calling regen at end of `resume_pending`. +- **I2 (thread_id collision)**: `_thread_id_for()` hashed `(url, batch_started_at)` — two cross-process pipelines starting at the same microsecond would collide. Added `os.getpid()` to the hash. + +The plan-review pattern: each adversarial pass rolled back the previous "ready" verdict. Six iterations was the diminishing-returns line. + +--- + +## 6. Phase 1.B.2: RAG + Cleanup (current) + +### What shipped (`phase-1b2-rag`) + +Two themes: + +**1. RAG via Chroma.** Previously `score_node._profile_text()` injected the entire `_profile/skill-inventory.md` (~2,500 tokens) into every score prompt. Phase 1.B.2 replaces that with semantic retrieval: + +- `compass/rag/indexer.py` parses `skill-inventory.md` into one chunk per `## SkillName` section, embeds each via `sentence-transformers all-MiniLM-L6-v2`, stores in a Chroma collection pinned to **cosine** distance +- `compass/rag/retriever.py` exposes `retrieve(query, k=8) -> list[RetrievedChunk]` with lazy index init +- `score_node` builds a query string from the JD's `required_skills + nice_to_have + summary` and retrieves the 8 most relevant skill-inventory chunks; injects those instead of the full file +- Token savings: ~2,500 → ~750 per scored JD +- Real-data verification: the same Sierra JobNote that scored 3.0 pre-RAG scored 3.0 post-RAG — no quality regression, just smaller context + +**2. Phase 1.B.1 carryover fixes.** + +- **msgpack deprecation**: every paused thread logged `Deserializing unregistered type compass.pipeline.state.RawJob from checkpoint. This will be blocked in a future version.` LangGraph 1.2 has a `JsonPlusSerializer(allowed_msgpack_modules=[(module, classname), ...])` constructor that suppresses the warning. Took **5 plan-review iterations** to find the right API form — the obvious-looking `with_msgpack_allowlist` method is a no-op when the default `allowed_msgpack_modules=True`. +- **checkpoint DB bloat**: every paused thread accumulates ~10 checkpoint blobs in `~/.compass/checkpoints.db`, never deleted. New `_purge_thread_checkpoints(thread_id)` in `resume.py` runs after `mark_resolved`. Bounded growth. + +### Why RAG instead of just dumping the inventory + +The spec made RAG a portfolio-claim requirement. But the engineering rationale is real: + +- **Cost**: 2,500-token context costs ~3x what a 750-token one does at Gemini Flash rates. Over 50 jobs/day, that's real money. +- **Focus**: full inventory inject includes irrelevant chunks ("Voice — skip"). Retrieved chunks are the skills actually closest to the JD's vocabulary. +- **Demonstrable**: "I built RAG over my profile and reduced score-prompt tokens 70%" is a concrete interview talking point. + +### Why Chroma + all-MiniLM-L6-v2 + +- **Chroma**: simplest persistent vector DB with a real Python API. PersistentClient writes to a SQLite file. No service to run. Cosine distance pinned at collection-create. +- **all-MiniLM-L6-v2**: 90MB model, downloaded once to `~/.cache/torch/`. Fast enough to embed 20 chunks in <100ms on a laptop. Cosine similarity in [0, 1] is meaningful out of the box. +- **Alternative considered**: `nomic-embed-text` (better quality) — deferred to Phase 2.B as a one-line config change. + +### The bugs adversarial review caught (8 more) + +**6 plan iterations + 2 code-quality iterations.** Highlights: + +- **C1 (the silent metric bug)**: existing `~/.compass/chroma/` collections from before the cosine pin would keep the L2 default. `get_or_create_collection(metadata={"hnsw:space": "cosine"})` does NOT update the metric on an existing collection. The retriever's `1.0 - dist/2.0` formula then produces negative similarities. Fix: inspect existing collection's metadata, drop+recreate if not cosine. +- **`Assessor-current grades` heading pollution**: the assessor writes a `## Assessor-current grades` section to `skill-inventory.md` containing every skill name. If embedded into the index, it would dominate retrieval for any query. Excluded via `_EXCLUDED_HEADINGS`. +- **`_kebab` parenthetical stripping**: heading `## Fine-Tuning (awareness only — explicit anti-claim)` would otherwise become the unreadable ID `fine-tuning-awareness-only-explicit-anti-claim-per-role-clarifications`. Regex strips parentheticals + em-dash commentary before kebab-casing while preserving internal hyphens (`Fine-Tuning` stays `fine-tuning`, not `fine`). +- **AI-slop cleanup** (8 patterns): plan-meta narrative in production docstrings, defensive `try/except: pass`, speculative `_load_model` / `_get_client` helpers, `await ... await ...` on one line, etc. + +### Post-tag findings (deferred to Phase 2.A) + +Adversarial post-smoke review surfaced four LLM-behavior bug classes — documented in [`KNOWN_DATA_QUALITY_ISSUES.md`](KNOWN_DATA_QUALITY_ISSUES.md): + +- **B1**: `extract_node` under-extracts `skills_required` on best-fit roles. Sierra Agent Architecture JD scored 3.0 because `skills_required: []` left no signal for matched_skills. +- **B2**: `extract_node` mis-reads OR-lists ("languages such as Python, TypeScript, Go") as AND-lists. +- **B3**: `score_node` occasionally marks candidate-strong skills as missing (Python listed missing despite candidate's Python=3). +- **B4 (FIXED)**: `intake_filter` title-keyword list was leaky. 5 false-negatives migrated post-fix. + +These are LLM-prompt-tuning bugs that need the Phase 2.A eval harness (30 labeled JDs + score-MAE measurement) to fix correctly. Whack-a-mole prompt tweaking without measurement is worse than waiting. + +--- + +## 7. The Skill Assessor Loop (the "unique angle") + +### What it does + +The candidate marks each skill in `skills/SkillName.md` with `evidence: [learning-vault://...]` URIs pointing to artifacts in the learning vault. The skill assessor agent (`compass/analysis/skill_assessor.py`) reads those URIs, applies a deliberately skeptical hiring-manager rubric, and proposes new grades. + +### The rubric + +``` +Level 0 No exposure. +Level 1 Tutorial-level: course notes, "hello world", or a read paper. +Level 2 Applied in a personal project. Repo or vault note showing real use. +Level 3 Shipped. Deployed, evals exist, OR used by people other than candidate. +Level 4 Production-grade. Shipped WITH observability + cost tracking + recovered from a real failure. +Level 5 Authority. Taught it, merged upstream PR, or fixed a non-trivial bug in the library itself. +``` + +The agent demands artifact citations. Conceptual evidence caps at L1. Jumps of 2+ levels require HITL. + +### Real first run (2026-05-19) + +After wiring 4 skills to actual evidence URIs: + +| Skill | Current | Proposed | Reasoning | +|---|---|---|---| +| MCP | 4 | **2** (requires_hitl=True) | "Empty or template documents in projects/minx/. Minimal content." 2-level downgrade triggers human review. | +| LangGraph | 1 | 1 | "Template files for Compass project, no LangGraph implementation artifacts visible. Decision log empty." | +| Eval_harness | 0 | 0 | "Hamel reading material status=queued, read_count=0. No applied work." | +| HiTL | 1 | 1 | "Decision log template without actual HiTL decisions or artifacts." | + +**This is the value working as intended.** The grader correctly refused to credit Akash's actual MCP/LangGraph/HiTL work — because the *learning vault* doesn't yet document it. The work itself lives in the compass codebase. Filling `learning-vault/projects/compass/decisions.md` with the real architectural decisions from Phases 0-1.B.2 will raise the grades on next run. + +This is the loop: study/build → write it down → cite the writeup as evidence → agent regrades → gap plan reprioritizes. It closes the feedback loop between "what the market wants" and "what I can credibly claim." + +--- + +## 8. Key Design Decisions (and why) + +### Why a markdown vault instead of a database? + +**Decision**: store all agent output as `.md` files with YAML frontmatter. + +**Why**: Obsidian renders it natively. Dataview queries give you SQL-like power over markdown. The user can edit fields by hand in Obsidian and the next pipeline run respects those edits (e.g. tightening a CompanyNote.tier). No separate UI to build. Version-controllable. Diff-able. + +**Trade-off**: schema enforcement happens at write time via Pydantic, but a user editing the file in Obsidian can introduce invalid values. Phase 1.A bug #d40f36e was a literal example — `tier: applynow` (typo) crashed the next run. Fixed by tolerating invalid Literal values on read. + +### Why MCP for HITL (and not a web UI)? + +**Decision**: the human approves paused jobs via MCP tools called from Claude Code, not via a separate web app. + +**Why**: Akash already lives in Claude Code daily. Building a Flask UI to surface 5 pending approvals would be over-engineered. MCP tools (`pending_approvals()`, `approve(tid, True, "LGTM")`) are zero-friction from his existing workflow. + +**Trade-off**: the dashboard doesn't surface pending approvals (Dataview can't read SQLite). Phase 2.B may add a thin web view if this becomes a daily-flow friction point. + +### Why three vaults (pipeline / compass-vault / learning-vault)? + +**Decision**: agent-written content goes to `~/Documents/compass-vault/`. Personal notes stay in `~/Documents/learning-vault/`. The pipeline code only writes to compass-vault. + +**Why**: separation of concerns. The agent can rewrite anything in compass-vault freely — it's the system's output. The learning-vault is Akash's brain — the agent never touches it; it only resolves `learning-vault://` URIs to read evidence. + +**Trade-off**: cross-vault evidence requires the URI protocol (`compass/vault/learning_bridge.py`). Slight indirection. But preserves the cognitive model: "the agent doesn't read my journal." + +### Why LangGraph (and not custom state management)? + +**Decision**: use LangGraph for the pipeline state machine. + +**Why**: `interrupt()` + `AsyncSqliteSaver` checkpointing is what makes real HITL work. Building this from scratch — durable pause-resume across process restarts with a SQLite checkpointer — would be a multi-week side project. LangGraph 1.2 gives it for free. + +**Trade-off**: LangGraph has its own conventions (state schema as TypedDict, edges defined declaratively). Required Phase 1.B.1 to be careful about the "compile graph inside async with checkpointer" invariant. The framework's once-only warning sets and msgpack allowlist API have required 5+ adversarial review iterations to use correctly. + +### Why subagent-driven development? + +**Decision**: most implementation work goes through fresh implementer subagents with combined spec+quality reviewers between tasks. + +**Why**: it caught 8+ real bugs across Phase 1.B.1, several of which were genuine silent-data-correctness defects the human reviewer wouldn't have spotted. Fresh subagent per task means no context pollution. The reviewer cycle creates a forcing function — implementer can't ship questionable code because a different agent will read it. + +**Trade-off**: more LLM invocations per task (implementer + 2 reviewers minimum). Higher token cost. But far cheaper than debugging silent bugs in production. + +### Why so much adversarial review? + +**Decision**: every plan goes through 3-6 adversarial review iterations before execution. Every implementation gets a spec+quality review. Live smoke verification is mandatory before tagging. + +**Why**: Phase 0 caught 16 silent bugs. Phase 1.A caught 20 more. Phase 1.B.1 caught 8 more. The pattern is consistent: tests pass + smoke tests pass + code looks correct → but real data is wrong. The only thing that catches this is *adversarial inspection of real outputs*. + +The portfolio cost of shipping a silently-broken Compass to recruiters is enormous. The cost of one more review pass is ~30 minutes. Math is obvious. + +**Trade-off**: process overhead. But it's now the project's defining methodology and (frankly) its strongest interview talking point — "I built a system that caught its own data-correctness bugs through structured adversarial review, where most agentic systems ship with the bugs." + +--- + +## 9. The State of the System Right Now + +### What works end-to-end + +- **Daily pipeline**: `uv run python -m compass.pipeline.graph` against the configured ATS boards → scrapes, filters, scores, writes. Real cost per 20 jobs: ~$0.05. +- **HITL flow**: above-threshold jobs pause; `pending_approvals()` MCP tool lists them; `approve(tid, True, "LGTM")` resumes into tailor + vault_write. Audit trail intact. +- **Timeout cancel**: `uv run python -m compass.hitl.timeout_checker` resumes stale pending approvals as `timed_out`, writes to agent-log per spec. +- **RAG**: `uv run python -m compass.rag.indexer --force` builds the Chroma index. `score_node` uses retrieved chunks instead of full inventory. +- **Gap plan**: regenerates after every pipeline run AND after every resume. Out-of-scope JobNotes excluded. +- **Skill assessor**: `assess_skills(scope=[...])` MCP tool grades skills against evidence URIs. Adversarial grader catches under-documented claims. + +### What's stubbed but not yet exercised in production + +- **Modal cron**: `timeout_checker.py` is callable but no schedule wires it. → Phase 1.B.3. +- **Langfuse traces**: callback API mismatch (`host` kwarg) — every run logs the error, pipeline continues. → Phase 1.B.3. +- **Eval harness**: 30 labeled JDs + nightly score-MAE regression. Not built. → Phase 2.A. +- **Application lifecycle**: tools work but `~/Documents/compass-vault/applications/` is empty — nobody has applied via the tool yet. + +### What's deferred and why + +[`docs/KNOWN_DATA_QUALITY_ISSUES.md`](KNOWN_DATA_QUALITY_ISSUES.md) lists 8 known bug classes (B1-B8) with severity, evidence, and Phase 2.A fix surface for each. The headline ones: + +- **B1**: extract under-extracts skills_required on best-fit JDs +- **B7**: gap_aggregator includes auto_rejected/timed_out/null at full weight +- **B8**: tailor_node has no programmatic constraint against hallucination + +Each needs a regression test against labeled JDs (Phase 2.A) before tuning, otherwise we whack-a-mole. + +--- + +## 10. What's Next + +### Phase 1.B.3 — automation + observability (next sub-phase) + +- **Modal cron** for daily scrape (9 AM Central) + weekly skill_assessor (Sunday 2 AM Central). The `timeout_checker` becomes a Modal-scheduled job. +- **Langfuse callback fix**. Then a public trace URL goes in the README. Portfolio claim. +- **Modal Secrets**: secrets out of `.env`, into Modal's secret manager. Compass repo can be public. +- **URL dedup for filtered-jobs**. Currently the same sales JDs hit `intake_filter` every run. Trivial cache. +- **I4 race fix**: `claim_pending` atomic transition prevents double-resume races once Modal cron and human MCP can fire simultaneously. + +### Phase 2.A — eval harness + +- 30 hand-labeled JDs in `compass/evals/dataset.json` +- Nightly: extract recall, score MAE vs ground truth, cost-per-run, p50 latency +- LangFuse-logged with regression alert if MAE > baseline + 2σ +- **Then** tune extract/score prompts to fix B1-B3. Measure each change. + +### Phase 2.B — portfolio polish + +- README: architecture diagram (mermaid), screenshots, public Langfuse trace URL, "what I learned" section, eval results table +- Public Langfuse trace URL reachable without auth +- Repo link in resume Projects section + +### Phase 2.C — blog post + +"Building an agent that grades my skills against the live job market." The skill_assessor loop is the differentiated story. + +### Phase 3+ — coverage expansion + +Workday, Apple, Google, AWS, Microsoft scrapers. YC WAAS, HN Who's Hiring. Long-tail. + +--- + +## 11. Glossary + +| Term | Meaning | +|---|---| +| **Agentic AI** | software that orchestrates LLM calls + tools to accomplish multi-step tasks (e.g. agent that searches the web, summarizes findings, drafts a response). Compass itself is agentic. | +| **ATS** | Applicant Tracking System (Greenhouse, Lever, Ashby). Standard interface job postings flow through. Compass scrapes each ATS's public unauthenticated API. | +| **CompassState** | the TypedDict shared between all LangGraph nodes. One JD's-worth of pipeline state. | +| **Canonical taxonomy** | the 95-skill registry at `_meta/skill-taxonomy.md`. Maps synonyms ("pydantic-ai", "Pydantic AI") to one canonical name. | +| **Chroma** | the persistent vector DB used for RAG. Stores `skill-inventory.md` chunks + their embeddings. | +| **`interrupt()`** | LangGraph function that pauses graph execution mid-node. The graph state is checkpointed; control returns to the caller. The same node re-runs on resume with the human's value substituted. | +| **JobNote** | one markdown file per scored JD at `compass-vault/jobs/`. Frontmatter has `match_score`, `hitl_decision`, `skills_matched`, etc. | +| **MCP** | Model Context Protocol. Anthropic's standard for exposing tools to LLM clients (Claude Code, Cursor). Compass's MCP server exposes 16 tools — `pending_approvals`, `approve`, `add_application`, etc. | +| **Modal** | the serverless platform where Phase 1.B.3 will host the daily cron. Free tier covers Compass's expected usage. | +| **Pydantic AI** | the library that gives LLMs typed structured output. Define a Pydantic model; the LLM is constrained to return data that validates against it. Used for extract + score nodes. | +| **RAG** | Retrieval-Augmented Generation. Instead of dumping all candidate-profile data into every LLM prompt, retrieve the relevant chunks via vector similarity. | +| **Role family** | the classification an in-scope JD falls into: `agent-engineer`, `applied-ai`, `swe-backend`, `fde-eng`, `research-eng`, or `out-of-scope`. Stored on JobNote frontmatter. | +| **score_threshold** | currently 3.5. Jobs scoring ≥ this pause for human approval. Below auto-reject. Captured into pipeline state at score time so pause-resume across env changes stays consistent. | +| **`learning-vault://` URI** | the protocol the skill_assessor uses to read evidence files from `~/Documents/learning-vault/`. Read-only; the agent never writes there. | + +--- + +## 12. Files You'll Touch Most + +If you want to understand any one piece deeply, these are the entry points: + +- **The whole pipeline**: [compass/pipeline/graph.py](../compass/pipeline/graph.py) — `run_pipeline()` is the top-level function. Every node hangs off the graph defined here. +- **A specific node**: `compass/pipeline/nodes/{intake,intake_filter,extract,score,reflect,hitl,tailor,vault_write}.py` +- **HITL infrastructure**: `compass/hitl/{state_store,resume,timeout_checker}.py` +- **RAG**: `compass/rag/{indexer,retriever}.py` +- **MCP tools**: [compass/mcp_server/server.py](../compass/mcp_server/server.py) +- **Skill assessor**: [compass/analysis/skill_assessor.py](../compass/analysis/skill_assessor.py) +- **Gap aggregator**: [compass/analysis/gap_aggregator.py](../compass/analysis/gap_aggregator.py) +- **Vault schemas**: [compass/vault/schemas.py](../compass/vault/schemas.py) — what every JobNote/SkillNote/CompanyNote/ApplicationNote frontmatter must look like +- **Config**: [compass/config.py](../compass/config.py) — every env var and default path lives here + +And the corresponding test files in `tests/` mirror the same structure. 259 tests at current tag. + +--- + +## 13. The Big Picture (one paragraph) + +You're building an agentic career coach that scrapes job boards, drops noise, scores in-scope JDs against your profile via Gemini Flash, pauses high-scoring jobs for your approval through real LangGraph `interrupt()`, generates Sonnet-tailored paragraphs for approved ones, derives a master gap plan from every scored JD via weighted demand math, and houses an adversarial skill grader that reads evidence from your personal learning vault and proposes grade changes when you've shipped new work. All output is markdown in an Obsidian vault. Cost ~$0.05 per 20-job batch. 259 tests passing. Six phases left to complete: automation cron, observability, eval harness, polish, blog post, coverage expansion. + +You're roughly halfway through the planned 8-10 sessions. The hardest architectural calls are made. What remains is operational discipline + measurement. From daa4bb52fcc77b900563cd78e8745ec1d8c6a126 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 13:44:19 -0500 Subject: [PATCH 26/46] style(tests): TYPE_CHECKING guard + ruff format on B6 regression test --- tests/analysis/test_gap_aggregator_role_filter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/analysis/test_gap_aggregator_role_filter.py b/tests/analysis/test_gap_aggregator_role_filter.py index 3d4dec7..ac68642 100644 --- a/tests/analysis/test_gap_aggregator_role_filter.py +++ b/tests/analysis/test_gap_aggregator_role_filter.py @@ -1,8 +1,13 @@ """Regression test for B6 fix: out-of-scope JobNotes must not contribute to gap plan.""" + from __future__ import annotations -from pathlib import Path + +from typing import TYPE_CHECKING + import frontmatter -import pytest + +if TYPE_CHECKING: + from pathlib import Path def _write_job(vault: Path, name: str, role_family: str, required: list[str], score: float) -> None: From ad297ef689f5db9e7fd1547e2de3af41bdd1d0b9 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 13:45:56 -0500 Subject: [PATCH 27/46] docs: obsidian leverage design proposals (P1-P8) --- docs/OBSIDIAN_LEVERAGE.md | 347 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 docs/OBSIDIAN_LEVERAGE.md diff --git a/docs/OBSIDIAN_LEVERAGE.md b/docs/OBSIDIAN_LEVERAGE.md new file mode 100644 index 0000000..27a9649 --- /dev/null +++ b/docs/OBSIDIAN_LEVERAGE.md @@ -0,0 +1,347 @@ +# Obsidian Leverage — Design Proposals + +> The vault is currently a markdown store with YAML frontmatter. Obsidian renders it but the only Obsidian-specific feature we use is Dataview on the dashboard. This doc maps the unused capabilities — wikilinks, graph view, embeds, Bases, Canvas, tags — to concrete project-relevant uses. Pick what to build; the proposals are sized. + +**Current state:** 23 JobNotes, 95 SkillNotes, 9 CompanyNotes, 0 ApplicationNotes. Skills appear in JobNote frontmatter as plain string lists (`skills_required: [Python, LangGraph]`). No bidirectional navigation. Graph view is empty. + +--- + +## P1. Bidirectional skill ↔ job wikilinks (your specific ask) + +**Goal:** click "Python" in a JobNote → see every job requiring Python. Click any skill in graph view → see the constellation of jobs that demand it. Find rare skills (skills appearing in 1-2 jobs) vs commodity ones (appearing in 10+). + +### How Obsidian wikilinks work + +`[[Python]]` in any markdown file: +- Renders as a clickable link +- Creates a backlink on the target file (`skills/Python.md` shows "Linked mentions" of the source) +- Appears as an edge in the graph view + +Obsidian resolves `[[Python]]` to any file named `Python.md` in the vault. Since `compass-vault/skills/Python.md` exists, the link works. (No `skills/` prefix needed unless there's an ambiguous filename collision.) + +### Proposal — non-invasive + +Keep the typed frontmatter (`skills_required: [Python, LangGraph]`) as the source of truth — Dataview queries and Python validation both depend on it. ADD a new section to the JobNote body that renders as wikilinks for human navigation + graph view. + +**Change to `compass/vault/writer.py:write_job_note`** — when writing the JobNote body, prepend a section like: + +```markdown +## Skills + +**Required:** [[Python]] · [[LangGraph]] · [[MCP]] +**Nice to have:** [[FastAPI]] +**Matched:** [[Python]] · [[MCP]] +**Missing:** [[LangGraph]] · [[Sub-agents]] + +--- +``` + +(One line per category. Bullet/list works too — `- [[Python]]` per line — but inline `·`-separated is denser and reads well.) + +**Change to gap_aggregator's `_sync_skill_counters`** (which already updates SkillNote frontmatter): also write a "Jobs Requiring This Skill" section to the SkillNote body. Two options: + +**Option A — static list, rewritten each run** (simpler, no plugin dependency): +```markdown +## Jobs requiring this skill + +- [[2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f|Sierra — Software Engineer, Agent Architecture]] · score 3.0 · apply-now +- [[2026-05-06-cognition-Software_Engineer-7c26fdaf|Cognition — Software Engineer]] · score 3.0 · apply-now +- [[2025-04-18-sierra-Software_Engineer_Agent-8f58539b|Sierra — Software Engineer, Agent]] · score 4.5 · apply-now +``` + +Sortable by score; renders as clickable links. Updated atomically on each gap-aggregator regen. + +**Option B — Dataview query block** (more powerful, requires Dataview plugin which is already used): +````markdown +## Jobs requiring this skill + +```dataview +TABLE company, match_score AS Score, tier +FROM "jobs" +WHERE econtains(skills_required, this.file.name) +SORT match_score DESC +``` +```` + +Always fresh, sortable in the UI, no per-job rewriting. + +**Recommendation: Option A first, Option B if it gets stale.** Option A is more robust because it works even if Dataview is disabled or breaks. Option B is fancier but couples to a plugin. + +### Effort + +- Writer body-section change: **30 min** (one new function, wire into `write_job_note`) +- SkillNote body update: **30 min** (extend `_sync_skill_counters` to write the Jobs section) +- Tests: 2 small unit tests +- One-time backfill: write a script that walks existing JobNotes and re-renders body sections (similar to the B6 migration script) +- **Total: ~2 hours** + +### What you get + +- Click `[[Python]]` in any JobNote → land on `skills/Python.md` → see every JobNote requiring it, ranked by score +- Graph view (Obsidian sidebar → "Open graph view") shows clusters. Skills appearing in many JobNotes become hub nodes. Rare skills are peripheral. +- Visual answer to your specific question — *"which skills are required across many jobs and which are rare/specific"* + +--- + +## P2. Skill rarity dashboard panel + +**Goal:** at a glance, see which skills are commodity (every JD wants them) vs differentiated (rare, niche, harder to source). + +**Implementation:** add a Dataview block to `compass-vault/dashboard.md`: + +````markdown +## Skill rarity — which gaps to prioritize + +```dataview +TABLE WITHOUT ID + file.link AS Skill, + appears_in_jobs AS "in jobs", + my_level AS "my level", + gap_score AS "gap score" +FROM "skills" +WHERE appears_in_jobs > 0 +SORT appears_in_jobs DESC, gap_score DESC +LIMIT 30 +``` + +## Rare/specialized skills (appears in 1-2 jobs only) + +```dataview +TABLE WITHOUT ID + file.link AS Skill, + appears_in_jobs AS "in jobs" +FROM "skills" +WHERE appears_in_jobs > 0 AND appears_in_jobs <= 2 +SORT appears_in_jobs ASC +``` +```` + +### Effort + +- **5 min** to add to dashboard.md +- No code changes + +### What you get + +- Top panel: skills sorted by demand frequency. The "commodity" skills (Python, Docker) at the top. Investing in them helps for many jobs. +- Second panel: skills appearing in 1-2 jobs only. These are differentiators — if you don't have them, only specific roles surface. If you DO have them, you can target those niche roles confidently. + +--- + +## P3. Auto-tags on JobNotes for filter dimensions + +**Goal:** flexible filtering via Obsidian's tag system. Find all "stretch" jobs across companies, all "apply-now-fit" jobs, all jobs requiring TypeScript, etc. + +**Current state:** JobNote has `tags: list[str]` in frontmatter but nothing populates it. + +**Proposal:** `vault_write_node` auto-generates tags based on JobNote fields: + +```python +tags = [] +# tier-derived +if tier == "apply-now": tags.append("#tier/apply-now") +elif tier == "6-month": tags.append("#tier/6mo") +elif tier == "stretch": tags.append("#tier/stretch") +# fit-derived +if match_score >= 4.0: tags.append("#fit/strong") +elif match_score >= 3.0: tags.append("#fit/decent") +elif match_score >= 2.0: tags.append("#fit/stretch") +else: tags.append("#fit/weak") +# role-family-derived +if role_family: tags.append(f"#role/{role_family}") +# hitl-decision-derived +if hitl_decision: tags.append(f"#decision/{hitl_decision}") +``` + +Use Obsidian's tag pane to filter by any combination. + +### Effort + +- **20 min** in `vault_write_node` + a one-time migration to apply tags to existing JobNotes +- 1 unit test verifying tag generation + +### What you get + +- Click any tag in Obsidian's tag pane → see every JobNote with that tag +- Compose tags: `#fit/strong AND #role/agent-engineer` to find strong-fit agent roles +- Tag-based Dataview queries become natural ("show all #decision/timed_out with score > 3.5 — these are the ones the auto-reject lost") + +--- + +## P4. SkillNote embeds candidate level into JobNote view + +**Goal:** when reading a JobNote, immediately see your level on each required skill without flipping pages. + +**How embeds work:** `![[Python#current-level]]` inlines the content under the "Current level" heading from `skills/Python.md`. Live-rendered in Obsidian. + +**Proposal:** in the JobNote body's `## Skills` section: + +```markdown +## Skills with candidate level + +| Skill | Required? | My level | +|---|---|---| +| Python | required | ![[Python#level]] | +| LangGraph | required | ![[LangGraph#level]] | +| MCP | required | ![[MCP#level]] | +``` + +This requires each SkillNote to have a `## Level` heading with the level number visible (e.g. `Level 4 (production)`). The seed script can be extended. + +**Alternative — Dataview-only**: + +````markdown +```dataview +TABLE WITHOUT ID skill AS Skill, my_level AS "My level" +FROM "skills" +WHERE contains(this.skills_required, skill) +``` +```` + +### Effort + +- Embeds path: requires modifying SkillNote body to have a `## Level` section (15 min seed + body-rendering change in writer) +- Dataview path: 5 min, no SkillNote changes +- **Total: 10-30 min depending on approach** + +### What you get + +- Open a JobNote → see "Python: 3, LangGraph: 1, MCP: 4" inline. No clicking. +- Quick scan of your gaps for any role. + +--- + +## P5. Canvas view of the gap plan + +**Goal:** visualize the top-N gaps as a graph showing which jobs each gap would unlock. + +**Obsidian Canvas** is a node-and-arrow visual editor. You drop notes onto a canvas; Obsidian renders them as cards. + +**Proposal:** a script that builds `study-plans/gap-plan.canvas` JSON from the master gap plan + JobNote relationships. Each top-10 gap becomes a center node; arrows fan out to the JobNotes that would benefit. Edge thickness = how many jobs depend on that skill. + +**Effort: 1 hour** for a simple generator. The Canvas JSON format is documented and stable. + +### What you get + +A visual map of "which skills should I learn next, ordered by how many doors they open." More motivating than a ranked list. + +This is portfolio-quality polish — recruiters love screenshots of this kind of thing. + +--- + +## P6. Bases (newer Obsidian feature) for table views + +**Goal:** SQL-like filtered views of JobNotes/ApplicationNotes without writing Dataview queries each time. + +**What Bases are:** `.base` files in Obsidian that define views over frontmatter. Like saved-query database views. Newer feature (released 2025). + +**Proposal:** ship 3 `.base` files in the vault: + +- `applications.base` — all ApplicationNotes, filterable by status/next_action_date +- `jobs-strong-fit.base` — JobNotes with match_score >= 4.0, sorted by date +- `skills-by-rarity.base` — SkillNotes sorted by appears_in_jobs ascending (rare first) + +### Effort + +- **30 min** to design + author 3 .base files +- No code changes +- Requires the user to be on Obsidian 1.7+ (Bases shipped 2025) + +### What you get + +Database-style table views that update automatically. More performant than Dataview for large vaults; first-class UI. + +--- + +## P7. Daily-note linking for organic evidence URIs + +**Goal:** close the loop between "I worked on a skill today" and "skill_assessor sees evidence of work." + +**Current state:** the candidate has `learning-vault/daily/YYYY-MM-DD.md` files but they're not linked to skills. + +**Proposal:** establish a convention in daily notes: + +```markdown +# 2026-05-19 + +Worked on Compass Phase 1.B.2: RAG via Chroma. Shipped the indexer + +retriever. Wrote a [[LangGraph#interrupt and resume]] integration that +checkpoints to SQLite. Tested HITL flow end-to-end. + +Skills practiced: [[LangGraph]] [[Chroma]] [[RAG]] [[MCP]] +``` + +When skill_assessor reads `learning-vault://daily/2026-05-19.md` as evidence for `LangGraph`, the skeptical grader sees: real work, specific artifact, last_modified today. Raises the grade. + +**Implementation:** no code change required. Pure convention. Optional: a small `compass/scripts/sync_daily_to_skills.py` that scans daily notes for `[[SkillName]]` patterns and auto-adds the daily-note URI to each referenced SkillNote's evidence list. **20 min.** + +### What you get + +Daily writing → automatic skill evidence → grade updates on next assessor run. The "build → write → regrade" loop the spec promised, with zero friction. + +--- + +## P8. Tagging interview-stage applications + +**Goal:** when an ApplicationNote moves through stages (applied → screen → onsite → offer), tag transitions show up in Obsidian's tag pane. + +**Implementation:** `compass/applications/lifecycle.py:update_application_status` already updates the `status` frontmatter. Extend it to also append a stage-transition tag to the ApplicationNote body: + +```markdown +## Stage transitions +- 2026-05-19 → #stage/applied +- 2026-05-22 → #stage/screen +- 2026-05-29 → #stage/onsite +``` + +Each tag becomes a backlink target. Click `#stage/onsite` in the tag pane → see all applications in onsite stage. + +### Effort + +- **20 min** in `lifecycle.py` +- The user has zero ApplicationNotes today so no backfill needed + +### What you get + +Visual interview-pipeline tracking in Obsidian. Combined with the existing `list_pending_actions` MCP tool, you get both temporal (next_action_date) and stage-based filtering. + +--- + +## Recommended bundle + +If you want a single focused project to ship next that delivers the biggest visual + workflow win: + +**Ship P1 + P2 + P3 together** as a small "Obsidian leverage" patch on top of `phase-1b2-rag`: + +1. **P1**: JobNote body gets `## Skills` section with `[[skill]]` wikilinks. SkillNote body gets `## Jobs requiring this skill` list. +2. **P2**: dashboard.md gets "Skill rarity" + "Rare/specialized skills" Dataview panels. +3. **P3**: JobNote frontmatter gets auto-generated tags from tier / score / role_family / hitl_decision. + +**Total effort: ~3 hours** including the one-time migration scripts to backfill existing JobNotes + SkillNotes. + +**Visible result:** +- Obsidian graph view becomes meaningful — clusters of jobs around hub skills, peripheral rare skills visible at a glance +- Click any skill in a JobNote → instantly see every related job +- Tag pane gives you `#fit/strong`, `#role/agent-engineer`, etc. for one-click filters +- Dashboard answers your exact question (which skills are commodity vs rare) at the top + +This is also strong portfolio material — Obsidian-as-product-UI for an agentic career coach is the kind of recruiter-facing screenshot that lands. + +--- + +## Things I'd defer + +- **P5 Canvas**: pretty but complex to maintain. Wait until daily use surfaces a need. +- **P6 Bases**: requires Obsidian 1.7+ and is still maturing. Try in a side branch first. +- **P7 daily-note linking**: depends on the user actually using daily notes consistently. Currently 3 daily notes exist (May 17-19). If you build the habit first, the tool follows. + +--- + +## Where this fits in the broader phase plan + +This is a **vault-rendering layer** — doesn't touch the pipeline, LLM, or HITL infrastructure. It's pure additive Obsidian polish. The cleanest place for it: + +- **Phase 1.B.2.x patch** (this branch, before tagging 1.B.3) +- OR **Phase 1.B.3** as a side-quest after Modal cron + Langfuse fix lands + +Recommendation: ship the **P1+P2+P3 bundle** as a small side-quest immediately. It's high-ROI, low-risk, and gives you a daily-use UI improvement before the 1.B.3 work starts (which is mostly invisible plumbing). From 19bcf162e9a95762acefebcc8afb56c652b29508 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 13:52:16 -0500 Subject: [PATCH 28/46] docs: use-case oriented obsidian leverage brainstorm (problems 1-9) --- docs/OBSIDIAN_LEVERAGE_PART_2.md | 728 +++++++++++++++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 docs/OBSIDIAN_LEVERAGE_PART_2.md diff --git a/docs/OBSIDIAN_LEVERAGE_PART_2.md b/docs/OBSIDIAN_LEVERAGE_PART_2.md new file mode 100644 index 0000000..1b09cf9 --- /dev/null +++ b/docs/OBSIDIAN_LEVERAGE_PART_2.md @@ -0,0 +1,728 @@ +# Obsidian Leverage — Part 2: Use-Case Oriented Ideas + +> Part 1 (`OBSIDIAN_LEVERAGE.md`) covered the foundational graph/link layer (P1-P8). This doc goes deeper — organized around the actual problems you face during a tier-2 agentic-AI job search, with Obsidian features mapped to each. Pick what helps; ignore what doesn't. + +--- + +## Problem 1: "What should I study TODAY?" + +You have a master gap plan, but it's a ranked list. You need a daily-flow recommendation that connects "what I worked on yesterday" with "what to do today." + +### Idea 1.1: Daily note → skill evidence auto-pipeline + +You already have `learning-vault/daily/YYYY-MM-DD.md`. Establish a convention: when you log work in a daily note, link the skill. Example: + +```markdown +# 2026-05-19 + +Today I built the Chroma RAG layer for Compass. Decided to use cosine +distance instead of L2 after testing both. Empirically verified the +`with_msgpack_allowlist` API is a no-op when the default is True. + +Worked on: [[LangGraph]] [[Chroma]] [[RAG]] [[MCP_server_authoring]] +``` + +**Automation:** a tiny script `compass/rag/sync_daily_evidence.py` scans `learning-vault/daily/*.md` for `[[SkillName]]` patterns and auto-appends the daily-note URI to each referenced SkillNote's `evidence:` list. Run weekly via Modal cron (Phase 1.B.3). + +**What you get:** daily writing → skill evidence URIs grow → next `assess_skills` run reflects the work. Closes the "build → write → regrade" loop the spec promised. + +**Effort:** ~30 min for the script. Behavioral cost: you have to actually write daily notes. + +### Idea 1.2: Heatmap-calendar plugin (community) + +Install `obsidian-heatmap-calendar`. Render a GitHub-contributions-style grid showing daily skill activity. Days with skills worked on glow green; idle days are gray. + +**Where:** a callout block on `dashboard.md`: + +````markdown +```heatmap-calendar +year: 2026 +colors: greens +entries: + # dataview-derived count of distinct skills worked on per day +``` +```` + +**What you get:** visual reinforcement loop. Bad week → see the gray patch → fix it. Good streak → see momentum. + +### Idea 1.3: "Today's study target" — a dynamic dashboard widget + +Dataview block on `dashboard.md` that computes: "the top-3 gaps from master-gap-plan, intersected with skills the user hasn't touched in the last 7 days." + +````markdown +```dataview +TABLE WITHOUT ID + file.link AS Skill, + gap_score AS Gap, + last_assessed AS "Last touched" +FROM "skills" +WHERE gap_score > 0.5 + AND (last_assessed = null OR date(last_assessed) < date(today) - dur(7d)) +SORT gap_score DESC +LIMIT 3 +``` +```` + +**What you get:** opens dashboard → sees "your top neglected high-gap skills are X, Y, Z." No decision fatigue. + +--- + +## Problem 2: "I have an interview Friday — what do I prepare?" + +You get a recruiter screen for Sierra Agent Engineer. You need to know which of your skills + stories to lead with, which gaps to acknowledge, which to deflect. + +### Idea 2.1: STAR story bank linked to skills + +Create `compass-vault/interview-prep/stories/*.md` — one note per STAR story (Situation, Task, Action, Result). Each story links to the skills it demonstrates: + +```markdown +# Cisco MCP Servers — 4-server architecture + +## Situation +Cisco needed natural-language access to their internal observability stack... + +## Task +Build MCP servers for [[Splunk]], [[Grafana]], [[Datadog]], [[ServiceNow]]... + +## Action +Designed a 4-server pattern with shared auth + tool registry. Used +[[FastMCP]] and [[Pydantic]] for typed surfaces. [[Python]] backend... + +## Result +Adopted across the SRE team. Reduced incident triage time by ~40%. +Talk: [[talks/cisco-mcp-internal-talk]] + +## Skills demonstrated +[[MCP]] [[MCP_server_authoring]] [[Python]] [[Multi-Agent]] [[Production_Concerns]] +``` + +**What you get:** when a JobNote requires `[[MCP]]`, Obsidian's backlinks panel on the MCP SkillNote shows every story you have for it. Or vice versa — open the JobNote, see which skills are required, click each one to find your relevant stories. + +### Idea 2.2: Interview-prep playbook auto-assembled per JobNote + +A Dataview query block embedded in EVERY JobNote (via writer template): + +````markdown +## Stories I have for this role's requirements + +```dataview +LIST +FROM "interview-prep/stories" +WHERE any(skills_demonstrated, (s) => contains(this.skills_required, s)) +SORT file.name DESC +``` + +## Skill gaps to acknowledge + +`= filter(this.skills_required, (s) => link("skills/" + s).my_level < 2)` +```` + +**What you get:** open any JobNote → instantly see which stories to lead with + which weaknesses to prepare a response for. Zero manual lookup. + +### Idea 2.3: Post-interview retro that closes the loop + +After every interview, a retro note in `learning-vault/interview-prep/postmortems/`: + +```markdown +# 2026-05-25 — Sierra Agent Engineer screen + +Result: passed → onsite + +## What worked +- Led with Cisco MCP story for the agent-orchestration question +- "Show me a hard debugging session" → described the C1 audit-trail + divergence we fixed in Compass 1.B.1 + +## What missed +- They asked about [[Evals]] specifically — I conceptually know it but + no shipped artifact. Acknowledged as a current gap. +- [[Modal]] cron experience — only theoretical right now (1.B.3 work + hasn't shipped). Pivoted to general k8s cron experience. + +## Linked +[[applications/2026-05-19-sierra-Software_Engineer_Agent_Architecture]] +``` + +**Automation hook:** add `learning-vault://interview-prep/postmortems/*.md` to every relevant SkillNote's `evidence:` on save. Skills you successfully described in interviews accumulate proof. Skills that got flagged as gaps move up the study priority. + +### Idea 2.4: "Difficult question bank" tagged by skill + +A folder `interview-prep/questions/` with notes per question type. Tag each with the skills it tests: + +```markdown +# How would you build an agentic system for X? + +Tags: #q/system-design #skill/agent-frameworks #skill/multi-agent + +## My current answer outline +1. Clarify constraints: latency, error tolerance, observability +2. Sketch [[LangGraph]] state machine + checkpointing +3. Tool layer via [[MCP]] +4. Eval harness via [[LangSmith]] or [[Langfuse]] +``` + +**What you get:** before any interview, search the tag pane for `#skill/agent-frameworks` → see every question you've prepped + your current answer. Refine answers over time as artifacts grow. + +--- + +## Problem 3: "I keep missing good jobs in the noise" + +The pipeline scrapes ~20 jobs/day. You can't scan all of them. The dashboard's "Apply now (top 5)" panel helps but you're missing context. + +### Idea 3.1: Quick-capture inbox + +`compass-vault/_inbox/` folder. When you see a JD on LinkedIn / X / Slack, paste it into `_inbox/2026-05-19-some-company.md`. The pipeline gains an `inbox_node` (or `_scrape_all` is extended) that picks up `_inbox/*.md` files and runs them through the normal pipeline. + +**What you get:** zero-friction capture. Don't lose interesting jobs to "I'll save the link in a Slack DM and forget." + +**Effort:** ~1 hour to add an inbox scraper. Same RawJob shape as the ATS scrapers; just a different source. + +### Idea 3.2: Custom "this week's best fits" Dataview + +The dashboard has "Apply now (top 5)" but it's score-sorted from all-time. Add a "best fit found this week" panel: + +````markdown +```dataview +TABLE WITHOUT ID + file.link AS Job, + company, + match_score AS Score, + tier +FROM "jobs" +WHERE date_found >= date(today) - dur(7d) + AND match_score >= 3.0 + AND tier != "skip" +SORT match_score DESC +LIMIT 10 +``` +```` + +**What you get:** Monday morning, open dashboard → "10 fresh roles worth a look this week." + +### Idea 3.3: Anti-pattern note → intake_filter feedback + +You keep getting AE/PM/management roles through the filter (or marginally past it). Maintain a personal anti-pattern note: + +```markdown +# _profile/anti-patterns.md + +## Roles I will NEVER apply to (auto-reject) +- Management track (Engineering Manager, Lead, Director, VP) +- Pre-sales / Solutions Engineer / Sales Engineer +- Customer Success / Solutions Architect (sales-track) +- Pure DevRel / Developer Advocate +- Internships + +## Roles I'm UNDECIDED on (route to manual review) +- Research-only (no shipped systems): scoring lower than 3.0 is fine +- Customer-facing FDE roles: prefer SF/NYC, decline remote-only +``` + +`intake_filter._llm_classify` reads this note as context, biasing its classifications. + +**What you get:** the more you reject a class of role, the more aggressively the filter learns to drop it. You teach the system over time. + +### Idea 3.4: Geographic + salary filter dimensions + +JobNote already has `location` + `salary_min` + `salary_max`. Add Dataview panels on the dashboard: + +````markdown +## NYC roles (apply-now tier) +```dataview +LIST WITHOUT ID file.link +FROM "jobs" +WHERE tier = "apply-now" AND (contains(location, "NYC") OR contains(location, "New York")) +SORT match_score DESC +``` + +## Above-target comp +```dataview +TABLE company, salary_min, salary_max, match_score +FROM "jobs" +WHERE salary_min >= 250000 OR salary_max >= 350000 +SORT salary_min DESC +``` +```` + +**What you get:** filter by your real constraints, not just score. + +--- + +## Problem 4: "I want to make better application decisions over time" + +You apply to 5 jobs. Three ghost you. Two screen you out. You don't have a record of WHY you applied and what the pattern is. + +### Idea 4.1: Decision journal + +A note per application with the decision rationale: + +```markdown +# 2026-05-19 — Why I applied to Sierra Agent Architecture + +## Why yes +- Score 3.0 + my MCP/LangGraph fit +- Stretch role — gets me to the agent-engineering interview table +- Sierra is well-funded, founder-mindset language matches +- Has [[Compass-style HITL]] in their JD → talking point + +## Why I almost said no +- Score is at threshold; not a slam-dunk fit +- Onsite in NYC; I'm Austin-based +- Compensation band not posted + +## Decision rule applied +"Above 3.0 + tier=apply-now + role-family=agent-engineer → always apply" + +## Result tracking +- 2026-05-19 — applied +- TBD — screen +- TBD — onsite +- TBD — outcome +``` + +**Pattern recognition:** after 6 months, review every "Why yes" — what predicted callbacks vs ghosting? Refine the decision rule. Update `_profile/preferences.md`. + +### Idea 4.2: Apply-rate dashboard + +Dataview block tracking apply rate per company / tier / role-family: + +````markdown +```dataview +TABLE WITHOUT ID + group AS "Bucket", + length(rows.file) AS "Total seen", + length(filter(rows.status, (s) => s = "applied")) AS "Applied", + length(filter(rows.status, (s) => s = "applied")) / length(rows.file) * 100 AS "Apply %" +FROM "jobs" +GROUP BY tier +``` +```` + +**What you get:** "I see 30 apply-now jobs/month, apply to 4. Conversion: 13%." Useful for setting goals + spotting weeks you're under-applying. + +### Idea 4.3: Offer comparison table + +When you have 2+ offers, a single note `offers/2026-Q2-comparison.md`: + +```markdown +# Q2 Offer Comparison + +| Attribute | Sierra | Decagon | Cognition | +|---|---|---|---| +| Base | $220K | $235K | $210K | +| Equity (4-yr) | $400K | $300K | $600K | +| Bonus | 15% | 20% | 10% | +| Role family | Agent Eng | Agent Eng | Applied AI | +| Tier | apply-now | apply-now | apply-now | +| Notes link | [[applications/sierra-...]] | [[applications/decagon-...]] | [[applications/cognition-...]] | +``` + +Dataview can auto-build this from ApplicationNotes once they exist. + +--- + +## Problem 5: "I lose track of my network and outreach" + +Recruiters DM you. Former coworkers refer roles. Friends-of-friends offer intros. You forget who said what when. + +### Idea 5.1: PersonNote per contact + +`compass-vault/people/Jane_Doe.md`: + +```markdown +--- +type: person +role: Engineering Manager +company: Sierra +relationship: ex-coworker | recruiter | referral | mutual +linkedin: https://linkedin.com/in/janedoe +first_contact: 2026-05-15 +last_contact: 2026-05-19 +status: warm +--- + +# Jane Doe — Sierra + +Met at the AI agents meetup in Austin (April 2026). Now EM of the agent +orchestration team. Offered to refer me when I apply. + +## Conversations +- 2026-05-15 — DM exchange about Compass project. Liked the HITL design. +- 2026-05-19 — Referral offered for [[applications/sierra-...]] + +## Related +- [[applications/2026-05-19-sierra-Software_Engineer_Agent_Architecture]] +- [[companies/sierra]] +``` + +**Graph view payoff:** people → companies → JobNotes → skills. The whole network as a navigable map. + +### Idea 5.2: Outreach cadence reminders + +Tag people with `#contact/follow-up-due`. Dataview panel on dashboard: + +````markdown +```dataview +LIST file.link +FROM "people" +WHERE last_contact != null AND date(last_contact) < date(today) - dur(30d) + AND status = "warm" +SORT last_contact ASC +``` +```` + +**What you get:** monthly nudge to keep the network warm without active tracking effort. + +### Idea 5.3: Referral attribution + +ApplicationNote frontmatter already has a `referral: bool` field (Phase 1.A). Extend to `referral_from: "[[people/Jane_Doe]]"`. Then a dashboard panel: + +````markdown +## Referrals and their outcomes +```dataview +TABLE WITHOUT ID + file.link AS Application, + referral_from AS "Via", + status, + applied_at +FROM "applications" +WHERE referral_from != null +SORT applied_at DESC +``` +```` + +After 6 months: which referrers actually convert. Helps you prioritize who to ask next. + +--- + +## Problem 6: "I want to see my progress over time" + +Are my skills actually improving? Am I applying to better-fitting jobs? Am I closing the gap? + +### Idea 6.1: Skill-level over time chart + +Each `skills/X.md` already has `last_assessed: datetime` and `my_level: int`. Add a `level_history:` list: + +```yaml +level_history: + - {date: 2026-01-15, level: 1, note: "first MCP server"} + - {date: 2026-03-20, level: 3, note: "Cisco 4-server architecture shipped"} + - {date: 2026-05-19, level: 4, note: "Compass MCP server with production HITL"} +``` + +`skill_assessor` appends to this list whenever it changes a grade. Mermaid renders the trajectory: + +````markdown +```mermaid +gantt + title MCP Skill Trajectory + dateFormat YYYY-MM-DD + section Levels + L1 :2026-01-15, 64d + L3 :2026-03-20, 60d + L4 :2026-05-19, 30d +``` +```` + +**What you get:** visual progress per skill. Motivating during plateaus. + +### Idea 6.2: Score trend on JobNotes you've already been scored against + +If you re-score the same JD over time (say, every 6 months), the score should rise as your skills improve. Dataview tracks this: + +````markdown +## JDs I'd score higher on now (vs original) +```dataview +TABLE WITHOUT ID + file.link AS Job, + match_score AS Original, + rescored_at, + rescored_score AS Current +FROM "jobs" +WHERE rescored_at != null AND rescored_score > match_score +SORT (rescored_score - match_score) DESC +``` +```` + +Even better: a "re-score me" MCP tool that lets you periodically refresh select JobNotes and capture the delta. + +### Idea 6.3: Application pipeline funnel + +Mermaid + Dataview combo: + +````markdown +```mermaid +sankey-beta +applied,screen,2 +screen,onsite,1 +onsite,offer,1 +applied,rejected,3 +screen,rejected,1 +``` +```` + +Auto-generated from ApplicationNote status fields. See your funnel conversion at a glance. + +--- + +## Problem 7: "The vault feels like a pile of files, not a product" + +You have 23 JobNotes, 95 SkillNotes, 9 CompanyNotes. The dashboard is the only "UI." Everything else feels like browsing folders. + +### Idea 7.1: Workspaces (saved layouts) + +Obsidian's "Workspaces" feature saves window layouts. Create three: + +- **Morning triage**: left = dashboard, center = pending_approvals output, right = filtered-jobs log +- **Interview prep**: left = JobNote, center = relevant SkillNotes (multiple panes), right = STAR stories +- **Deep work**: left = study-plan, center = today's daily note, right = compass-vault/_meta/agent-log (for forensics) + +Hotkey to switch. Zero code change — pure UI. + +### Idea 7.2: Callouts for high-impact information + +Obsidian renders `> [!callout]` blocks as visually distinct boxes. Use them in JobNotes: + +```markdown +> [!warning] Stretch role +> Score 3.0 is below the apply-now threshold but role family matches. +> Apply only if you can credibly tell a [[LangGraph]] + [[MCP]] story. + +> [!success] Strong fit +> Score 4.5, role-family agent-engineer, candidate has every required skill. +> Apply immediately. + +> [!info] Anti-claim risk +> JD requires [[Fine-Tuning]] which is on the candidate's anti-claim list. +> Acknowledge upfront; pivot to [[Prompt_engineering]] and [[Evals]]. +``` + +`vault_write_node` auto-generates the appropriate callout based on score + role + anti-claim intersection. JobNote at a glance tells you what to do. + +### Idea 7.3: Tier-based CSS + +A snippet in `.obsidian/snippets/compass-tiers.css`: + +```css +.frontmatter[data-tier="apply-now"] { border-left: 4px solid #4caf50; } +.frontmatter[data-tier="6-month"] { border-left: 4px solid #ff9800; } +.frontmatter[data-tier="stretch"] { border-left: 4px solid #2196f3; } +.frontmatter[data-tier="out-of-scope"] { opacity: 0.5; } +``` + +Visual signal. Scan a folder of JobNotes; tier is instant. + +### Idea 7.4: Custom icons via the Iconize plugin + +Folders + tags can have custom icons. Make the vault feel like a product: +- 🎯 `jobs/` +- 🧠 `skills/` +- 🏢 `companies/` +- 📋 `applications/` +- 📝 `_inbox/` +- 🔍 `_meta/` + +Surface-level but reduces cognitive friction during navigation. + +### Idea 7.5: Mermaid pipeline diagram embedded in dashboard + +Auto-update a Mermaid flowchart on the dashboard showing the LATEST pipeline run's path: + +````markdown +```mermaid +flowchart LR + A[scraped: 20] --> B{intake_filter} + B -->|in-scope: 6| C[extract] + B -->|out-of-scope: 14| F[(filtered-jobs.md)] + C --> D[score] + D --> E{hitl} + E -->|paused: 0| P((awaiting human)) + E -->|auto-reject: 4| W[vault] + E -->|approved: 2| T[tailor] + T --> W +``` +```` + +Auto-regenerated by `_append_run_log`. Visual operational dashboard. + +--- + +## Problem 8: "I want the assessor's grading to be transparent" + +The skill_assessor proposes new levels. You see the reasoning, but it's a wall of text per skill. Hard to compare across skills. + +### Idea 8.1: Assessor results dashboard + +After every `assess_skills` run, write the results to `_meta/last-assessor-run.md` as a structured table: + +````markdown +# Last assessor run: 2026-05-19T13:14 + +```dataview +TABLE WITHOUT ID + skill AS Skill, + current_level AS "Now", + proposed_level AS "Proposed", + confidence, + requires_hitl AS HITL? +FROM "_meta/last-assessor-run" +SORT (proposed_level - current_level) DESC +``` +```` + +One glance → all skill movements ranked by delta. Cleanly readable. + +### Idea 8.2: HITL queue for assessor-flagged skills + +The assessor sets `requires_hitl=True` for 2+ level jumps. These should pile up in a "review queue" similar to the JobNote `pending_approvals`. A note `_meta/skill-grade-pending.md` listing them; an MCP tool `apply_skill_grade(skill, accept | reject)`. + +**What you get:** explicit human override on grade changes. The audit trail mirrors HITL on JobNotes. + +### Idea 8.3: Anti-claim explicit on skill page + +`skills/Fine-Tuning.md` has `grade_override: 0` (per `_profile/role-clarifications.md`). Render this prominently with a callout: + +```markdown +> [!danger] Anti-claim — DO NOT claim this skill in applications +> This skill is explicitly disclaimed in `_profile/role-clarifications.md`. +> Compass auto-rejects JDs requiring this skill at score level 0. +``` + +**What you get:** can't accidentally claim something you said you wouldn't. + +--- + +## Problem 9: "I want to capture lessons from building Compass itself" + +Compass IS your portfolio. The decisions you made building it are interview gold. But the decisions live in commit messages, not anywhere you can practice talking about them. + +### Idea 9.1: Promote decision-doc into the learning vault + +Each major architectural decision (the 5 plan-review iterations for the msgpack API, the C1 audit-trail divergence fix, the cosine-pinning silent bug) becomes a `learning-vault/projects/compass/decisions/YYYY-MM-DD-title.md` note. + +Format: +```markdown +# 2026-05-19 — msgpack allowlist API: the 5-iteration silent-bug saga + +## Context +LangGraph 1.2 logged "Deserializing unregistered type" on every paused +thread resume. The fix was supposed to be a one-line allowlist. + +## Decision +Use the constructor form `JsonPlusSerializer(allowed_msgpack_modules= +[(module, classname), ...])`. NOT the post-construction +`with_msgpack_allowlist` method. + +## Alternatives considered +1. Bare module string `['compass.pipeline.state']` — silently fails +2. Class list `[RawJob, ...]` — silently fails +3. Post-construction `serde.with_msgpack_allowlist(...)` — no-op when + default `_allowed_msgpack_modules=True` + +## Why it tipped +Five plan-review iterations with empirical probes finally confirmed only +the constructor form actually suppresses the warning. The deprecation +warning's own suggested fix is misleading. + +## Lesson +"The API does what the docs say" is not a safe assumption for evolving +libraries. Add empirical probes at plan-review time for any new API surface. + +## Skills demonstrated +[[LangGraph]] [[Python]] [[Testing]] [[Production_Concerns]] +``` + +**What you get:** real interview-ready stories. When asked "tell me about a difficult debugging session" — you have 10+ written-up answers, each linked to the skills they demonstrate. + +### Idea 9.2: "What I'd do differently" log + +A single accumulating note `learning-vault/projects/compass/retrospective.md`: + +```markdown +## What I'd do differently in Compass + +1. **Plan-review iterations cost time but save MORE time.** Six iterations + on the 1.B.2 plan caught 4 critical bugs before execution. Without it, + I'd have debugged them at runtime. Default to 3+ iterations. + +2. **Test isolation in pytest.** Raw `module._var = lambda: x` assignments + leak across tests. Use `monkeypatch.setattr` always. Fixed at c4c7fe8; + would have caught earlier if pytest-randomly was installed from day 1. + +3. **LLM behavior bugs need eval harness, not prompt-tuning.** Discovered + in 1.B.2 post-smoke that extract under-extracts on best-fit JDs. Tried + to ad-hoc fix the prompt; rolled back. Building the 30-JD eval set + first (Phase 2.A) is the right order. +``` + +**What you get:** lessons internalized + interview-ready. "What's a hard lesson you've learned recently?" — read directly from this note. + +--- + +## Recommended Layered Rollout + +If P1+P2+P3 from Part 1 is the foundation, here's what to layer on after: + +### Layer 1 (high value, low effort, ~2 hours total) +- **Idea 1.3** "Today's study target" dashboard widget (~10 min) +- **Idea 3.2** "This week's best fits" Dataview panel (~5 min) +- **Idea 7.2** Auto-callouts on JobNotes (~30 min in writer + smoke test) +- **Idea 7.3** Tier-based CSS snippet (~5 min) +- **Idea 8.1** Assessor results dashboard table (~30 min — write structured output) + +### Layer 2 (medium value, more time, ~5 hours) +- **Idea 1.1** Daily note → skill evidence sync script (~30 min) +- **Idea 2.1** STAR story bank scaffolding (~1 hour — establish convention + 1-2 example stories) +- **Idea 4.1** Decision journal convention + dashboard panel (~30 min) +- **Idea 9.1** Promote 3-5 Compass architectural decisions to learning vault (~2 hours) +- **Idea 3.1** Quick-capture inbox + inbox_scraper (~1 hour) + +### Layer 3 (when daily use exposes the need) +- **Idea 5.1** PersonNote system (once your network grows) +- **Idea 6.1** Skill-level over time + Mermaid trajectory +- **Idea 6.3** Application funnel Sankey +- **Idea 7.5** Mermaid pipeline diagram on dashboard +- **Idea 8.2** HITL queue for assessor grade changes + +### Layer 4 (portfolio polish, do before public README) +- **Idea 4.3** Offer comparison table (when you have 2+ offers) +- **Idea 7.1** Workspaces (after layouts stabilize) +- **Idea 9.2** "What I'd do differently" retrospective (interview material) + +--- + +## What I'd actually do first if I were you + +If you only have **30 minutes** today and want concrete progress on the system feeling more useful: + +1. **Idea 7.2 + 7.3** (callouts + CSS) — 35 minutes, immediately visible visual lift on every JobNote +2. **Idea 1.3 + 3.2** (two new dashboard panels) — 15 minutes, daily-flow improvement + +If you have **a full afternoon (~3 hours)** and want a single multi-component improvement: + +- **Ship Part 1's P1+P2+P3 bundle.** That's the foundation. Everything else in this Part 2 doc layers on top of P1's wikilink graph more powerfully than it would standalone. + +If you have **a full day** and want to invest in interview prep: + +- **Idea 2.1** (STAR story bank) + **Idea 9.1** (promote 3-5 Compass decisions) — 4-5 hours combined. By end of day you have 8-10 written-up stories tagged by skill, ready to deploy in a Friday recruiter screen. + +--- + +## Why this matters for the job-search use case specifically + +Most candidates have: +- A resume (one-size-fits-all) +- A list of LinkedIn jobs (chronological, not score-sorted) +- A vague sense of which skills they should learn next +- No record of why they applied where, or what worked in interviews + +What Compass + Obsidian leverage gives you: +- Per-JD score + tailored framing +- Score-sorted recommendation engine +- Empirical gap plan from real JD market signal +- Decision audit trail (which roles, why, what happened) +- Network graph (people → companies → roles) +- Interview prep that maps directly to JobNote requirements +- Skill trajectory tracking + +This is the kind of system a Sierra/Decagon/Cognition hiring manager wants to *see*. Not just hear about — but click through. The Obsidian rendering layer IS the demo. From 8e6de54ff18c94aea17271b5059b397fc3233191 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 14:10:56 -0500 Subject: [PATCH 29/46] docs: 2-week portfolio sprint handoff doc --- docs/TWO_WEEK_SPRINT.md | 391 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 docs/TWO_WEEK_SPRINT.md diff --git a/docs/TWO_WEEK_SPRINT.md b/docs/TWO_WEEK_SPRINT.md new file mode 100644 index 0000000..c2ee9c3 --- /dev/null +++ b/docs/TWO_WEEK_SPRINT.md @@ -0,0 +1,391 @@ +# Compass 2-Week Portfolio Sprint — Handoff Doc + +> **You're picking this up in a fresh chat. Read this top-to-bottom first. It tells you what state the project is in, what we're shipping over the next 14 days, what to do today, and how to work with the user.** + +--- + +## TL;DR for a fresh agent + +- **Goal:** ship a complete portfolio-ready Compass in 14 days (ending ~2026-06-02). +- **State today (2026-05-19):** `phase-1b2-rag` tag landed; 7 commits past it on `phase-1b2-rag` branch; 259 tests passing; ruff clean; vault state coherent. +- **Daily commitment:** 2-4 focused hours, every day, including weekends. +- **What's done:** Phases 0.A, 0.B, 1.A, 1.B.1, 1.B.2. Skill assessor wired with real evidence URIs (4 skills). B6 role_family migration shipped. +- **What remains:** Obsidian Part 1 → Phase 1.B.3 → Phase 2.A → Phase 2.B (+ folded Part-2 ideas) → Phase 2.C blog post. +- **Today's first action:** start Day 1 of the sprint (Obsidian Part 1 — JobNote/SkillNote body wikilinks + dashboard panels + auto-tags). + +--- + +## The 14-day plan + +| Day | Phase | Work | Hours | +|---|---|---|---| +| 1 | Obsidian P1 | Writer change: add `## Skills` section with `[[Python]]·[[LangGraph]]` wikilinks to JobNote body. Backfill migration script for 23 existing JobNotes. | 3 | +| 2 | Obsidian P1+P2+P3 | SkillNote body gets `## Jobs requiring this skill` list (regenerated by `gap_aggregator._sync_skill_counters`). Dashboard panels: "Skill rarity" + "Rare/specialized." Auto-tags on JobNote frontmatter. Backfill SkillNote bodies + JobNote tags. Tests. **Ship the Obsidian foundation.** | 3 | +| 3 | Phase 1.B.3 | Plan-write the phase. Langfuse callback fix (`compass/pipeline/graph.py:_langfuse_config` — `host=`/`secret_key=` kwargs cause `TypeError`). URL dedup for `_meta/filtered-jobs.md` (intake_filter writes log but never reads — same OUT titles hit LLM every run). | 3 | +| 4 | Phase 1.B.3 | `claim_pending` atomic transition in `compass/hitl/state_store.py` for the I4 double-resume race. Regression tests. | 2 | +| 5 | Phase 1.B.3 | `modal_app.py` at repo root + Modal Secrets for `OPENROUTER_API_KEY` / `LANGFUSE_*`. Cron schedules: daily scrape 9 AM CT, weekly skill_assessor Sunday 2 AM CT. First Modal deploy. | 3 | +| 6 | **USE IT** | Run pipeline manually 2-3 times. Apply to 1-2 real jobs via `add_application` MCP tool — first time exercising that codepath on real data. | 1-2 | +| 7 | Phase 2.A | Eval harness scaffolding. `compass/evals/dataset.json` schema. Runner code (`compass/evals/runner.py`) computing score MAE + skill recall. Label 10 of the existing JobNotes. | 4 | +| 8 | Phase 2.A | Label 10 more JDs (target: 20 labeled). Run eval harness → first MAE/recall baseline. Log results to Langfuse if working. | 3 | +| 9 | Phase 2.A | Prompt-tune `extract_node` (fix B1+B2) using eval feedback. Iterate 2-3 times; stop when MAE drops or stops improving. Document baseline in `docs/EVAL_BASELINE.md`. | 3 | +| 10 | Phase 2.B | Rewrite top-level `README.md`. Architecture diagram (Mermaid). Capture screenshots: dashboard, graph view, master gap plan, JobNote with audit trail. | 3 | +| 11 | Phase 2.B + folded Part-2 | Promote 3-5 Compass architectural decisions to `learning-vault/projects/compass/decisions/*.md` (the msgpack saga, C1 audit-trail divergence, cosine-pinning silent bug, adversarial review methodology). Wire them as `evidence:` URIs to relevant skills. Run `assess_skills` → screenshot the grade changes. | 3 | +| 12 | Phase 2.B | Daily-note → skill evidence auto-sync script. Assessor results dashboard panel. Public Langfuse trace URL embedded in README. Eval baseline numbers in README. | 3 | +| 13 | Phase 2.C | Blog post draft (1500-2000 words) — *"Building an agent that grades my skills against the live job market."* Outline + first draft. | 4 | +| 14 | Phase 2.C | Edit blog post. Final README polish. Make repo public on GitHub. Push. **Done.** | 3 | + +**Total: ~40 hours over 14 days.** Realistic if you commit to daily cadence. + +--- + +## Where the project actually is (the honest state) + +### Code +- Tag: `phase-1b2-rag` +- HEAD: 7 commits past the tag on `phase-1b2-rag` branch: + - `3828d8f` intake_filter OUT keyword widening (5 leak titles caught) + - `7ebd4a1` docs: expanded `KNOWN_DATA_QUALITY_ISSUES.md` + - `db3b132` B6 fix: role_family migration script + `gap_aggregator` skips out-of-scope + - `5b39d58` docs: PROJECT_OVERVIEW.md (510 lines) + - `daa4bb5` style: TYPE_CHECKING guard on test_gap_aggregator_role_filter.py + - `ad297ef` docs: OBSIDIAN_LEVERAGE.md (P1-P8) + - `19bcf16` docs: OBSIDIAN_LEVERAGE_PART_2.md (use-case oriented) +- Tests: **259 passing** (`uv run pytest -q`) +- Ruff: clean (`uv run ruff check && uv run ruff format --check`) +- pytest-randomly: NOT installed; consider `uv add --dev pytest-randomly` early to catch isolation leaks + +### Vault state +- `~/Documents/compass-vault/jobs/` — 23 JobNotes; 5 migrated to `role_family: out-of-scope` (correctly drop from gap plan) +- `~/Documents/compass-vault/skills/` — 95 SkillNotes; 4 have populated `evidence:` URIs (MCP, LangGraph, Eval_harness, HiTL) +- `~/Documents/compass-vault/applications/` — empty; `add_application` codepath never exercised on real data +- `~/Documents/compass-vault/study-plans/master-gap-plan.md` — fresh; top gaps: TypeScript (9 jobs), Go (5), RAG (4) +- `~/.compass/checkpoints.db` — 20 KB (purged from 13.8 MB on 2026-05-19; 0 stale threads) +- `~/.compass/hitl.db` — 0 pending approvals + +### What's known broken (deferred, documented) +- **Langfuse callback** — every run logs `LangchainCallbackHandler.__init__() got an unexpected keyword argument 'host'`, runs without traces. Fix in Day 3 (Phase 1.B.3). +- **`state_store.claim_pending` missing** — Modal cron + MCP approve will race. Fix in Day 4 (Phase 1.B.3). +- **B1-B3 LLM behavior bugs** — `extract` under-extracts on best-fit JDs; mis-reads OR-lists; `score` sometimes marks candidate-strong skills as missing. See `docs/KNOWN_DATA_QUALITY_ISSUES.md`. Fix in Day 7-9 (Phase 2.A via eval harness). + +--- + +## Required reading (in order, before Day 1) + +1. **`/Users/akmini/Documents/compass/CLAUDE.md`** — repo conventions, what you can/can't do +2. **`/Users/akmini/Documents/compass/docs/PROJECT_OVERVIEW.md`** — the 510-line plain-language walkthrough of the whole project. Get the architecture in your head. +3. **`/Users/akmini/Documents/compass/docs/KNOWN_DATA_QUALITY_ISSUES.md`** — what's documented as deferred, with severity + fix surface +4. **`/Users/akmini/Documents/compass/docs/OBSIDIAN_LEVERAGE.md`** — P1-P8 foundation proposals; **Day 1-2 work uses P1+P2+P3 from here** +5. **`/Users/akmini/Documents/compass/docs/OBSIDIAN_LEVERAGE_PART_2.md`** — deeper use-case ideas; **Day 11-12 folds in Ideas 9.1 + 1.1 + 8.1 from here** +6. **`/Users/akmini/Documents/compass/docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md`** — the authoritative master spec; the Definition of Done lives there + +You can skim PHASE_0_COMPLETE.md / PHASE_1A_COMPLETE.md / phase-1b1 plan / phase-1b2-rag plan for context but the PROJECT_OVERVIEW.md is the consolidated version. + +--- + +## Critical lessons carried forward (do not relearn) + +These were earned the hard way across Phases 0 → 1.B.2. Every one of them cost a real bug or a real review iteration: + +1. **Tests check shape, smoke tests check counts, neither catches data-correctness bugs on real inputs.** After ANY claim a phase is "ready," run adversarial probing on real vault output before believing the claim. The pattern repeats every phase: rubber-stamp review finds 0 bugs; adversarial review with grep-before-assert finds 3-7 real defects. + +2. **Module-level `from compass.config import X` freezes the value at import time** and silently breaks the `temp_*` test fixtures. Always late-bind via `import compass.config as cfg; cfg.X` inside function bodies. The `temp_vault` fixture monkeypatches `compass.config.VAULT_PATH`; module-level capture defeats it. + +3. **A LangGraph graph compiled without a checkpointer silently breaks `interrupt()`.** Never compile `build_graph()` at module level. Call it INSIDE `async with AsyncSqliteSaver.from_conn_string(...)` only. + +4. **`or`-fallback against falsy values is a recurring trap.** `state.get("score_threshold") or SCORE_THRESHOLD` would mis-fallback on `0.0`. Use `is None` checks always. + +5. **JSON-serializability of MCP return shapes** matters. FastMCP can't transmit `datetime` or Pydantic instances. Use `model_dump(mode="json")` or hand-build plain dicts. + +6. **Production code must be terse, WHY-only docstrings.** Match `compass/hitl/state_store.py` calibration. No plan-meta narrative ("verified at planning time"). No defensive paranoia (`try: ... except Exception: pass`). This is a portfolio repo — recruiters will read the source. + +7. **`with_msgpack_allowlist` is a no-op when default `allowed_msgpack_modules=True`.** Pass the allowlist via the CONSTRUCTOR: `JsonPlusSerializer(allowed_msgpack_modules=[(module, classname), ...])`. The `_warned_unregistered_types` set is per-session; clear it before measuring warning suppression in tests. + +8. **Chroma's `get_or_create_collection` does NOT update an existing collection's metadata.** Inspect existing metadata; drop+recreate if metric mismatches. + +9. **Derived-field staleness**: counters (`appears_in_jobs`, `roles_seen`) are recomputed at gap-aggregator time. But `role_family` is stored once at intake and never reconciled. If you add a new "set-once classification" field in future work, you need a migration script. + +10. **Adversarial plan review catches defects pre-execution.** Cost: 30-60 min per pass. Benefit: 1-7 real bugs per pass. Cap at 3 iterations before escalating. + +--- + +## Communication style with the user + +Read the user's CLAUDE.md preferences. Specifically: + +- **Be direct and concise.** Don't restate what they said. +- **Lead with the answer or action.** Not the reasoning. +- **Don't add comments / docstrings / type annotations to code you didn't change.** +- **Don't create files unless necessary. Prefer editing existing ones.** +- **When you save to the vault: tell them path + why.** + +What they DON'T want: +- Long menus of options when one recommendation is obviously right +- Excessive caveats +- AI-slop code (defensive paranoia, speculative helpers, chatty comments, plan-meta narrative in production) +- Apologetic preambles ("I apologize for...") +- Mention of the date/time unless asked + +What they DO want: +- Honest harsh assessments when asked +- Real adversarial reviews that find real bugs +- Direct recommendations +- Code that reads like a senior wrote it + +When unsure between two approaches, briefly state both and recommend one. Don't paralyze them with options. + +--- + +## Workflow for each day's work + +The user has been using subagent-driven development heavily. Follow the same pattern: + +1. **Read the day's task** in this doc. Note the files affected. +2. **For each phase, write a per-phase implementation plan** in `docs/superpowers/plans/YYYY-MM-DD-.md` using the `superpowers:writing-plans` skill. +3. **Dispatch a plan-document-reviewer** (general-purpose agent) with the actual codebase grepping required. 2 passes minimum. +4. **Apply fixes from review.** +5. **Execute via `superpowers:subagent-driven-development`**: + - Fresh implementer subagent per task + - Combined spec+quality reviewer subagent between tasks + - Pause before any task that hits production (real LLM, real vault writes) +6. **Live smoke** at the end of each phase. Snapshot vault + DBs first. Verify on real data. +7. **Tag** the phase: `phase-1b2-obsidian-p1`, `phase-1b3-automation`, `phase-2a-eval`, `phase-2b-polish`, `phase-2c-public`. + +--- + +## Day 1 — START HERE + +### Goal + +Add `## Skills` section with `[[Python]] · [[LangGraph]] · [[MCP]]` style wikilinks to every JobNote body. Backfill the 23 existing JobNotes via a one-shot migration script. This is the foundation that makes the Obsidian graph view meaningful. + +### Files to modify + +- `compass/vault/writer.py` — `write_job_note` function (currently around line 60-105). Add a body section between `jd_summary` and `## Full JD`. +- `scripts/migrate_jobnote_skills_section.py` — new one-shot migration (model after `scripts/migrate_role_family.py`). +- `tests/vault/test_writer.py` — add 1-2 tests verifying the new section. + +### Expected output + +A JobNote written by `write_job_note` should look like: + +```markdown +--- +... frontmatter ... +--- +# Sierra — Software Engineer, Agent Architecture + +...jd_summary text... + +## Skills + +**Required:** [[Python]] · [[LangGraph]] · [[MCP]] · [[Sub-agents]] +**Nice to have:** [[FastAPI]] · [[Pydantic AI]] +**Matched:** [[Python]] · [[MCP]] +**Missing:** [[LangGraph]] · [[Sub-agents]] + +## Full JD + +...full JD text... +``` + +Sections render `[[Python]]` as a clickable wikilink in Obsidian. Empty categories (e.g. no nice-to-have skills) should be omitted from the rendered output, not shown as "Nice to have: (none)". + +### Migration script behavior + +`scripts/migrate_jobnote_skills_section.py`: +- Walks `~/Documents/compass-vault/jobs/*.md` +- For each, reads frontmatter `skills_required`, `skills_nice_to_have`, `skills_matched`, `skills_missing` +- Replaces existing `## Skills` section if present, OR inserts before `## Full JD` if not +- Dry-run by default (lists changes); `--apply` to commit +- Same pattern as `scripts/migrate_role_family.py` + +### Test cases + +```python +def test_jobnote_body_has_skills_section(temp_vault): + note = JobNote( + company="Test", title="Engineer", url="x://1", source="manual", + date_found=date.today(), match_score=4.0, + skills_required=["Python", "LangGraph"], + skills_nice_to_have=["FastAPI"], + skills_matched=["Python"], + skills_missing=["LangGraph"], + ) + path = write_job_note(note) + body = path.read_text() + assert "## Skills" in body + assert "[[Python]]" in body + assert "[[LangGraph]]" in body + assert "[[FastAPI]]" in body + + +def test_jobnote_body_omits_empty_skill_categories(temp_vault): + note = JobNote( + company="Test", title="Engineer", url="x://2", source="manual", + date_found=date.today(), match_score=2.0, + skills_required=["Python"], + skills_nice_to_have=[], # empty + skills_matched=[], # empty + skills_missing=["Python"], + ) + path = write_job_note(note) + body = path.read_text() + assert "**Nice to have:**" not in body # empty category omitted + assert "**Matched:**" not in body + assert "**Missing:** [[Python]]" in body +``` + +### Execution + +1. Write the implementation plan via `superpowers:writing-plans` (this is small enough that a tight plan with 4-5 task steps is fine; doesn't need full multi-task plan structure) +2. One adversarial review pass (general-purpose agent, grep against the writer code) +3. Implement (subagent-driven if helpful; or inline since this is small) +4. Run dry-run migration: `uv run python -m scripts.migrate_jobnote_skills_section` +5. Inspect output; verify 23 JobNotes would be updated cleanly +6. Apply: `uv run python -m scripts.migrate_jobnote_skills_section --apply` +7. Open one JobNote in Obsidian, confirm graph view shows new edges +8. Commit + +### Definition of done for Day 1 + +- `compass/vault/writer.py` modified to render `## Skills` section with wikilinks +- 2 unit tests passing +- 23 existing JobNotes backfilled via migration +- Open Obsidian → click `[[Python]]` on any JobNote → lands on `skills/Python.md` and sees Linked Mentions populated +- Full test suite: 259+ passing +- Ruff clean +- Committed + +**Estimated time: 3 hours including the plan + review cycle.** + +--- + +## Day 2 — Obsidian P1 (continued) + P2 + P3 + +Once Day 1 is committed: + +1. **SkillNote body update**: extend `compass/analysis/gap_aggregator._sync_skill_counters` to also write a `## Jobs requiring this skill` section to each SkillNote body. **Preserve existing body content** (the seeded notes have descriptions like `_Category: mcp · Tier-2 demand: highest_`). Insert or replace just the `## Jobs requiring this skill` block. + + Format: + ```markdown + ## Jobs requiring this skill + + - [[2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f|Sierra — Software Engineer, Agent Architecture]] · score 3.0 · apply-now + - [[2026-05-06-cognition-Software_Engineer-7c26fdaf|Cognition — Software Engineer]] · score 3.0 · apply-now + ``` + +2. **Dashboard panels** (`~/Documents/compass-vault/dashboard.md`): add two Dataview blocks. See `docs/OBSIDIAN_LEVERAGE.md` P2 for the exact queries. + +3. **Auto-tags on JobNote frontmatter** (`compass/pipeline/nodes/vault_write.py`): generate `#tier/apply-now`, `#fit/strong`, `#role/agent-engineer`, `#decision/approved` style tags. Backfill the 23 existing JobNotes via the existing migration pattern. + +4. Tests for each. + +5. Tag `phase-1b2-obsidian-p1` when all P1+P2+P3 work shipped. + +--- + +## Day 3+ planning + +When you reach each phase, write the per-phase implementation plan using `superpowers:writing-plans`: + +- Day 3-5: `docs/superpowers/plans/2026-05-22-compass-phase-1b3-automation.md` +- Day 7-9: `docs/superpowers/plans/2026-05-26-compass-phase-2a-eval-harness.md` +- Day 10-12: `docs/superpowers/plans/2026-05-29-compass-phase-2b-portfolio-polish.md` +- Day 13-14: `docs/superpowers/plans/2026-06-01-compass-phase-2c-blog-post.md` + +Each plan needs 2 adversarial review passes minimum. Compress from the previous 3-6 pattern; accept slightly higher residual risk for the 2-week timeline. + +--- + +## What you SHOULD NOT do + +- **Don't add scope.** When tempted to add "just one more Obsidian feature" or "one more JD to label" or "one more scraper," stop. Stick to the plan. +- **Don't touch the deferred B1-B3 LLM bugs ad-hoc.** Wait for Phase 2.A's eval harness. Ad-hoc prompt tuning without measurement is whack-a-mole. +- **Don't refactor "while you're in there."** If a piece of code is ugly but works, leave it. Phase 2.B's polish phase handles cosmetic improvements. +- **Don't create more brainstorm/design docs.** Two Obsidian leverage docs exist. The implementation plans are per-phase, not master docs. +- **Don't relitigate decisions.** The 2-week plan is the plan. Within each phase, the per-phase plan is the plan. Execute. +- **Don't bypass the adversarial review step.** It's compressed from 3-6 passes to 2, but not zero. The pattern of "rubber-stamp review finds 0 bugs; adversarial review finds 5" repeats every phase. + +--- + +## What to do when something goes wrong + +### Tests fail unexpectedly +- Check `uv run pytest --tb=short` for the failure mode +- Test isolation leak? Use `monkeypatch.setattr` not raw `module._var = ...` (see commit `c4c7fe8`) +- Schema validation failure? Some new field needed in `compass/vault/schemas.py`? + +### Pipeline run fails +- Check `~/Documents/compass-vault/_meta/agent-log.md` for the error +- Check `~/Documents/compass-vault/_meta/pipeline-runs.md` for the run summary row +- Langfuse callback errors are expected pre-Day-3; ignore them + +### A subagent reports DONE but you don't trust it +- Run the actual verification commands yourself +- `git diff` to see the real changes +- `uv run pytest -q` to confirm tests pass +- Inspect real vault output if the change affects vault state + +### You hit a langgraph / chromadb / pydantic-ai library quirk +- The lesson from Phases 1.B.1 + 1.B.2: probe empirically before assuming. Five plan-review iterations on the msgpack API surfaced the real working form. +- If a documented API doesn't work, run a one-liner probe with `uv run python -c "..."` to find the actual contract. + +### You realize you missed a bug class during review +- Document it in `docs/KNOWN_DATA_QUALITY_ISSUES.md` with severity + fix surface +- If P0 (blocks current phase): stop and fix +- If lower: continue; fix in Phase 2.A or post-launch + +--- + +## Resources + +- **Phase 1.B.2 plan** (for plan format calibration): `docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md` +- **Phase 1.B.1 plan**: `docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md` +- **Master spec**: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- **CLAUDE.md**: `/Users/akmini/Documents/compass/CLAUDE.md` +- **The user's vault**: `~/Documents/compass-vault/` +- **The user's learning vault**: `~/Documents/learning-vault/` (read-only via `learning-vault://` URIs) + +--- + +## Pre-flight check before Day 1 + +Run this: + +```bash +cd /Users/akmini/Documents/compass +git status # expect: clean +git branch --show-current # expect: phase-1b2-rag +git log --oneline -5 # confirm: 19bcf16 is HEAD +uv run pytest -q # expect: 259 passed +uv run ruff check # expect: All checks passed +ls ~/Documents/compass-vault/jobs/ | wc -l # expect: 23 +``` + +If all green: start Day 1. + +If any red: STOP. Diagnose before proceeding. + +--- + +## Communication first interaction with the user + +When the user opens a fresh chat and you have this doc, your first message should be something like: + +> Read the handoff. State: `phase-1b2-rag` HEAD `19bcf16`, 259 tests pass, vault coherent. Day 1 starts with Obsidian Part 1 — JobNote body wikilinks + migration. Estimated 3 hours. +> +> Want me to write the Day 1 implementation plan and start, or do you want to review the handoff first? + +Don't ask permission for things the doc already authorized. Don't restate the plan. Get to work. + +--- + +## Honest closing note + +The codebase is in good shape entering this sprint. 7+ adversarial review cycles. Known issues documented. Tests green. No surprises waiting. + +The biggest risk over the next 14 days is **scope creep**, not technical debt. Hold the line on this plan. Ship daily. Let imperfections in adjacent features stay until Phase 2.B polish or post-launch iteration. + +Two weeks from now: a working agentic career coach in your daily workflow + a public repo + a published blog post + measurable eval baselines. That's the bar. + +Go. From 2dd50ca15a7f9fd73796adc427adf708b143dc78 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 14:21:34 -0500 Subject: [PATCH 30/46] feat(vault): render `## Skills` wikilink section in JobNote bodies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Day-1 Obsidian P1: writer now emits a `## Skills` block between the LLM summary and `## Full JD`, rendering required / nice-to-have / matched / missing as `[[Python]] · [[LangGraph]]` wikilinks so the Obsidian graph view edges JobNotes to SkillNotes. Empty categories are omitted; skill names with characters that get rewritten by `_safe_segment` use the alias form `[[AWS_Bedrock|AWS Bedrock]]` so the link resolves. Backfilled 18 existing JobNotes via scripts/migrate_jobnote_skills_section (idempotent — replaces an existing block if present, inserts before `## Full JD` otherwise). --- compass/vault/writer.py | 35 +++++++ scripts/migrate_jobnote_skills_section.py | 118 ++++++++++++++++++++++ tests/vault/test_writer.py | 64 ++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 scripts/migrate_jobnote_skills_section.py diff --git a/compass/vault/writer.py b/compass/vault/writer.py index 579d029..570a8b6 100644 --- a/compass/vault/writer.py +++ b/compass/vault/writer.py @@ -57,6 +57,38 @@ def _to_metadata(model: BaseModel) -> dict: return model.model_dump(mode="json", by_alias=True) +def _wikilink(skill: str) -> str: + """Render a skill as an Obsidian wikilink pointing at skills/.md. + + SkillNotes are stored under filenames produced by `_safe_segment`. When the + skill name contains characters that get rewritten (e.g. "AWS Bedrock" → + "AWS_Bedrock", "C++" → "C__"), emit the alias form `[[target|display]]` so + the link resolves AND the display matches what the user typed. + """ + target = _safe_segment(skill) + return f"[[{target}|{skill}]]" if target != skill else f"[[{skill}]]" + + +def _render_skills_section(note: JobNote) -> str: + """Render `## Skills` body block. Empty categories are omitted entirely.""" + rows: list[tuple[str, list[str]]] = [ + ("Required", note.skills_required), + ("Nice to have", note.skills_nice_to_have), + ("Matched", note.skills_matched), + ("Missing", note.skills_missing), + ] + lines = ["## Skills", ""] + any_row = False + for label, skills in rows: + if not skills: + continue + any_row = True + lines.append(f"**{label}:** " + " · ".join(_wikilink(s) for s in skills)) + if not any_row: + return "" + return "\n".join(lines) + "\n" + + def write_job_note(note: JobNote, full_description: str | None = None) -> Path: """Write a JobNote to vault/jobs/. Idempotent on URL — same URL overwrites the same file. @@ -80,6 +112,9 @@ def write_job_note(note: JobNote, full_description: str | None = None) -> Path: target = jobs_dir / _job_filename(note) body = f"# {note.company} — {note.title}\n\n{note.jd_summary}\n" + skills_block = _render_skills_section(note) + if skills_block: + body += f"\n{skills_block}" if full_description: body += f"\n## Full JD\n\n{full_description.strip()}\n" post = frontmatter.Post(content=body) diff --git a/scripts/migrate_jobnote_skills_section.py b/scripts/migrate_jobnote_skills_section.py new file mode 100644 index 0000000..03961d1 --- /dev/null +++ b/scripts/migrate_jobnote_skills_section.py @@ -0,0 +1,118 @@ +"""One-time migration: backfill the `## Skills` wikilink section into existing JobNote bodies. + +Day-1 Obsidian P1 work added a `## Skills` block to `write_job_note` that +renders `[[Python]] · [[LangGraph]]` style wikilinks so the Obsidian graph view +edges JobNotes to SkillNotes. Existing JobNotes written before this change +have the right frontmatter but no body section; this script rewrites them +in-place by re-rendering with the current `write_job_note` logic. + +Idempotent: re-running is safe — replaces an existing `## Skills` block if +present, inserts before `## Full JD` (or appends) otherwise. + +Dry-run by default; --apply to commit. + +Usage: + uv run python -m scripts.migrate_jobnote_skills_section # dry-run + uv run python -m scripts.migrate_jobnote_skills_section --apply # commit +""" + +from __future__ import annotations + +import argparse +import re +import sys +from typing import TYPE_CHECKING + +import frontmatter + +from compass.config import VAULT_PATH +from compass.vault.schemas import JobNote +from compass.vault.writer import _render_skills_section + +if TYPE_CHECKING: + from pathlib import Path + + +# Matches `## Skills` heading through (but not including) the next `## ` heading or EOF. +_SKILLS_BLOCK = re.compile(r"(?ms)^## Skills\s*\n.*?(?=^## |\Z)") + + +def _splice_skills_section(body: str, rendered: str) -> str: + """Insert or replace the `## Skills` block in a JobNote body. + + - If a `## Skills` block already exists, replace it (re-runs stay clean). + - Else, insert before the first `## ` heading (typically `## Full JD`). + - Else, append at end. + - If `rendered` is empty (note has zero skills in all four lists), strip + any existing block and return. + """ + if _SKILLS_BLOCK.search(body): + # The regex consumes through the blank line separator before the next + # heading. Re-add a single blank line so subsequent re-runs are no-ops. + # If `rendered` is empty (no skills), substitute with empty — collapse + # leftover whitespace below. + replacement = (rendered + "\n") if rendered else "" + new_body = _SKILLS_BLOCK.sub(replacement, body, count=1) + return re.sub(r"\n{3,}", "\n\n", new_body) + + if not rendered: + return body + + next_heading = re.search(r"(?m)^## ", body) + block = rendered if rendered.endswith("\n") else rendered + "\n" + if next_heading: + idx = next_heading.start() + return body[:idx] + block + "\n" + body[idx:] + # No subheading at all — append after a blank line. + return body.rstrip() + "\n\n" + block + + +def _rewrite_body(path: Path) -> tuple[bool, str]: + """Return (changed, new_full_text). Does not write.""" + post = frontmatter.load(path) + try: + note = JobNote.model_validate(post.metadata) + except Exception as exc: + print(f" SKIP {path.name}: frontmatter does not validate as JobNote ({exc})") + return False, "" + rendered = _render_skills_section(note) + new_body = _splice_skills_section(post.content, rendered) + if new_body == post.content: + return False, "" + post.content = new_body + return True, frontmatter.dumps(post) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Actually rewrite the JobNotes") + args = parser.parse_args() + + jobs_dir = VAULT_PATH / "jobs" + paths = sorted(jobs_dir.glob("*.md")) + changes: list[tuple[Path, str]] = [] + for path in paths: + changed, new_text = _rewrite_body(path) + if changed: + changes.append((path, new_text)) + + if not changes: + print(f"All {len(paths)} JobNote(s) already have a current `## Skills` section.") + return 0 + + print(f"Will update {len(changes)} of {len(paths)} JobNote(s):\n") + for path, _ in changes: + print(f" {path.name}") + + if not args.apply: + print(f"\nDry-run. Pass --apply to rewrite {len(changes)} files.") + return 0 + + for path, new_text in changes: + path.write_text(new_text, encoding="utf-8") + print(f"\nMigrated {len(changes)} JobNote(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/vault/test_writer.py b/tests/vault/test_writer.py index e32ce67..87b7b8d 100644 --- a/tests/vault/test_writer.py +++ b/tests/vault/test_writer.py @@ -83,6 +83,70 @@ def test_write_job_note_omits_full_jd_section_when_not_provided(temp_vault): assert "## Full JD" not in path.read_text() +def test_jobnote_body_has_skills_section_with_wikilinks(temp_vault): + """Day 1 Obsidian P1: JobNote body renders a `## Skills` block with + wikilinks so the graph view shows JobNote → SkillNote edges. Section + sits between the LLM summary and `## Full JD`.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["Python", "LangGraph"], + skills_nice_to_have=["FastAPI"], + skills_matched=["Python"], + skills_missing=["LangGraph"], + ) + path = write_job_note(note, full_description="raw jd text here") + body = path.read_text() + assert "## Skills" in body + assert "[[Python]]" in body + assert "[[LangGraph]]" in body + assert "[[FastAPI]]" in body + assert "**Required:**" in body + assert "**Nice to have:**" in body + assert "**Matched:**" in body + assert "**Missing:**" in body + # ## Skills must appear before ## Full JD + assert body.index("## Skills") < body.index("## Full JD") + + +def test_jobnote_body_omits_empty_skill_categories(temp_vault): + """Empty categories are omitted entirely — no '(none)' placeholder.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["Python"], + skills_nice_to_have=[], + skills_matched=[], + skills_missing=["Python"], + ) + path = write_job_note(note) + body = path.read_text() + assert "**Nice to have:**" not in body + assert "**Matched:**" not in body + assert "**Required:** [[Python]]" in body + assert "**Missing:** [[Python]]" in body + + +def test_jobnote_skill_wikilink_aliases_unsafe_filenames(temp_vault): + """Skills with spaces or punctuation resolve to a safe-segment filename; + the wikilink must point at the actual file via alias form so the link + resolves AND the display matches the user-facing skill name.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["AWS Bedrock", "C++"], + skills_matched=[], + skills_missing=[], + skills_nice_to_have=[], + ) + path = write_job_note(note) + body = path.read_text() + assert "[[AWS_Bedrock|AWS Bedrock]]" in body + # `_safe_segment` collapses non-word runs to `_` then strips trailing `_`, + # so "C++" becomes the file "C.md" — alias keeps the original display. + assert "[[C|C++]]" in body + + def test_write_job_note_idempotent_on_duplicate_url(temp_vault): """Writing the same URL twice should overwrite the same file, not create a second.""" from compass.vault.writer import write_job_note From 713e37675f187533400ef647fdb63cd90d08ac89 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 15:44:22 -0500 Subject: [PATCH 31/46] feat(obsidian): SkillNote body backlinks (P2) + JobNote auto-tags (P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2 — `gap_aggregator._sync_skill_backlinks` writes a `## Jobs requiring this skill` block to every SkillNote body, listing every JobNote whose required/ nice-to-have skills include the canonical name. Sorted by match_score DESC. Preserves existing body content above the block (seed notes carry category descriptions). Idempotent — re-runs replace the block in place. P3 — `vault_write_node` now auto-generates Obsidian-tag-pane-filterable tags on every JobNote write: `#tier/{apply-now,opportunistic,...}`, `#fit/{strong,decent,stretch,weak}`, `#role/{agent-engineer,...}`, `#decision/{approved,timed_out,...}`. Empty role_family and absent hitl_decision correctly omit their tag. Dashboard panels for skill rarity + rare/specialized skills added to compass-vault/dashboard.md (zero-code Dataview blocks). 8 new tests. 270 passing. Ruff clean. --- compass/analysis/gap_aggregator.py | 84 ++++++++++++++++ compass/pipeline/nodes/vault_write.py | 41 +++++++- tests/analysis/test_skill_backlinks.py | 130 +++++++++++++++++++++++++ tests/pipeline/test_auto_tags.py | 61 ++++++++++++ 4 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 tests/analysis/test_skill_backlinks.py create mode 100644 tests/pipeline/test_auto_tags.py diff --git a/compass/analysis/gap_aggregator.py b/compass/analysis/gap_aggregator.py index 193c960..065c603 100644 --- a/compass/analysis/gap_aggregator.py +++ b/compass/analysis/gap_aggregator.py @@ -269,12 +269,96 @@ def regenerate(write: bool = True) -> tuple[list[GapPlanEntry], str]: rendered = render_master_plan(entries, jobs_n=len(jobs)) if write: _sync_skill_counters(entries) + _sync_skill_backlinks(jobs) _sync_company_counters(jobs) MASTER_GAP_PLAN_PATH.parent.mkdir(parents=True, exist_ok=True) MASTER_GAP_PLAN_PATH.write_text(rendered, encoding="utf-8") return entries, rendered +_SKILL_BACKLINKS_HEADING = "## Jobs requiring this skill" +_SKILL_BACKLINKS_BLOCK = re.compile( + r"(?ms)^## Jobs requiring this skill\s*\n.*?(?=^## |\Z)" +) + + +def _sync_skill_backlinks(jobs: list[JobSummary]) -> None: + """Write a `## Jobs requiring this skill` block into each SkillNote body + that lists every JobNote whose required/nice-to-have skills include the + canonical name. Mirrors P2 of OBSIDIAN_LEVERAGE.md. + + Preserves any human-edited content above the block (skill notes are seeded + with category headers + brief descriptions; we don't want to clobber that). + Replaces an existing block if present, appends otherwise. + + Jobs are ordered by match_score DESC so the strongest fits sit at the top + of each SkillNote. + """ + from collections import defaultdict + + from compass.vault.taxonomy import normalize + + skills_dir = VAULT_PATH / "skills" + if not skills_dir.exists(): + return + + by_skill: dict[str, list[JobSummary]] = defaultdict(list) + for job in jobs: + seen: set[str] = set() + for raw in (*job.skills_required, *job.skills_nice_to_have): + canon = normalize(raw) + if not canon or canon in seen: + continue + seen.add(canon) + by_skill[canon].append(job) + + for path in skills_dir.glob("*.md"): + text = path.read_text(encoding="utf-8") + m = re.match(r"^(---\n.*?\n---\n)(.*)", text, re.DOTALL) + if not m: + continue + head, body = m.group(1), m.group(2) + # Resolve canonical from frontmatter + fm_m = re.match(r"^---\n(.*?)\n---\n", text, re.DOTALL) + if not fm_m: + continue + fm = yaml.safe_load(fm_m.group(1)) or {} + canonical = fm.get("skill") + if not canonical: + continue + + bl_jobs = sorted( + by_skill.get(canonical, []), + key=lambda j: (-j.match_score, j.company, j.title), + ) + rendered = _render_skill_backlinks(bl_jobs) + + if _SKILL_BACKLINKS_BLOCK.search(body): + replacement = rendered + "\n" if rendered else "" + new_body = _SKILL_BACKLINKS_BLOCK.sub(replacement, body, count=1) + new_body = re.sub(r"\n{3,}", "\n\n", new_body) + elif rendered: + new_body = body.rstrip() + "\n\n" + rendered + "\n" + else: + new_body = body + + if new_body != body: + path.write_text(head + new_body, encoding="utf-8") + + +def _render_skill_backlinks(jobs: list[JobSummary]) -> str: + if not jobs: + return "" + lines = [_SKILL_BACKLINKS_HEADING, ""] + for j in jobs: + stem = j.file.stem # filename minus .md — matches Obsidian's wikilink target + display = f"{j.company} — {j.title}" + score = f"score {j.match_score:.1f}" + tier = j.tier or "unknown" + lines.append(f"- [[{stem}|{display}]] · {score} · {tier}") + return "\n".join(lines) + "\n" + + def _sync_company_counters(jobs: list[JobSummary]) -> None: """Rewrite `roles_seen` on each `companies/*.md` to match the actual JobNote count for that company. Same pattern as `_sync_skill_counters` — treat the diff --git a/compass/pipeline/nodes/vault_write.py b/compass/pipeline/nodes/vault_write.py index ca4c08f..0e5f1ac 100644 --- a/compass/pipeline/nodes/vault_write.py +++ b/compass/pipeline/nodes/vault_write.py @@ -33,6 +33,37 @@ logger = logging.getLogger(__name__) +def _build_auto_tags( + *, + tier: str, + match_score: float, + role_family: str, + hitl_decision: str | None, +) -> list[str]: + """Generate Obsidian tag-pane-filterable tags from JobNote fields. + + Tag taxonomy is intentionally shallow (one slash) so Obsidian's nested-tag + view groups them. Composed filters like `#fit/strong AND #role/agent-engineer` + happen naturally in the tag pane and in Dataview `contains(tags, "...")` calls. + """ + tags: list[str] = [] + if tier: + tags.append(f"#tier/{tier}") + if match_score >= 4.0: + tags.append("#fit/strong") + elif match_score >= 3.0: + tags.append("#fit/decent") + elif match_score >= 2.0: + tags.append("#fit/stretch") + else: + tags.append("#fit/weak") + if role_family: + tags.append(f"#role/{role_family}") + if hitl_decision: + tags.append(f"#decision/{hitl_decision}") + return tags + + def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | None]: """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" from compass.config import SCORE_THRESHOLD @@ -117,6 +148,13 @@ async def vault_write_node(state: CompassState) -> dict: break hitl_decision, hitl_at = _derive_hitl_decision(state) + role_family = state.get("role_family") or "" + auto_tags = _build_auto_tags( + tier=company_tier, + match_score=score.score, + role_family=role_family, + hitl_decision=hitl_decision, + ) note = JobNote( company=job.company, title=job.title, @@ -131,8 +169,9 @@ async def vault_write_node(state: CompassState) -> dict: remote=("remote" if job.remote else None), seniority=req.seniority, years_required=req.years_experience, - role_family=state.get("role_family") or "", + role_family=role_family, tier=company_tier, + tags=auto_tags, skills_required=req.required_skills, skills_nice_to_have=req.nice_to_have_skills, skills_matched=score.matched_skills, diff --git a/tests/analysis/test_skill_backlinks.py b/tests/analysis/test_skill_backlinks.py new file mode 100644 index 0000000..e42118e --- /dev/null +++ b/tests/analysis/test_skill_backlinks.py @@ -0,0 +1,130 @@ +"""P2 Obsidian leverage: gap_aggregator writes a `## Jobs requiring this skill` +block into each SkillNote body so Linked Mentions + Dataview can surface the +reverse mapping (every JD asking for Python, sorted by match_score). + +The block preserves existing body content above it (seed notes carry category +descriptions like `_Category: mcp · Tier-2 demand: highest_` that must not be +clobbered) and replaces itself on re-runs (idempotent). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import frontmatter + +if TYPE_CHECKING: + from pathlib import Path + + +def _seed_skill(vault: Path, canonical: str, category: str, body_extra: str = "") -> None: + (vault / "skills").mkdir(parents=True, exist_ok=True) + safe = canonical.replace(" ", "_").replace("/", "_") + text = ( + "---\n" + "type: skill\n" + f"skill: {canonical}\n" + f"category: {category}\n" + "appears_in_jobs: 0\n" + "my_level: 0\n" + "---\n" + f"# {canonical}\n" + f"{body_extra}" + ) + (vault / "skills" / f"{safe}.md").write_text(text, encoding="utf-8") + + +def _seed_job( + vault: Path, + name: str, + *, + company: str, + title: str, + required: list[str], + score: float, + tier: str = "apply-now", +) -> None: + (vault / "jobs").mkdir(parents=True, exist_ok=True) + post = frontmatter.Post( + "# job\n", + company=company, + title=title, + url=f"https://x/{name}", + source="manual", + date_found="2026-05-19", + match_score=score, + score_reasoning="t", + role_family="agent-engineer", + tier=tier, + skills_required=required, + skills_nice_to_have=[], + skills_matched=[], + skills_missing=required, + jd_summary="t", + ) + (vault / "jobs" / f"{name}.md").write_text(frontmatter.dumps(post), encoding="utf-8") + + +def test_skill_backlinks_block_added_to_skillnote(temp_vault): + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language", body_extra="_Category: language_\n") + _seed_job( + temp_vault, "sierra", company="Sierra", title="Agent Eng", + required=["Python"], score=4.0, + ) + _seed_job( + temp_vault, "decagon", company="Decagon", title="MTS", + required=["Python"], score=3.0, + ) + + gap_aggregator.regenerate(write=True) + + body = (temp_vault / "skills" / "Python.md").read_text() + assert "## Jobs requiring this skill" in body + # Existing description preserved + assert "_Category: language_" in body + # Both jobs linked, with company — title display + assert "[[sierra|Sierra — Agent Eng]]" in body + assert "[[decagon|Decagon — MTS]]" in body + # Sierra (score 4.0) appears before Decagon (score 3.0) + assert body.index("Sierra — Agent Eng") < body.index("Decagon — MTS") + # Score + tier rendered + assert "score 4.0" in body + assert "apply-now" in body + + +def test_skill_backlinks_idempotent_on_rerun(temp_vault): + """Two regenerate calls must not duplicate the block.""" + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language") + _seed_job( + temp_vault, "sierra", company="Sierra", title="Agent Eng", + required=["Python"], score=4.0, + ) + gap_aggregator.regenerate(write=True) + body1 = (temp_vault / "skills" / "Python.md").read_text() + gap_aggregator.regenerate(write=True) + body2 = (temp_vault / "skills" / "Python.md").read_text() + + assert body1 == body2 + assert body2.count("## Jobs requiring this skill") == 1 + + +def test_skill_backlinks_removed_when_no_jobs_reference_skill(temp_vault): + """If a skill's last referencing job is removed, the block goes away.""" + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language") + _seed_job( + temp_vault, "sierra", company="Sierra", title="Agent Eng", + required=["Python"], score=4.0, + ) + gap_aggregator.regenerate(write=True) + assert "## Jobs requiring this skill" in (temp_vault / "skills" / "Python.md").read_text() + + # Remove the only job + (temp_vault / "jobs" / "sierra.md").unlink() + gap_aggregator.regenerate(write=True) + assert "## Jobs requiring this skill" not in (temp_vault / "skills" / "Python.md").read_text() diff --git a/tests/pipeline/test_auto_tags.py b/tests/pipeline/test_auto_tags.py new file mode 100644 index 0000000..d45483f --- /dev/null +++ b/tests/pipeline/test_auto_tags.py @@ -0,0 +1,61 @@ +"""P3 Obsidian leverage: vault_write_node auto-generates Obsidian tag-pane +filterable tags from JobNote fields so `#fit/strong AND #role/agent-engineer` +queries work naturally in the tag pane and Dataview.""" + +from __future__ import annotations + +from compass.pipeline.nodes.vault_write import _build_auto_tags + + +def test_build_auto_tags_strong_fit_approved(): + tags = _build_auto_tags( + tier="apply-now", + match_score=4.2, + role_family="agent-engineer", + hitl_decision="approved", + ) + assert tags == [ + "#tier/apply-now", + "#fit/strong", + "#role/agent-engineer", + "#decision/approved", + ] + + +def test_build_auto_tags_fit_buckets(): + assert "#fit/strong" in _build_auto_tags( + tier="apply-now", match_score=4.0, role_family="x", hitl_decision=None, + ) + assert "#fit/decent" in _build_auto_tags( + tier="apply-now", match_score=3.5, role_family="x", hitl_decision=None, + ) + assert "#fit/stretch" in _build_auto_tags( + tier="apply-now", match_score=2.5, role_family="x", hitl_decision=None, + ) + assert "#fit/weak" in _build_auto_tags( + tier="apply-now", match_score=1.5, role_family="x", hitl_decision=None, + ) + + +def test_build_auto_tags_omits_none_decision(): + """When hitl never ran (extract errored, etc.) no #decision/ tag is emitted.""" + tags = _build_auto_tags( + tier="opportunistic", match_score=3.0, role_family="agent-engineer", hitl_decision=None, + ) + assert not any(t.startswith("#decision/") for t in tags) + + +def test_build_auto_tags_omits_empty_role_family(): + """role_family can be the empty string when classification deferred. No tag in that case.""" + tags = _build_auto_tags( + tier="apply-now", match_score=4.0, role_family="", hitl_decision="approved", + ) + assert not any(t.startswith("#role/") for t in tags) + + +def test_build_auto_tags_includes_all_tiers(): + for tier in ["apply-now", "opportunistic", "backend-prep", "stretch", "skip", "unknown"]: + tags = _build_auto_tags( + tier=tier, match_score=3.0, role_family="agent-engineer", hitl_decision=None, + ) + assert f"#tier/{tier}" in tags From eb35632f2deea033554bb365640d828d72b25954 Mon Sep 17 00:00:00 2001 From: Akash Aedavelli Date: Tue, 19 May 2026 15:57:04 -0500 Subject: [PATCH 32/46] feat(intake): enforce reject_if_* rules + drop stale postings pre-LLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Item 1 — `compass.vault.reader.load_reject_rules` parses `reject_if_title_contains` and `reject_if_jd_contains` YAML blocks out of preferences.md. `intake_filter_node` applies them as substring matches BEFORE the keyword classifier, so senior/staff/principal titles and `5+ years` / `PhD required` JDs are dropped at zero LLM cost. On a 41-board scrape this is ~40-60% of incoming volume. Item 2 — `_scrape_all` filters each board's results to `date_posted` within the last 30 days then sorts by date_posted DESC (None-dated jobs sink to the bottom) before round-robin interleave + MAX_JOBS_PER_RUN cap. Each board now contributes its freshest postings to the cap rather than arbitrary order. Undated postings still get a shot — many ATSes don't expose date_posted and we can't distinguish stale from undated. 7 new tests. 278 passing. Ruff clean. --- compass/pipeline/graph.py | 45 ++++++++++++++-- compass/pipeline/nodes/intake_filter.py | 24 +++++++++ compass/vault/reader.py | 37 +++++++++++++ tests/pipeline/test_intake_filter.py | 71 +++++++++++++++++++++++++ tests/pipeline/test_scrape_recency.py | 52 ++++++++++++++++++ 5 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 tests/pipeline/test_scrape_recency.py diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index c9ae612..572c1e5 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -406,11 +406,21 @@ def _count_unknown_skills_seen_this_run(start_wall: datetime) -> int: ) -async def _scrape_all() -> list[RawJob]: - """Scrape all configured sources concurrently, interleave round-robin, cap. +MAX_POSTING_AGE_DAYS = 30 # Drop JDs older than this — postings >30d are usually filled or stale. + - Interleaving prevents a single high-volume source from exhausting - MAX_JOBS_PER_RUN before quieter sources get a chance. +async def _scrape_all() -> list[RawJob]: + """Scrape all configured sources concurrently, drop stale postings, sort + each board by recency, then interleave round-robin and cap. + + Order matters: + 1. Drop postings with `date_posted` older than MAX_POSTING_AGE_DAYS — stale + roles aren't worth LLM cost or vault clutter. + 2. Sort each board's results by date_posted DESC (None last) so when the + interleave-cap below fires, each board contributes its FRESHEST jobs. + 3. Round-robin interleave so a single high-volume board doesn't exhaust + MAX_JOBS_PER_RUN before quieter boards get a chance. + 4. Cap to MAX_JOBS_PER_RUN. """ from compass.config import ASHBY_BOARDS, GREENHOUSE_BOARDS, LEVER_COMPANIES, MAX_JOBS_PER_RUN from compass.scrapers.ashby import scrape_ashby_many @@ -422,6 +432,11 @@ async def _scrape_all() -> list[RawJob]: scrape_lever_many(LEVER_COMPANIES), scrape_ashby_many(ASHBY_BOARDS), ) + + gh = _filter_and_sort_by_recency(gh) + lv = _filter_and_sort_by_recency(lv) + ash = _filter_and_sort_by_recency(ash) + interleaved: list[RawJob] = [] iters = [iter(gh), iter(lv), iter(ash)] while iters: @@ -436,6 +451,28 @@ async def _scrape_all() -> list[RawJob]: return interleaved[:MAX_JOBS_PER_RUN] +def _filter_and_sort_by_recency(jobs: list[RawJob]) -> list[RawJob]: + """Drop >30d-old postings (when date_posted is known); sort by date_posted + DESC with None-dated jobs sinking to the bottom. Postings without a + `date_posted` aren't dropped — many ATSes don't expose it consistently and + we can't tell stale from undated.""" + from datetime import date, timedelta + + cutoff = date.today() - timedelta(days=MAX_POSTING_AGE_DAYS) + kept = [j for j in jobs if j.date_posted is None or j.date_posted >= cutoff] + kept.sort(key=lambda j: (j.date_posted is None, -_date_to_ordinal(j.date_posted))) + return kept + + +def _date_to_ordinal(d: object) -> int: + """date.toordinal() wrapper that treats None as 0 — only used for sort key.""" + from datetime import date + + if isinstance(d, date): + return d.toordinal() + return 0 + + if __name__ == "__main__": result = asyncio.run(run_pipeline()) print( diff --git a/compass/pipeline/nodes/intake_filter.py b/compass/pipeline/nodes/intake_filter.py index 718ae0d..ede3900 100644 --- a/compass/pipeline/nodes/intake_filter.py +++ b/compass/pipeline/nodes/intake_filter.py @@ -20,6 +20,7 @@ import compass.config as cfg from compass.pipeline.role_family import keyword_classify, llm_classify, upgrade_family +from compass.vault.reader import load_reject_rules if TYPE_CHECKING: from compass.pipeline.state import CompassState @@ -48,6 +49,29 @@ async def intake_filter_node(state: CompassState) -> dict: body = job.description or "" + # Hard rejects from preferences.md — runs BEFORE LLM-stage classification so + # senior/staff/principal/lead titles and "5+ years" / "PhD required" JDs are + # dropped at zero LLM cost. On a 41-board scrape this is ~40-60% of volume. + rules = load_reject_rules() + title_lc = (job.title or "").lower() + body_lc = body.lower() + for needle in rules["title"]: + if needle and needle in title_lc: + _log_filtered(job.company, job.title, f"title rejects: {needle!r}") + logger.info( + "intake_filter: dropped %s — %s (title rule: %r)", + job.company, job.title, needle, + ) + return {"in_scope": False, "role_family": "out-of-scope"} + for needle in rules["jd"]: + if needle and needle in body_lc: + _log_filtered(job.company, job.title, f"jd rejects: {needle!r}") + logger.info( + "intake_filter: dropped %s — %s (jd rule: %r)", + job.company, job.title, needle, + ) + return {"in_scope": False, "role_family": "out-of-scope"} + decided, family = keyword_classify(job.title) if decided is True: upgraded = upgrade_family(family, body) diff --git a/compass/vault/reader.py b/compass/vault/reader.py index 0e1ce9a..6ec6538 100644 --- a/compass/vault/reader.py +++ b/compass/vault/reader.py @@ -4,9 +4,11 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING import frontmatter +import yaml from compass.config import VAULT_PATH @@ -50,3 +52,38 @@ def list_job_notes() -> list[Path]: if not jobs_dir.exists(): return [] return sorted(jobs_dir.glob("*.md")) + + +# ── reject rules (preferences.md) ───────────────────────────────────────────── + +# Match a labelled YAML block: `