diff --git a/.env.example b/.env.example index 7295449..791dba3 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,7 @@ EMBEDDING_MODEL=all-MiniLM-L6-v2 # ── HiTL ───────────────────────────────────────────────────────────────────── HITL_STATE_DB=~/.compass/hitl.db +HITL_CHECKPOINT_DB=~/.compass/checkpoints.db HITL_TIMEOUT_HOURS=4 # ── Pipeline ───────────────────────────────────────────────────────────────── diff --git a/compass/analysis/gap_aggregator.py b/compass/analysis/gap_aggregator.py index 48a3902..bebf3cb 100644 --- a/compass/analysis/gap_aggregator.py +++ b/compass/analysis/gap_aggregator.py @@ -68,6 +68,11 @@ def load_jobs() -> list[JobSummary]: fm = _parse_frontmatter(f) if not fm: continue + # Skip out-of-scope JobNotes — they shouldn't influence the gap plan. + # role_family is set once at intake; if a stale entry's title would + # now classify as out-of-scope, the migration script handles that. + if fm.get("role_family") == "out-of-scope": + continue out.append( JobSummary( file=f, @@ -264,12 +269,102 @@ def regenerate(write: bool = True) -> tuple[list[GapPlanEntry], str]: rendered = render_master_plan(entries, jobs_n=len(jobs)) if write: _sync_skill_counters(entries) + _sync_skill_backlinks(jobs) _sync_company_counters(jobs) MASTER_GAP_PLAN_PATH.parent.mkdir(parents=True, exist_ok=True) - MASTER_GAP_PLAN_PATH.write_text(rendered, encoding="utf-8") + # Atomic write: tmp + os.replace so a process kill mid-write leaves + # the previous good file in place rather than a half-written one. + # Path.write_text uses truncate-then-write which is NOT atomic on + # macOS APFS — MCP get_master_gap_plan tool would read garbage. + import os as _os + + tmp_path = MASTER_GAP_PLAN_PATH.with_suffix(".md.tmp") + tmp_path.write_text(rendered, encoding="utf-8") + _os.replace(tmp_path, MASTER_GAP_PLAN_PATH) return entries, rendered +_SKILL_BACKLINKS_HEADING = "## Jobs requiring this skill" +_SKILL_BACKLINKS_BLOCK = re.compile(r"(?ms)^## Jobs requiring this skill\s*\n.*?(?=^## |\Z)") + + +def _sync_skill_backlinks(jobs: list[JobSummary]) -> None: + """Write a `## Jobs requiring this skill` block into each SkillNote body + that lists every JobNote whose required/nice-to-have skills include the + canonical name. Mirrors P2 of OBSIDIAN_LEVERAGE.md. + + Preserves any human-edited content above the block (skill notes are seeded + with category headers + brief descriptions; we don't want to clobber that). + Replaces an existing block if present, appends otherwise. + + Jobs are ordered by match_score DESC so the strongest fits sit at the top + of each SkillNote. + """ + from collections import defaultdict + + from compass.vault.taxonomy import normalize + + skills_dir = VAULT_PATH / "skills" + if not skills_dir.exists(): + return + + by_skill: dict[str, list[JobSummary]] = defaultdict(list) + for job in jobs: + seen: set[str] = set() + for raw in (*job.skills_required, *job.skills_nice_to_have): + canon = normalize(raw) + if not canon or canon in seen: + continue + seen.add(canon) + by_skill[canon].append(job) + + for path in skills_dir.glob("*.md"): + text = path.read_text(encoding="utf-8") + m = re.match(r"^(---\n.*?\n---\n)(.*)", text, re.DOTALL) + if not m: + continue + head, body = m.group(1), m.group(2) + # Resolve canonical from frontmatter + fm_m = re.match(r"^---\n(.*?)\n---\n", text, re.DOTALL) + if not fm_m: + continue + fm = yaml.safe_load(fm_m.group(1)) or {} + canonical = fm.get("skill") + if not canonical: + continue + + bl_jobs = sorted( + by_skill.get(canonical, []), + key=lambda j: (-j.match_score, j.company, j.title), + ) + rendered = _render_skill_backlinks(bl_jobs) + + if _SKILL_BACKLINKS_BLOCK.search(body): + replacement = rendered + "\n" if rendered else "" + new_body = _SKILL_BACKLINKS_BLOCK.sub(replacement, body, count=1) + new_body = re.sub(r"\n{3,}", "\n\n", new_body) + elif rendered: + new_body = body.rstrip() + "\n\n" + rendered + "\n" + else: + new_body = body + + if new_body != body: + path.write_text(head + new_body, encoding="utf-8") + + +def _render_skill_backlinks(jobs: list[JobSummary]) -> str: + if not jobs: + return "" + lines = [_SKILL_BACKLINKS_HEADING, ""] + for j in jobs: + stem = j.file.stem # filename minus .md — matches Obsidian's wikilink target + display = f"{j.company} — {j.title}" + score = f"score {j.match_score:.1f}" + tier = j.tier or "unknown" + lines.append(f"- [[{stem}|{display}]] · {score} · {tier}") + return "\n".join(lines) + "\n" + + def _sync_company_counters(jobs: list[JobSummary]) -> None: """Rewrite `roles_seen` on each `companies/*.md` to match the actual JobNote count for that company. Same pattern as `_sync_skill_counters` — treat the diff --git a/compass/config.py b/compass/config.py index 2e99940..1ca650c 100644 --- a/compass/config.py +++ b/compass/config.py @@ -44,6 +44,9 @@ # ── HiTL ────────────────────────────────────────────────────────────────────── HITL_STATE_DB: Path = Path(os.getenv("HITL_STATE_DB", "~/.compass/hitl.db")).expanduser() +HITL_CHECKPOINT_DB: Path = Path( + os.getenv("HITL_CHECKPOINT_DB", "~/.compass/checkpoints.db") +).expanduser() HITL_TIMEOUT_HOURS: int = int(os.getenv("HITL_TIMEOUT_HOURS", "4")) # ── Pipeline ────────────────────────────────────────────────────────────────── @@ -54,7 +57,9 @@ # ── Tier weights (gap_aggregator) — overrides from preferences.md at runtime ─ DEFAULT_TIER_WEIGHTS: dict[str, float] = { "apply-now": 1.0, - "6-month": 0.7, + "opportunistic": 0.85, + "backend-prep": 0.5, + "6-month": 0.7, # legacy — pre-3-month-pivot "stretch": 0.3, "skip": 0.0, "unknown": 0.5, diff --git a/compass/evals/dataset.py b/compass/evals/dataset.py index 58f45e4..57396df 100644 --- a/compass/evals/dataset.py +++ b/compass/evals/dataset.py @@ -1,34 +1,133 @@ -""" -Eval dataset management. - -The dataset is a JSON file: compass/evals/labeled_dataset.json -Format: -[ - { - "id": "eval-001", - "jd_text": "...", - "expected_score": 4.2, - "expected_skills": ["LangGraph", "Pydantic", "RAG"], - "notes": "Strong match — all required skills present" - } -] - -Start with 30 examples — enough to detect regressions. -Half synthesized with an LLM, half hand-labeled from real JDs. +"""Eval dataset — labeled JDs the harness compares Compass output against. + +Storage: `compass/evals/labeled_dataset.json` (NOT in the vault — the dataset +is repo-tracked code artifact, not a vault note). + +Schema per record: + id unique stable string (eval-001, eval-002, …) + jd_text raw JD body (post-HTML-strip) + source where the JD came from — usually a JobNote filename + expected_score float 0-5, what a human reviewer says the fit is + expected_skills list[str], all skills a human reviewer says the JD asks for + (the union — both "required" and "nice-to-have") + notes free-text human-justification for the labels + +Hand-labeled at first; later we can also add an LLM-judge column for cheap +sanity checking on un-labeled JDs (see compass/evals/judge.py). + +Start with 20 examples covering the full tier spread — bank rotational entry, +mid-SaaS, frontier startup, Tier 1.5 data-infra, applied-AI at Anthropic. +Enough variance to surface tier-specific extract/score biases. """ +from __future__ import annotations + +import json from pathlib import Path +from pydantic import BaseModel, Field + DATASET_PATH = Path(__file__).parent / "labeled_dataset.json" -def load_dataset() -> list[dict]: - """Load the labeled evaluation dataset.""" - raise NotImplementedError("load_dataset not yet implemented") +class EvalRecord(BaseModel): + """One labeled JD in the eval dataset.""" + + id: str + jd_text: str + source: str = "" # e.g. JobNote filename — provenance for hand-labels + expected_score: float = Field(ge=0.0, le=5.0) + expected_skills: list[str] = Field(default_factory=list) + notes: str = "" + + +def load_dataset(path: Path | None = None) -> list[EvalRecord]: + """Load the labeled evaluation dataset. Returns [] when the file is missing + (first-time setup) so the runner can still execute against the LLM-judge + path without requiring hand labels.""" + p = path or DATASET_PATH + if not p.exists(): + return [] + raw = json.loads(p.read_text(encoding="utf-8")) + return [EvalRecord.model_validate(r) for r in raw] + + +def save_dataset(records: list[EvalRecord], path: Path | None = None) -> Path: + """Save the dataset to JSON. Pretty-printed so git diffs are readable + when a human adds a new example. + + ATOMIC: writes to a `.tmp` sibling first then `os.replace`. A SIGKILL or + disk-full mid-write previously truncated the JSON, making the entire + labeled dataset un-parseable on next load. With the atomic swap, the + previous good file stays in place on any failure. + """ + import os as _os + + p = path or DATASET_PATH + p.parent.mkdir(parents=True, exist_ok=True) + tmp_path = p.with_suffix(p.suffix + ".tmp") + tmp_path.write_text( + json.dumps( + [r.model_dump() for r in records], + indent=2, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + _os.replace(tmp_path, p) + return p def add_example( - jd_text: str, expected_score: float, expected_skills: list[str], notes: str = "" -) -> None: - """Add a new labeled example to the dataset.""" - raise NotImplementedError("add_example not yet implemented") + jd_text: str, + expected_score: float, + expected_skills: list[str], + *, + source: str = "", + notes: str = "", + path: Path | None = None, +) -> EvalRecord: + """Append one labeled example. Auto-generates a stable ID based on the + current dataset size — `eval-001`, `eval-002`, …. Caller is responsible + for not duplicating jd_text (no dedup enforced; intentional flexibility).""" + records = load_dataset(path) + new_id = f"eval-{len(records) + 1:03d}" + record = EvalRecord( + id=new_id, + jd_text=jd_text, + source=source, + expected_score=expected_score, + expected_skills=list(expected_skills), + notes=notes, + ) + records.append(record) + save_dataset(records, path) + return record + + +def add_from_jobnote( + jobnote_path: Path, expected_score: float, expected_skills: list[str], notes: str = "" +) -> EvalRecord: + """Convenience: read JD body + summary from a JobNote in the vault, + add as a labeled record. Used during the "label 10 JobNotes from the + first refresh" Day-7 sprint task. + """ + import frontmatter + + post = frontmatter.load(jobnote_path) + company = post.metadata.get("company", "") + title = post.metadata.get("title", "") + # The JD body is everything in `## Full JD` section, falling back to the + # whole post content if the section isn't present (older JobNotes). + text = post.content + if "## Full JD" in text: + text = text.split("## Full JD", 1)[1].strip() + source = f"{jobnote_path.name} ({company} — {title})" + return add_example( + jd_text=text, + expected_score=expected_score, + expected_skills=expected_skills, + source=source, + notes=notes, + ) diff --git a/compass/evals/judge.py b/compass/evals/judge.py new file mode 100644 index 0000000..0fb2dac --- /dev/null +++ b/compass/evals/judge.py @@ -0,0 +1,87 @@ +"""LLM-as-judge — produces synthetic "expected" labels for a JD without +manual annotation. + +Use case: you've just done a refresh and have 100 new JobNotes. Hand-labeling +all of them takes hours, but you want a directional signal NOW on whether the +score_node + extract_node are doing the right thing. The judge reads the JD +body + Compass's output and emits its own (skills, score) — which the runner +can compare against Compass. + +This is NOT a substitute for hand-labels. Two LLMs may share biases. But it +catches gross failures (e.g. extract_node returning 2 skills when the JD +lists 12) at near-zero cost. + +Cost: ~$0.002 per JD on Flash. 100 JDs = $0.20. +""" + +from __future__ import annotations + +import logging + +from pydantic import BaseModel, Field + +from compass.llm import make_agent + +logger = logging.getLogger(__name__) + + +class JudgeVerdict(BaseModel): + """Structured judgment of one JD.""" + + expected_skills: list[str] = Field( + description="Every distinct technical skill / framework / tool the JD asks for." + ) + expected_score: float = Field( + ge=0.0, + le=5.0, + description="0-5 estimate of fit between the candidate profile and the JD.", + ) + reasoning: str = Field(description="2-3 sentence justification.") + + +_SYSTEM_PROMPT = """You are an independent reviewer evaluating an AI agent's +job-matching output. You are given: +- A raw job description. +- A candidate profile (resume + role-clarifications). +- The agent's own extraction (skills it found) and score (0-5). + +Your job is to produce YOUR OWN reading of: + expected_skills: every distinct technical skill the JD ACTUALLY asks for. + List both required and nice-to-have. Use exact phrases from + the JD when possible. Do NOT list skills the agent guessed + that aren't actually in the JD. + expected_score: your independent 0-5 score of candidate-to-JD fit. Use the + same scale as the agent (5 = perfect match with production + evidence; 3 = decent match with some gaps; 1 = poor match). + reasoning: 2-3 sentences justifying your score, naming the strongest match + signal and the biggest gap. + +Be HONEST. Don't anchor on the agent's numbers — produce your own read first. +If you disagree with the agent, say so plainly in the reasoning. +""" + + +def _build_agent(): + # Use SCORE_MODEL by default — Gemini Flash is plenty for judging accuracy + # of another LLM call. Override via env if you want a stronger judge. + return make_agent("score", output_type=JudgeVerdict, system_prompt=_SYSTEM_PROMPT) + + +async def judge_jd( + jd_text: str, + profile_text: str, + agent_predicted_skills: list[str], + agent_predicted_score: float, +) -> JudgeVerdict: + """Single-JD judgment. Tests patch this function (the pydantic-ai Agent + itself is harder to stub).""" + agent = _build_agent() + prompt = ( + f"# CANDIDATE PROFILE\n{profile_text}\n\n" + f"# JOB DESCRIPTION\n{jd_text}\n\n" + f"# AGENT OUTPUT (to be evaluated, not to anchor on)\n" + f"agent_extracted_skills: {', '.join(agent_predicted_skills) or '(none)'}\n" + f"agent_score: {agent_predicted_score:.2f}\n" + ) + result = await agent.run(prompt) + return result.output diff --git a/compass/evals/metrics.py b/compass/evals/metrics.py new file mode 100644 index 0000000..e2caf67 --- /dev/null +++ b/compass/evals/metrics.py @@ -0,0 +1,133 @@ +"""Eval metric primitives — pure functions, no I/O, no LLM calls. + +Used by both the human-label runner (compares to EvalRecord.expected_*) and +the LLM-as-judge runner (compares to a judge model's expected_*). +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class ScoreMetrics: + """Aggregate metrics across N JDs.""" + + n: int + score_mae: float # mean absolute error of Compass score vs expected + score_rmse: float # sqrt of mean squared error — penalizes big misses + score_bias: float # mean signed error — positive = Compass over-scores + extract_skill_recall: float # fraction of expected_skills that extract_node found + extract_skill_precision: float # fraction of extracted skills that match expected + match_skill_recall: ( + float # fraction of expected_skills that score_node attributed as matched_skills + ) + + +def score_mae(predicted: list[float], expected: list[float]) -> float: + """Mean absolute error. Returns 0.0 for empty input — runner handles n=0.""" + if not predicted: + return 0.0 + pairs = list(zip(predicted, expected, strict=True)) + return sum(abs(p - e) for p, e in pairs) / len(pairs) + + +def score_rmse(predicted: list[float], expected: list[float]) -> float: + """Root mean squared error. Same length contract as `score_mae`.""" + if not predicted: + return 0.0 + pairs = list(zip(predicted, expected, strict=True)) + return (sum((p - e) ** 2 for p, e in pairs) / len(pairs)) ** 0.5 + + +def score_bias(predicted: list[float], expected: list[float]) -> float: + """Mean signed error. Positive = Compass scores HIGHER than the human; + negative = Compass scores LOWER. Signed bias is more useful than MAE + when deciding which direction to tune the prompt.""" + if not predicted: + return 0.0 + pairs = list(zip(predicted, expected, strict=True)) + return sum(p - e for p, e in pairs) / len(pairs) + + +def skill_recall(predicted_skills: list[str], expected_skills: list[str]) -> float: + """Of the expected skills, what fraction did the predictor find? + + Case-insensitive set match — the canonical taxonomy enforces casing + elsewhere, but humans labeling examples may type "Langgraph" or + "langGraph" inconsistently. + + Returns 1.0 when `expected_skills` is empty (vacuous truth — no skills + to find means we found all of them). Returns 0.0 when expected is + non-empty but predicted is empty. + """ + if not expected_skills: + return 1.0 + exp = {s.lower() for s in expected_skills} + pred = {s.lower() for s in predicted_skills} + return len(exp & pred) / len(exp) + + +def skill_precision(predicted_skills: list[str], expected_skills: list[str]) -> float: + """Of the predicted skills, what fraction were correct? + + Returns 1.0 when `predicted_skills` is empty (vacuous — no false + positives possible).""" + if not predicted_skills: + return 1.0 + exp = {s.lower() for s in expected_skills} + pred = {s.lower() for s in predicted_skills} + return len(exp & pred) / len(pred) + + +def aggregate( + predicted_scores: list[float], + expected_scores: list[float], + predicted_skill_lists: list[list[str]], + expected_skill_lists: list[list[str]], + matched_skill_lists: list[list[str]] | None = None, +) -> ScoreMetrics: + """Aggregate per-JD metrics across the whole dataset. + + `matched_skill_lists` is the `score_result.matched_skills` for each JD — + used to compute `match_skill_recall` separately from extract recall. + Passing None makes match_skill_recall fall back to extract_skill_recall. + """ + n = len(predicted_scores) + if n == 0: + return ScoreMetrics( + n=0, + score_mae=0.0, + score_rmse=0.0, + score_bias=0.0, + extract_skill_recall=0.0, + extract_skill_precision=0.0, + match_skill_recall=0.0, + ) + + matched_lists = ( + matched_skill_lists if matched_skill_lists is not None else predicted_skill_lists + ) + + extract_recalls = [ + skill_recall(pred, exp) + for pred, exp in zip(predicted_skill_lists, expected_skill_lists, strict=True) + ] + extract_precisions = [ + skill_precision(pred, exp) + for pred, exp in zip(predicted_skill_lists, expected_skill_lists, strict=True) + ] + match_recalls = [ + skill_recall(matched, exp) + for matched, exp in zip(matched_lists, expected_skill_lists, strict=True) + ] + + return ScoreMetrics( + n=n, + score_mae=score_mae(predicted_scores, expected_scores), + score_rmse=score_rmse(predicted_scores, expected_scores), + score_bias=score_bias(predicted_scores, expected_scores), + extract_skill_recall=sum(extract_recalls) / n, + extract_skill_precision=sum(extract_precisions) / n, + match_skill_recall=sum(match_recalls) / n, + ) diff --git a/compass/evals/runner.py b/compass/evals/runner.py index 30c0840..49ea7fb 100644 --- a/compass/evals/runner.py +++ b/compass/evals/runner.py @@ -1,19 +1,330 @@ +"""Eval runner — runs Compass extract+score on every record in the dataset +and computes aggregate metrics. + +Two modes: + --labels Compare against `EvalRecord.expected_*` (hand-labeled). Requires + a populated dataset. Default. + --judge Compare against LLM-as-judge output. Cheaper but lower-confidence. + Useful BEFORE you've hand-labeled anything. + +Run: + uv run python -m compass.evals.runner # labels (default) + uv run python -m compass.evals.runner --judge # LLM judge + uv run python -m compass.evals.runner --judge --limit 10 # 10 random JDs + +Output: prints a metrics summary to stdout and writes a results JSON to +`compass/evals/results-{mode}-{timestamp}.json`. """ -Eval harness runner — runs nightly, logs results to Langfuse. -Run: uv run python -m compass.evals.runner +from __future__ import annotations -What it measures: - - Score MAE (mean absolute error vs human labels) - - Skill extraction recall (did we find all skills a human identified?) - - Cost per eval run - - Tokens per node +import argparse +import asyncio +import json +import logging +import random +import sys +import time +from datetime import datetime +from pathlib import Path -Results are logged to Langfuse as a Dataset Run. -A summary is printed to stdout and written to compass/evals/results.json. +from compass.evals.dataset import EvalRecord, load_dataset +from compass.evals.metrics import ScoreMetrics, aggregate +from compass.pipeline.nodes.extract import _extract +from compass.pipeline.nodes.score import _score +from compass.pipeline.state import JobRequirements, JobScore, RawJob +from compass.vault.reader import read_profile_section, read_resume + +logger = logging.getLogger(__name__) + +RESULTS_DIR = Path(__file__).parent + + +async def _run_extract_and_score( + record: EvalRecord, +) -> tuple[JobRequirements | None, JobScore | None, float]: + """Run the SAME extract → score logic the production graph runs, against + one labeled JD. Returns (req, score, wall_seconds). + + Production fidelity matters here — earlier versions called `_extract` and + `_score` directly, which skipped the post-LLM normalization and constraint + layers, making the metrics measure something other than what the user + actually experiences. Now we apply: + + extract: `_normalize_skill_list` (taxonomy folding) + seniority-from-title + fallback. Matches `extract_node`. + score: `_score_with_retry` (truncated-reasoning retry) + + `_constrain_to_jd_skills` (drop hallucinated matched/missing). + Matches `score_node`. + + We still call `_extract` and `_score` as the patchable surface so tests + can stub them; the post-processing wraps the stub output. + """ + from compass.pipeline.nodes.extract import ( + _normalize_skill_list, + _seniority_with_title_fallback, + ) + from compass.pipeline.nodes.score import _constrain_to_jd_skills, _reasoning_complete + + start = time.monotonic() + try: + raw_req = await _extract(record.jd_text) + except Exception as e: + logger.warning("extract failed for %s: %s", record.id, e) + return None, None, time.monotonic() - start + + # Mirror extract_node: canonicalize skills, fall-back seniority. + unknown: list[str] = [] + title_hint = "" # eval JDs don't carry titles — seniority stays as LLM said + req = JobRequirements( + required_skills=_normalize_skill_list(raw_req.required_skills, record.jd_text, unknown), + nice_to_have_skills=_normalize_skill_list( + raw_req.nice_to_have_skills, record.jd_text, unknown + ), + years_experience=raw_req.years_experience, + seniority=_seniority_with_title_fallback(raw_req.seniority, title_hint), + remote_policy=raw_req.remote_policy, + summary=raw_req.summary, + ) + + profile = f"{read_resume()}\n\n{read_profile_section('role-clarifications')}" + fake_job = RawJob( + company="(eval)", + title="(eval)", + url=f"eval://{record.id}", + source="manual", + description=record.jd_text, + ) + try: + raw_score = await _score(req, profile, fake_job) + except Exception as e: + logger.warning("score failed for %s: %s", record.id, e) + return req, None, time.monotonic() - start + + # Mirror score_node: retry on truncated reasoning, constrain matched/missing + # to the JD's actual skill universe. Without this the metric measures + # un-constrained LLM output rather than what the pipeline actually persists. + if not _reasoning_complete(raw_score.reasoning): + logger.info("eval: reasoning looked truncated for %s — retrying once", record.id) + try: + raw_score = await _score(req, profile, fake_job) + except Exception as e: + logger.warning("score retry failed for %s: %s", record.id, e) + constrained = _constrain_to_jd_skills(raw_score, req) + return req, constrained, time.monotonic() - start + + +async def run_against_labels(records: list[EvalRecord]) -> tuple[ScoreMetrics, list[dict]]: + """Run Compass on each record, compare to EvalRecord.expected_*.""" + per_record: list[dict] = [] + predicted_scores: list[float] = [] + expected_scores: list[float] = [] + predicted_skill_lists: list[list[str]] = [] + expected_skill_lists: list[list[str]] = [] + matched_skill_lists: list[list[str]] = [] + + for r in records: + req, score, elapsed = await _run_extract_and_score(r) + if req is None or score is None: + per_record.append( + {"id": r.id, "error": "extract or score failed", "elapsed_s": elapsed} + ) + continue + extracted_skills = list(req.required_skills) + list(req.nice_to_have_skills) + predicted_scores.append(score.score) + expected_scores.append(r.expected_score) + predicted_skill_lists.append(extracted_skills) + expected_skill_lists.append(r.expected_skills) + matched_skill_lists.append(list(score.matched_skills)) + per_record.append( + { + "id": r.id, + "source": r.source, + "expected_score": r.expected_score, + "predicted_score": score.score, + "score_delta": round(score.score - r.expected_score, 2), + "expected_skills_n": len(r.expected_skills), + "extracted_skills_n": len(extracted_skills), + "missed_skills": sorted( + {s.lower() for s in r.expected_skills} - {s.lower() for s in extracted_skills} + ), + "extra_skills": sorted( + {s.lower() for s in extracted_skills} - {s.lower() for s in r.expected_skills} + ), + "elapsed_s": round(elapsed, 2), + } + ) + + metrics = aggregate( + predicted_scores, + expected_scores, + predicted_skill_lists, + expected_skill_lists, + matched_skill_lists, + ) + return metrics, per_record + + +async def run_against_judge(records: list[EvalRecord]) -> tuple[ScoreMetrics, list[dict]]: + """Run Compass on each record, compare to an LLM-as-judge verdict. + + No EvalRecord.expected_* needed — the judge produces them on the fly. + Useful for first-pass sanity checks before you hand-label anything. + """ + from compass.evals.judge import judge_jd + + profile = f"{read_resume()}\n\n{read_profile_section('role-clarifications')}" + + per_record: list[dict] = [] + predicted_scores: list[float] = [] + expected_scores: list[float] = [] + predicted_skill_lists: list[list[str]] = [] + expected_skill_lists: list[list[str]] = [] + matched_skill_lists: list[list[str]] = [] + + for r in records: + req, score, elapsed = await _run_extract_and_score(r) + if req is None or score is None: + per_record.append( + {"id": r.id, "error": "extract or score failed", "elapsed_s": elapsed} + ) + continue + extracted_skills = list(req.required_skills) + list(req.nice_to_have_skills) + try: + verdict = await judge_jd(r.jd_text, profile, extracted_skills, score.score) + except Exception as e: + per_record.append({"id": r.id, "error": f"judge failed: {e}"}) + continue + # IMPORTANT: the judge prompt instructs it to use exact phrases from the + # JD. The JD uses raw forms ("LangGraph", "pydantic-ai"); Compass's + # extract uses canonical forms ("LangGraph", "Pydantic AI"). Comparing + # them directly systematically deflates recall on multi-word skills + # with punctuation variants. Normalize the judge's output through the + # same taxonomy folding extract_node uses so both sides are canonical. + from compass.pipeline.nodes.extract import _normalize_skill_list + + judge_skills_canonical = _normalize_skill_list(list(verdict.expected_skills), r.jd_text) + predicted_scores.append(score.score) + expected_scores.append(verdict.expected_score) + predicted_skill_lists.append(extracted_skills) + expected_skill_lists.append(judge_skills_canonical) + matched_skill_lists.append(list(score.matched_skills)) + per_record.append( + { + "id": r.id, + "source": r.source, + "judge_score": verdict.expected_score, + "predicted_score": score.score, + "score_delta": round(score.score - verdict.expected_score, 2), + "judge_skills_n": len(verdict.expected_skills), + "extracted_skills_n": len(extracted_skills), + "judge_skills_missed": sorted( + {s.lower() for s in verdict.expected_skills} + - {s.lower() for s in extracted_skills} + ), + "judge_reasoning": verdict.reasoning, + "elapsed_s": round(elapsed, 2), + } + ) + + metrics = aggregate( + predicted_scores, + expected_scores, + predicted_skill_lists, + expected_skill_lists, + matched_skill_lists, + ) + return metrics, per_record + + +def _format_summary(metrics: ScoreMetrics, mode: str) -> str: + return ( + f"\n=== Eval results ({mode}) ===\n" + f" n records: {metrics.n}\n" + f" score MAE: {metrics.score_mae:.2f} (0 = perfect)\n" + f" score RMSE: {metrics.score_rmse:.2f}\n" + f" score bias (signed): {metrics.score_bias:+.2f} (+ = over-scores, - = under-scores)\n" + f" extract skill recall: {metrics.extract_skill_recall:.1%} (B1 detector)\n" + f" extract skill precision: {metrics.extract_skill_precision:.1%}\n" + f" match skill recall: {metrics.match_skill_recall:.1%} (score_node attribution accuracy)\n" + ) + + +async def run_evals(*, mode: str = "labels", limit: int | None = None) -> dict: + """Programmatic entry point — used by the MCP tool wrapper. + + Returns a dict with `metrics`, `per_record`, and `results_path`. + """ + records = load_dataset() + if not records: + return { + "error": ( + "labeled_dataset.json is empty. Add examples via " + "compass.evals.dataset.add_example() or use --judge mode." + ), + "metrics": None, + } + if limit is not None and limit < len(records): + records = random.sample(records, limit) + + if mode == "judge": + metrics, per_record = await run_against_judge(records) + else: + metrics, per_record = await run_against_labels(records) + + # Include microseconds so two runs within the same second don't overwrite. + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + out_path = RESULTS_DIR / f"results-{mode}-{timestamp}.json" + out_path.write_text( + json.dumps( + { + "mode": mode, + "timestamp": timestamp, + "metrics": { + "n": metrics.n, + "score_mae": metrics.score_mae, + "score_rmse": metrics.score_rmse, + "score_bias": metrics.score_bias, + "extract_skill_recall": metrics.extract_skill_recall, + "extract_skill_precision": metrics.extract_skill_precision, + "match_skill_recall": metrics.match_skill_recall, + }, + "per_record": per_record, + }, + indent=2, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + return {"metrics": metrics, "per_record": per_record, "results_path": str(out_path)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--judge", + action="store_true", + help="Use LLM-as-judge instead of hand-labels (cheaper but lower confidence).", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Sample N records instead of running the full dataset.", + ) + args = parser.parse_args() + + result = asyncio.run(run_evals(mode="judge" if args.judge else "labels", limit=args.limit)) + if "error" in result: + print(result["error"], file=sys.stderr) + return 1 + + metrics = result["metrics"] + print(_format_summary(metrics, "judge" if args.judge else "labels")) + print(f"Results written to: {result['results_path']}") + return 0 -This is what you show in interviews when asked about eval methodology. -The chart of precision vs cost across model configs is generated here. -""" -raise NotImplementedError("eval runner not yet implemented — build this in Phase 3") +if __name__ == "__main__": + sys.exit(main()) diff --git a/compass/hitl/__init__.py b/compass/hitl/__init__.py index e69de29..e222fa2 100644 --- a/compass/hitl/__init__.py +++ b/compass/hitl/__init__.py @@ -0,0 +1,14 @@ +"""HiTL pause/resume infrastructure for Compass. + +The audit-trail mapping in vault_write._derive_hitl_decision matches on this +prefix; both sites must import HITL_TIMEOUT_FEEDBACK_PREFIX to stay in sync. +""" + +HITL_TIMEOUT_FEEDBACK_PREFIX = "auto-cancelled after" +"""Leading substring of the feedback string used to mark a timeout-driven resume. + +Used by: + - compass.hitl.timeout_checker — writes f"{HITL_TIMEOUT_FEEDBACK_PREFIX} {hrs}h timeout" + - compass.pipeline.nodes.vault_write._derive_hitl_decision — matches + feedback.startswith(HITL_TIMEOUT_FEEDBACK_PREFIX) to set hitl_decision="timed_out" +""" diff --git a/compass/hitl/resume.py b/compass/hitl/resume.py new file mode 100644 index 0000000..7a1176f --- /dev/null +++ b/compass/hitl/resume.py @@ -0,0 +1,106 @@ +"""Resume a paused LangGraph thread by re-opening the checkpointer and +recompiling the graph. Single source of truth for resumes — the timeout +checker and MCP `approve` tool both go through here.""" + +from __future__ import annotations + +import logging +from typing import Any + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver +from langgraph.types import Command + +from compass.hitl import state_store + +logger = logging.getLogger(__name__) + + +async def resume_pending( + thread_id: str, + *, + decision: dict[str, Any], + status_override: str | None = None, +) -> dict: + """Drive a paused thread to completion. + + decision: passed as `Command(resume=decision)`. Expected shape + {"approved": bool, "feedback": str | None}. + status_override: optional explicit status to write to the state store + (timeout_checker uses "timed_out"). If None, derived from + decision["approved"]: True -> "approved", False -> "rejected". + Raises LookupError if the thread isn't in the state store. + Raises ValueError if the thread is already resolved. + """ + from compass.config import HITL_CHECKPOINT_DB + + row = await state_store.get_pending(thread_id) + if row is None: + raise LookupError(f"no pending thread {thread_id!r}") + if row["status"] != "pending": + raise ValueError(f"thread {thread_id!r} already resolved ({row['status']!r})") + + # ATOMIC CLAIM before invoking the graph: prevents Modal cron and MCP + # approve from both passing the get_pending check, both calling ainvoke + # on the same checkpoint, and writing conflicting JobNotes (one + # "timed_out", one "approved"). If another consumer already claimed the + # row, abort cleanly with the same ValueError shape an already-resolved + # row would produce — callers handle this identically. + if not await state_store.claim_pending(thread_id): + raise ValueError( + f"thread {thread_id!r} was claimed by another resume consumer " + f"(Modal cron + MCP approve race avoided)" + ) + + # Late import to avoid potential circular import: graph.py imports state_store + from compass.pipeline.graph import _build_checkpoint_serde, build_graph + + config = {"configurable": {"thread_id": thread_id}} + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + # Derive from the FINAL graph state, not the input decision. A graph that + # auto-rejected on resume (e.g. SCORE_THRESHOLD edited between pause and + # resume, hitl_node short-circuited before consuming interrupt()) would + # otherwise be recorded as "approved" in state_store while the JobNote + # carries hitl_decision="auto_rejected" — audit-trail divergence. + resolved_status = "approved" if final.get("human_approved") is True else "rejected" + await state_store.mark_resolved( + thread_id, + status=resolved_status, + feedback=decision.get("feedback"), + ) + await _purge_thread_checkpoints(thread_id) + + if final.get("vault_written"): + # Resumed thread wrote a JobNote — derived counters (skills/appears_in_jobs, + # companies/roles_seen) now drift unless we resync. Phase 0 bug #12 + Phase + # 1.A bug #1 family: run_pipeline() calls regenerate() at end of batch, but + # resume paths (MCP approve + timeout_checker) bypass that. Call it here so + # every resume keeps the vault internally consistent. + from compass.analysis import gap_aggregator + + try: + gap_aggregator.regenerate(write=True) + except Exception: + # Counter sync failure must not block the resume — log and continue. + logger.exception("hitl: gap_aggregator.regenerate failed after resume") + + logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) + return final + + +async def _purge_thread_checkpoints(thread_id: str) -> None: + """Drop LangGraph per-step history for a resolved thread; state_store is the audit trail.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + await conn.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,)) + await conn.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,)) + await conn.commit() diff --git a/compass/hitl/state_store.py b/compass/hitl/state_store.py new file mode 100644 index 0000000..b825a23 --- /dev/null +++ b/compass/hitl/state_store.py @@ -0,0 +1,227 @@ +"""Pending-approval queue for the HiTL flow — aiosqlite, separate from the +LangGraph checkpoint DB. + +Public surface (all coroutines): + add_pending(thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills) + get_pending(thread_id) -> row | None + list_pending() -> list[row] # oldest first + list_timed_out(timeout_hours) -> list[row] # pending AND older than cutoff + mark_resolved(thread_id, status, feedback=None) + +A "row" is a plain dict with the schema documented in the implementation plan. +matched_skills / missing_skills are JSON-encoded in the DB and decoded on read. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +from typing import Any + +import aiosqlite + +logger = logging.getLogger(__name__) + +_VALID_STATUSES = {"pending", "resuming", "approved", "rejected", "timed_out", "error"} + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, + missing_skills TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','resuming','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +""" + + +def _now() -> _dt.datetime: + """Wall clock as UTC-aware. Indirected so tests can freeze it.""" + return _dt.datetime.now(_dt.UTC) + + +def _db_path(): + """Late-bound HITL_STATE_DB lookup — see module-level discipline rule.""" + import compass.config as cfg + + cfg.HITL_STATE_DB.parent.mkdir(parents=True, exist_ok=True) + return cfg.HITL_STATE_DB + + +async def _connect() -> aiosqlite.Connection: + conn = await aiosqlite.connect(_db_path()) + conn.row_factory = aiosqlite.Row + # WAL mode lets concurrent readers + a single writer coexist without + # exclusive-lock contention. Phase 1.B.3 Modal cron + a human pressing + # `approve` in MCP will race on this file; cheaper to set the pragma now + # than debug a flaky lock-timeout in production. PRAGMA is per-DB and + # persists across opens. + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA busy_timeout=5000") # 5s retry on transient lock + await conn.executescript(_SCHEMA) + await conn.commit() + return conn + + +def _row_to_dict(row: aiosqlite.Row) -> dict[str, Any]: + d = dict(row) + d["matched_skills"] = json.loads(d["matched_skills"]) + d["missing_skills"] = json.loads(d["missing_skills"]) + return d + + +async def add_pending( + *, + thread_id: str, + job_url: str, + company: str, + title: str, + score: float, + score_reasoning: str, + matched_skills: list[str], + missing_skills: list[str], +) -> None: + """Insert a new pending row. INSERT OR IGNORE — re-pausing the same thread_id is a no-op.""" + conn = await _connect() + try: + cursor = await conn.execute( + """ + INSERT OR IGNORE INTO pending_approvals + (thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills, created_at, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending') + """, + ( + thread_id, + job_url, + company, + title, + float(score), + score_reasoning, + json.dumps(matched_skills), + json.dumps(missing_skills), + _now().isoformat(), + ), + ) + await conn.commit() + if cursor.rowcount == 0: + logger.info("hitl: re-pause ignored for thread_id=%s (already pending)", thread_id) + finally: + await conn.close() + + +async def get_pending(thread_id: str) -> dict[str, Any] | None: + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + row = await cur.fetchone() + finally: + await conn.close() + return _row_to_dict(row) if row else None + + +async def list_pending() -> list[dict[str, Any]]: + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE status = 'pending' ORDER BY created_at ASC" + ) as cur: + rows = await cur.fetchall() + finally: + await conn.close() + return [_row_to_dict(r) for r in rows] + + +async def list_timed_out(*, timeout_hours: int) -> list[dict[str, Any]]: + cutoff = (_now() - _dt.timedelta(hours=timeout_hours)).isoformat() + conn = await _connect() + try: + async with conn.execute( + "SELECT * FROM pending_approvals " + "WHERE status = 'pending' AND created_at < ? " + "ORDER BY created_at ASC", + (cutoff,), + ) as cur: + rows = await cur.fetchall() + finally: + await conn.close() + return [_row_to_dict(r) for r in rows] + + +async def claim_pending(thread_id: str) -> bool: + """Atomic single-writer claim: transitions `pending → resuming` if and only + if the row is currently 'pending'. Returns True on successful claim, + False if another consumer beat us to it (row is already resuming, approved, + rejected, timed_out, or doesn't exist). + + This is the race fix between Modal cron's timeout-checker and an MCP + `approve_job` call landing on the same thread_id at the same time. + Without it, both consumers' get_pending+status='pending' check would + pass, both would call graph.ainvoke on the same checkpoint, and we'd + get conflicting JobNote writes (one "timed_out", one "approved"). + """ + conn = await _connect() + try: + cursor = await conn.execute( + "UPDATE pending_approvals SET status = 'resuming' " + "WHERE thread_id = ? AND status = 'pending'", + (thread_id,), + ) + await conn.commit() + return cursor.rowcount > 0 + finally: + await conn.close() + + +async def mark_resolved( + thread_id: str, + *, + status: str, + feedback: str | None = None, +) -> None: + if status not in _VALID_STATUSES or status in ("pending", "resuming"): + raise ValueError(f"invalid resolve status: {status!r}") + + conn = await _connect() + try: + # Atomic guarded UPDATE: only transitions rows currently in-flight + # (pending = never claimed; resuming = claim_pending succeeded but + # the graph hasn't finalized yet). Accepts both so callers can either + # claim+finalize (the new race-safe path) or directly finalize a + # never-claimed row (the legacy/test path). + cursor = await conn.execute( + "UPDATE pending_approvals " + "SET status = ?, feedback = ?, resolved_at = ? " + "WHERE thread_id = ? AND status IN ('pending', 'resuming')", + (status, feedback, _now().isoformat(), thread_id), + ) + await conn.commit() + + if cursor.rowcount == 0: + # Distinguish "no such thread" from "already resolved" for the caller. + async with conn.execute( + "SELECT status FROM pending_approvals WHERE thread_id = ?", + (thread_id,), + ) as cur: + existing = await cur.fetchone() + if existing is None: + raise LookupError(f"no pending row for thread_id {thread_id!r}") + raise ValueError(f"thread {thread_id!r} already resolved ({existing['status']!r})") + finally: + await conn.close() + + logger.info("hitl: thread %s resolved as %s", thread_id, status) diff --git a/compass/hitl/timeout_checker.py b/compass/hitl/timeout_checker.py new file mode 100644 index 0000000..7e81867 --- /dev/null +++ b/compass/hitl/timeout_checker.py @@ -0,0 +1,75 @@ +"""Auto-cancel pending approvals older than HITL_TIMEOUT_HOURS. + +Module entrypoint (CLI): python -m compass.hitl.timeout_checker +Async API: check_and_resume_timeouts(timeout_hours: int | None = None) + +In Phase 1.B.1 this is human-run. In Phase 1.B.3 a Modal cron will import and +schedule it — see modal_app.py.""" + +from __future__ import annotations + +import asyncio +import logging + +from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX, state_store +from compass.hitl.resume import resume_pending + +logger = logging.getLogger(__name__) + + +async def check_and_resume_timeouts(*, timeout_hours: int | None = None) -> int: + """Resume every stale pending row with {"approved": False}. Returns the + count successfully timed-out (i.e. exclude rows that errored).""" + from compass.config import HITL_TIMEOUT_HOURS + + hrs = timeout_hours if timeout_hours is not None else HITL_TIMEOUT_HOURS + stale = await state_store.list_timed_out(timeout_hours=hrs) + if not stale: + return 0 + + logger.info("hitl: %d pending approval(s) past %dh timeout", len(stale), hrs) + timed_out = 0 + for row in stale: + tid = row["thread_id"] + try: + await resume_pending( + tid, + decision={ + "approved": False, + "feedback": f"{HITL_TIMEOUT_FEEDBACK_PREFIX} {hrs}h timeout", + }, + status_override="timed_out", + ) + # Spec § DoD line 230: HiTL timeout must log to _meta/agent-log.md + _log_to_agent_log( + f"hitl-timeout: auto-cancelled {tid} ({row['company']} / {row['title']}, " + f"score={row['score']:.2f}) after {hrs}h" + ) + timed_out += 1 + except Exception as e: + logger.exception("hitl: timeout resume failed for %s — marking as error", tid) + try: + await state_store.mark_resolved(tid, status="error", feedback=f"resume error: {e}") + _log_to_agent_log(f"hitl-timeout-error: {tid} resume failed: {e}") + except Exception: + logger.exception("hitl: also failed to mark %s as error", tid) + return timed_out + + +def _log_to_agent_log(line: str) -> None: + """Append a timestamped row to compass-vault/_meta/agent-log.md. + + Late-binds via compass.vault.writer.append_agent_log so the temp_vault + fixture works in tests. + """ + from compass.vault.writer import append_agent_log + + try: + append_agent_log(line) + except Exception: + logger.exception("hitl: failed to write to agent-log.md") + + +if __name__ == "__main__": + n = asyncio.run(check_and_resume_timeouts()) + print(f"Timed out: {n}") diff --git a/compass/llm.py b/compass/llm.py index 318963a..b6201cb 100644 --- a/compass/llm.py +++ b/compass/llm.py @@ -30,11 +30,21 @@ def get_model_id(node: str) -> str: - """Return the OpenRouter model id for a node, reading env at call time.""" + """Return the OpenRouter model id for a node. + + Reads from `compass.config` rather than `os.environ` directly. That way + tests that monkeypatch `compass.config.SCORE_MODEL` (etc.) actually + affect which model the LLM call uses. Pre-fix, reading `os.environ` + directly bypassed the patched constants and tests silently exercised + the production model. + """ env_name = _NODE_ENV.get(node) if env_name is None: raise ValueError(f"unknown node {node!r}; expected one of {sorted(_NODE_ENV)}") - model_id = os.environ.get(env_name) + # Late-bind via cfg. so monkeypatched test values take effect. + import compass.config as cfg + + model_id = getattr(cfg, env_name, None) or os.environ.get(env_name) if not model_id: raise ValueError(f"no model configured for node {node!r} (env {env_name} unset)") return model_id @@ -42,7 +52,9 @@ def get_model_id(node: str) -> str: def _get_model(node: str) -> OpenAIChatModel: """Build a pydantic-ai OpenAIChatModel pointed at OpenRouter for this node.""" - api_key = os.environ.get("OPENROUTER_API_KEY") + import compass.config as cfg + + api_key = getattr(cfg, "OPENROUTER_API_KEY", None) or os.environ.get("OPENROUTER_API_KEY") if not api_key: raise ValueError("OPENROUTER_API_KEY is not set") provider = OpenAIProvider(base_url=OPENROUTER_BASE_URL, api_key=api_key) diff --git a/compass/mcp_server/server.py b/compass/mcp_server/server.py index 90674b0..c2eed4f 100644 --- a/compass/mcp_server/server.py +++ b/compass/mcp_server/server.py @@ -38,17 +38,27 @@ update_application_status(app_id, ...) -> transition status with optional next-action fields list_pending_actions(through_date) -> ApplicationNotes with due next_action_date tailor_resume(job_id) -> tailoring suggestions from JobNote frontmatter + + HiTL approvals: + pending_approvals() -> paused threads awaiting approval (oldest first) + approve(thread_id, approved, feedback) -> resume a paused thread; runs tailor on approve """ from __future__ import annotations +import logging + from mcp.server.fastmcp import FastMCP from compass.analysis import gap_aggregator, skill_assessor from compass.config import VAULT_PATH +from compass.hitl import state_store as _state_store +from compass.hitl.resume import resume_pending as _resume_pending from compass.vault.learning_bridge import path_to_uri, resolve, scan_evidence from compass.vault.taxonomy import all_canonicals +logger = logging.getLogger(__name__) + mcp = FastMCP("compass") @@ -90,6 +100,8 @@ async def score_jd(jd_text: str) -> dict: "jobs_processed": 0, "jobs_written": 0, "errors": [], + "thread_id": None, + "score_threshold": None, } extract_result = await extract_node(state) if extract_result.get("errors"): @@ -109,6 +121,215 @@ async def score_jd(jd_text: str) -> dict: } +@mcp.tool() +async def add_job_from_url( + url: str, + company: str | None = None, + title: str | None = None, +) -> dict: + """Fetch a JD by URL, run the full Compass pipeline, write a JobNote. + + Unlocks every site Compass's auto-scrapers can't reach: JPM Oracle + Cloud, Capital One, iCIMS, LinkedIn, custom careers pages. The user + finds the role manually and pastes the URL — Compass scores it. + + Strategy: + 1. If the URL matches a known ATS pattern (greenhouse/lever/ashby/ + workday public JSON), use the structured scraper for clean data. + 2. Otherwise, static-fetch the page + strip HTML. Many corporate + careers pages (Oracle Cloud, LinkedIn) are JS-rendered and will + return a near-empty body — in that case the caller should use + `add_job_from_text` and paste the JD body explicitly. + + Returns {"path": str, "score": float, "title": str, "company": str} on + success, {"error": str} on failure. Does NOT call tailor (skip Sonnet + cost on this codepath — caller can re-invoke tailor explicitly later). + """ + from compass.pipeline.add_url import fetch_rawjob_from_url + + try: + job = await fetch_rawjob_from_url(url, company=company, title=title) + except Exception as e: + return {"error": f"fetch failed: {type(e).__name__}: {e}"} + if job is None: + return {"error": "could not extract JD from URL — try add_job_from_text"} + return await _run_partial_pipeline_and_write(job) + + +@mcp.tool() +async def add_job_from_text( + company: str, + title: str, + url: str, + jd_text: str, +) -> dict: + """Run the Compass pipeline on a pasted JD (for sites we can't fetch). + + Use when `add_job_from_url` returns "could not extract" — typical for + JPM Oracle Cloud, LinkedIn JS-rendered pages, Workday tenants not in + the YAML. Paste the JD body from the browser; Compass scores it and + writes a JobNote with the original URL. + + Returns {"path": str, "score": float, ...} on success. + """ + from datetime import date + from urllib.parse import urlparse + + from compass.pipeline.state import RawJob + + # Same scheme allowlist as add_job_from_url — block file:/ftp:/javascript: + # /data: URLs from being persisted as JobNote.url. Without this, a typo or + # malicious paste could write a bogus URL into the vault. + scheme = (urlparse(url).scheme or "").lower() + if scheme not in {"http", "https"}: + return {"error": f"only http/https URLs allowed, got scheme={scheme!r}"} + if not company or not company.strip(): + return {"error": "company must be a non-empty string"} + if not title or not title.strip(): + return {"error": "title must be a non-empty string"} + + job = RawJob( + company=company, + title=title, + url=url, + source="manual", + description=jd_text, + date_posted=date.today(), + ) + return await _run_partial_pipeline_and_write(job) + + +async def _run_partial_pipeline_and_write(job) -> dict: + """Shared helper: intake_filter → extract → score → vault_write. + + Skips tailor (Sonnet cost) and HiTL (the user already chose to add this + JD by URL — they've implicitly approved). gap_aggregator NOT run inline; + the next batch run picks up the new JobNote. + """ + from compass.pipeline.nodes.extract import extract_node + from compass.pipeline.nodes.intake_filter import intake_filter_node + from compass.pipeline.nodes.score import score_node + from compass.pipeline.nodes.vault_write import vault_write_node + from compass.pipeline.state import CompassState # noqa: TC001 + + state: CompassState = { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": None, + "in_scope": None, + "role_family": None, + "agent_signal_count": None, + "human_approved": True, # implicit approval by adding via URL + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": None, + "score_threshold": None, + } + for node in (intake_filter_node, extract_node, score_node, vault_write_node): + update = await node(state) + state = {**state, **update} # type: ignore[typeddict-item] + if state.get("errors"): + return {"error": state["errors"][-1]} + if state.get("in_scope") is False: + return { + "error": "intake_filter dropped the JD — see _meta/filtered-jobs.md", + "role_family": state.get("role_family"), + } + score = state.get("score_result") + if score is None: + return {"error": "score node returned None"} + return { + "path": "vault/jobs/" + (job.company + "-" + job.title), + "company": job.company, + "title": job.title, + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": score.matched_skills, + "missing_skills": score.missing_skills, + "role_family": state.get("role_family"), + "agent_signal_count": state.get("agent_signal_count"), + } + + +@mcp.tool() +async def run_evals(mode: str = "labels", limit: int | None = None) -> dict: + """Run the Compass eval harness — measures extract + score accuracy. + + `mode`: + "labels" (default) — compare against EvalRecord.expected_* hand-labels + in compass/evals/labeled_dataset.json. + "judge" — LLM-as-judge mode, no hand labels needed. Use for + first-pass sanity checks. + + `limit`: optionally sample N records instead of the full dataset. + + Returns metrics summary + path to per-record results JSON. + """ + from compass.evals.runner import run_evals as _run + + result = await _run(mode=mode, limit=limit) + if "error" in result: + return {"error": result["error"]} + m = result["metrics"] + return { + "mode": mode, + "n_records": m.n, + "score_mae": round(m.score_mae, 3), + "score_rmse": round(m.score_rmse, 3), + "score_bias": round(m.score_bias, 3), + "extract_skill_recall": round(m.extract_skill_recall, 3), + "extract_skill_precision": round(m.extract_skill_precision, 3), + "match_skill_recall": round(m.match_skill_recall, 3), + "results_path": result["results_path"], + } + + +@mcp.tool() +async def generate_cover_letter(job_filename: str) -> dict: + """Draft a cover letter for a JobNote in the vault. + + `job_filename` is the base name of a file in `compass-vault/jobs/` + (e.g. `2026-05-19-databricks-AI_Engineer-abcdef12.md`). The tool reads + the JobNote's frontmatter (company / title / skills / JD summary), + pulls the company's targeting notes from YAML if available, and writes + a structured 250-400 word cover letter to + `compass-vault/cover-letters/`. + + Use this AFTER you decide to apply to a role — it's a Sonnet call so + don't burn it on the whole vault. + + Returns {"path": str, "preview": str (first 300 chars)} on success. + """ + from compass.config import VAULT_PATH + from compass.pipeline.cover_letter import generate_cover_letter_from_jobnote + + # Path-traversal guard: `job_filename` is user-supplied; reject anything + # that resolves outside compass-vault/jobs/. Without this, a string like + # "../_profile/resume.md" would load + cover-letter-ize the resume file. + jobs_dir = (VAULT_PATH / "jobs").resolve() + job_path = (VAULT_PATH / "jobs" / job_filename).resolve() + try: + job_path.relative_to(jobs_dir) + except ValueError: + return {"error": f"job_filename must be inside jobs/: {job_filename!r}"} + if not job_path.exists(): + return {"error": f"JobNote not found: {job_filename}"} + try: + out_path, body = await generate_cover_letter_from_jobnote(job_path) + except Exception as e: + logger.exception("generate_cover_letter: failed") + return {"error": f"{type(e).__name__}: {e}"} + return { + "path": str(out_path.relative_to(VAULT_PATH)), + "preview": body[:300] + ("…" if len(body) > 300 else ""), + } + + # ── Vault read ─────────────────────────────────────────────────────────────── @@ -151,8 +372,24 @@ def get_skill_gaps(job_id: str) -> dict: @mcp.tool() def get_profile(section: str) -> str: - """Read a file from _profile/. section is the bare filename (e.g. 'resume', 'skill-inventory').""" - path = VAULT_PATH / "_profile" / f"{section}.md" + """Read a file from _profile/. section is the bare filename (e.g. 'resume', 'skill-inventory'). + + SECURITY: validates that the resolved path stays inside `_profile/`. Pre-fix, + a section like `../.env` resolved to the vault root and could leak the + user's API keys. The same path-containment pattern is used in + `generate_cover_letter` and `_load_jobnote` for the same threat class. + + Late-binds VAULT_PATH via `cfg.VAULT_PATH` so the temp_vault test fixture's + monkeypatch works — see CLAUDE.md lesson #2. + """ + import compass.config as cfg + + profile_dir = (cfg.VAULT_PATH / "_profile").resolve() + path = (cfg.VAULT_PATH / "_profile" / f"{section}.md").resolve() + try: + path.relative_to(profile_dir) + except ValueError: + return f"(invalid section name — must be inside _profile/: {section!r})" if not path.exists(): return f"(no such profile section: {section})" return path.read_text(encoding="utf-8") @@ -329,5 +566,46 @@ async def tailor_resume(job_id: str) -> dict: } +# ── HiTL approvals ─────────────────────────────────────────────────────────── + + +@mcp.tool() +async def pending_approvals() -> list[dict]: + """List jobs paused at hitl awaiting human approval. Oldest first. + + Returned rows include: thread_id, job_url, company, title, score, + score_reasoning, matched_skills, missing_skills, created_at (ISO8601 UTC), + status, resolved_at, feedback. All values are JSON-safe primitives. + """ + rows = await _state_store.list_pending() + # rows are already plain dicts of JSON-safe primitives + list[str] + return rows + + +@mcp.tool() +async def approve(thread_id: str, approved: bool, feedback: str | None = None) -> dict: + """Resume a paused thread. approved=True runs tailor + vault_write; + approved=False skips tailor and writes the rejected JobNote. + + Returns {"vault_written": bool, "human_approved": bool, "human_feedback": str | None} + on success, or {"error": "..."} if the thread is unknown or already resolved. + """ + try: + final = await _resume_pending( + thread_id, + decision={"approved": approved, "feedback": feedback}, + ) + except (LookupError, ValueError) as e: + return {"error": str(e)} + # Strip non-JSON-safe state from the return (current_job is a Pydantic model, + # extracted_requirements / score_result are Pydantic). Phase 1.A bug #11 + # pattern: FastMCP cannot serialize Pydantic instances or datetime objects. + return { + "vault_written": bool(final.get("vault_written")), + "human_approved": bool(final.get("human_approved")), + "human_feedback": final.get("human_feedback"), + } + + if __name__ == "__main__": mcp.run() diff --git a/compass/pipeline/add_url.py b/compass/pipeline/add_url.py new file mode 100644 index 0000000..1cf2cf4 --- /dev/null +++ b/compass/pipeline/add_url.py @@ -0,0 +1,150 @@ +"""Fetch a single job posting by URL and build a RawJob. + +Routes by URL pattern: + - boards.greenhouse.io / job-boards.greenhouse.io → greenhouse public API + - jobs.lever.co → lever public API + - jobs.ashbyhq.com → ashby public API + - *.myworkdayjobs.com → workday JSON endpoint + - everything else → generic static fetch + strip + +Generic fetch is best-effort. JS-rendered pages (Oracle Cloud, LinkedIn, +some custom careers UIs) return near-empty bodies; in that case the caller +should fall back to `add_job_from_text` with a manual paste. +""" + +from __future__ import annotations + +import html +import logging +import re +from datetime import date +from urllib.parse import urlparse + +import httpx + +from compass.pipeline.state import RawJob + +logger = logging.getLogger(__name__) + +_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0 Safari/537.36 compass-job-scraper/0.1" +) +_TIMEOUT = 20.0 +_SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.DOTALL | re.IGNORECASE) +_TAG_RE = re.compile(r"<[^>]+>") +_TITLE_RE = re.compile(r"]*>([^<]+)", re.IGNORECASE) + + +def _strip_html(raw: str) -> str: + text = _SCRIPT_STYLE_RE.sub(" ", raw) + text = _TAG_RE.sub(" ", text) + text = html.unescape(text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def _detect_provider(url: str) -> str: + """Return 'greenhouse' | 'lever' | 'ashby' | 'workday' | 'generic'.""" + host = (urlparse(url).hostname or "").lower() + if "greenhouse.io" in host: + return "greenhouse" + if "lever.co" in host: + return "lever" + if "ashbyhq.com" in host: + return "ashby" + if "myworkdayjobs.com" in host: + return "workday" + return "generic" + + +async def _fetch_generic(url: str) -> tuple[str | None, str | None]: + """Static-fetch a page + return (page_title, stripped_body). The body + may be near-empty for JS-rendered pages — caller decides what to do.""" + try: + async with httpx.AsyncClient( + timeout=_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "text/html,*/*"}, + follow_redirects=True, + ) as c: + r = await c.get(url) + if r.status_code != 200: + logger.warning("add_url: generic fetch %s returned status=%s", url, r.status_code) + return None, None + html_text = r.text + except httpx.HTTPError as e: + logger.warning("add_url: generic fetch %s: %s", url, e) + return None, None + title_m = _TITLE_RE.search(html_text) + page_title = title_m.group(1).strip() if title_m else None + body = _strip_html(html_text) + return page_title, body + + +_ALLOWED_SCHEMES = {"http", "https"} + + +async def fetch_rawjob_from_url( + url: str, + *, + company: str | None = None, + title: str | None = None, +) -> RawJob | None: + """Return a RawJob built from `url`, or None when the body looks empty. + + `company` and `title` overrides bypass any structured-API detection — + useful for generic URLs where the page title isn't a clean role name. + Both default to inferred values from the URL's host or page . + + Returns None (not raise) when extraction fails so MCP callers can show + a clean "try paste-text instead" message. Also returns None — without + even attempting the fetch — when the URL scheme isn't http(s); avoids + accidental file://, ftp://, javascript:, data: URLs reaching httpx. + """ + parts_for_scheme_check = urlparse(url) + if parts_for_scheme_check.scheme.lower() not in _ALLOWED_SCHEMES: + logger.warning( + "add_url: refusing non-http(s) URL: scheme=%r url=%r", + parts_for_scheme_check.scheme, + url[:80], + ) + return None + if not parts_for_scheme_check.hostname: + logger.warning("add_url: refusing URL with no hostname: %r", url[:80]) + return None + + # Provider detection is exposed for tests + future structured-API + # routing. For one-off URLs the generic static path is fast enough. + _detect_provider(url) + + # Structured-API path: not implemented here — for one-off URLs the + # generic path is fast enough and avoids per-provider parsing edge + # cases. The full-batch scrapers handle structured fetching. + page_title, body = await _fetch_generic(url) + if not body or len(body) < 200: + # Likely JS-rendered (Oracle Cloud / LinkedIn / etc.). Drop — + # caller should use add_job_from_text. + logger.info("add_url: body too short (%d chars) — likely JS-rendered", len(body or "")) + return None + + inferred_company = company + if not inferred_company: + host = (urlparse(url).hostname or "").lower() + # Cheap heuristic: pull the company from the host's first label. + # E.g. jobs.ashbyhq.com/sierra/... → use page_title; jpmc.fa.oraclecloud.com + # → 'jpmc'. The user is encouraged to pass `company` explicitly when this + # is wrong. + parts = host.split(".") + inferred_company = parts[0] if parts and parts[0] not in {"jobs", "www"} else "(unknown)" + + inferred_title = title or page_title or "(unknown)" + + return RawJob( + company=inferred_company, + title=inferred_title, + url=url, + source="manual", + description=body, + date_posted=date.today(), + ) diff --git a/compass/pipeline/cover_letter.py b/compass/pipeline/cover_letter.py new file mode 100644 index 0000000..3040b86 --- /dev/null +++ b/compass/pipeline/cover_letter.py @@ -0,0 +1,165 @@ +"""Cover-letter generator — on-demand, per-application. + +Not part of the per-job graph (too expensive to run on every scrape). Invoked +via the MCP tool `generate_cover_letter(job_id)` when the user is about to +apply to a specific role. + +Saves output to `compass-vault/cover-letters/{date}-{company}-{title}-{hash}.md` +so the user has a discoverable record per JD without cluttering JobNote +frontmatter. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from datetime import date +from typing import TYPE_CHECKING + +import frontmatter +from pydantic import BaseModel + +if TYPE_CHECKING: + from pathlib import Path + +from compass.config import VAULT_PATH +from compass.llm import make_agent +from compass.vault.reader import read_profile_section, read_resume + +logger = logging.getLogger(__name__) + + +class CoverLetterDraft(BaseModel): + """Structured cover-letter output.""" + + opening: str + body: str # 2-3 paragraphs body, separated by blank lines + closing: str + + +_SYSTEM_PROMPT = """You write cover letters for a 1.5-YoE software engineer +applying to AI / agent / applied-AI roles. + +Output a SHORT, SPECIFIC cover letter. Three sections: +- opening: 2-3 sentences. State the role you're applying for and ONE concrete + reason this candidate is a good fit (not generic enthusiasm). Reference a + named project or skill from the candidate profile that maps directly to the + JD's top requirement. +- body: 2-3 paragraphs (3-5 sentences each). Each paragraph anchors on a + specific project or experience from the candidate profile that maps to the + JD's required skills. Include concrete numbers when the profile provides + them. Do NOT invent projects or claims. +- closing: 2-3 sentences. State why this company specifically (use the + company notes from the targeting context if provided). End with a + forward-looking next step. + +STRICT RULES: +- Never invent projects, employers, numbers, or claims. Only use facts from + the CANDIDATE PROFILE and ROLE CLARIFICATIONS sections. +- Never use clichés like "I am writing to apply for...", "I am excited to + apply...", "passionate about", "team player", "results-oriented". +- Plain prose. No emoji, no bullet points, no markdown. +- Total length 250-400 words across all three sections. +- Match the candidate's voice from the resume — direct, technical, specific. +""" + + +def _build_agent(): + return make_agent("tailor", output_type=CoverLetterDraft, system_prompt=_SYSTEM_PROMPT) + + +async def _draft( + company: str, + title: str, + jd_summary: str, + required: list[str], + nice_to_have: list[str], + matched: list[str], + missing: list[str], + profile_text: str, + company_notes: str | None, +) -> CoverLetterDraft: + agent = _build_agent() + prompt = ( + f"CANDIDATE PROFILE\n{profile_text}\n\n" + f"# COMPANY\n{company}\n" + f"# ROLE\n{title}\n" + f"# JD SUMMARY\n{jd_summary}\n\n" + f"required: {', '.join(required) or '(none)'}\n" + f"nice-to-have: {', '.join(nice_to_have) or '(none)'}\n" + f"already-matched: {', '.join(matched) or '(none)'}\n" + f"to-shore-up: {', '.join(missing) or '(none)'}\n" + ) + if company_notes: + prompt += f"\n# WHY THIS COMPANY (from candidate notes)\n{company_notes}\n" + result = await agent.run(prompt) + return result.output + + +_FILENAME_BAD = re.compile(r"[^\w\-.]+") + + +def _safe_segment(s: str) -> str: + return _FILENAME_BAD.sub("_", s).strip("_") + + +def _cover_letter_path(company: str, title: str, job_url: str) -> Path: + """Mirror the JobNote filename pattern so cover-letter files line up + visually with the JobNotes they target.""" + today = date.today().isoformat() + url_suffix = hashlib.sha1(job_url.encode("utf-8")).hexdigest()[:8] + return ( + VAULT_PATH + / "cover-letters" + / f"{today}-{_safe_segment(company)}-{_safe_segment(title)}-{url_suffix}.md" + ) + + +async def generate_cover_letter_from_jobnote(job_path: Path) -> tuple[Path, str]: + """Read a JobNote from disk, draft a cover letter, write it next to the + JobNote. Returns (output_path, body_text).""" + from compass.vault.target_companies import get_company_meta + + post = frontmatter.load(job_path) + md = post.metadata + company = str(md.get("company") or "(unknown)") + title = str(md.get("title") or "(unknown)") + job_url = str(md.get("url") or job_path.name) + jd_summary = str(md.get("jd_summary") or post.content[:1000]) + required = list(md.get("skills_required") or []) + nice_to_have = list(md.get("skills_nice_to_have") or []) + matched = list(md.get("skills_matched") or []) + missing = list(md.get("skills_missing") or []) + + profile = f"{read_resume()}\n\n{read_profile_section('role-clarifications')}" + company_meta = get_company_meta(company) + company_notes = (company_meta.get("notes") if company_meta else None) or None + + draft = await _draft( + company, + title, + jd_summary, + required, + nice_to_have, + matched, + missing, + profile, + company_notes, + ) + + body = f"{draft.opening}\n\n{draft.body}\n\n{draft.closing}\n" + out_path = _cover_letter_path(company, title, job_url) + out_path.parent.mkdir(parents=True, exist_ok=True) + front = { + "type": "cover-letter", + "company": company, + "title": title, + "job_ref": job_url, + "drafted_at": date.today().isoformat(), + } + out_path.write_text( + frontmatter.dumps(frontmatter.Post(body, **front)) + "\n", + encoding="utf-8", + ) + return out_path, body diff --git a/compass/pipeline/graph.py b/compass/pipeline/graph.py index 7d6dd91..c76e6e3 100644 --- a/compass/pipeline/graph.py +++ b/compass/pipeline/graph.py @@ -17,15 +17,19 @@ from __future__ import annotations import asyncio +import hashlib import logging import time from datetime import datetime import frontmatter +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from langgraph.graph import END, START, StateGraph from compass.analysis import gap_aggregator from compass.config import MAX_CONCURRENT_JOBS, VAULT_PATH +from compass.hitl import state_store from compass.pipeline.nodes.extract import extract_node from compass.pipeline.nodes.hitl import hitl_node from compass.pipeline.nodes.intake import intake_node @@ -61,7 +65,22 @@ def _route_after_hitl(state: CompassState) -> str: return "vault_write" -def build_graph(): +def _build_checkpoint_serde() -> JsonPlusSerializer: + """Allow our Pydantic state classes on the msgpack allowlist. + + Must be set at construction — `with_msgpack_allowlist` is a no-op when the + default `allowed_msgpack_modules=True` (langgraph's non-STRICT default). + """ + return JsonPlusSerializer( + allowed_msgpack_modules=[ + ("compass.pipeline.state", "RawJob"), + ("compass.pipeline.state", "JobRequirements"), + ("compass.pipeline.state", "JobScore"), + ] + ) + + +def build_graph(checkpointer=None): builder = StateGraph(CompassState) builder.add_node("intake", intake_node) builder.add_node("intake_filter", intake_filter_node) @@ -93,15 +112,22 @@ def build_graph(): builder.add_edge("tailor", "vault_write") builder.add_edge("vault_write", END) - return builder.compile() + return builder.compile(checkpointer=checkpointer) def _vault_url_set() -> set[str]: - """Build the set of URLs already in the vault — ONCE per batch. + """Build the set of NORMALIZED URLs already in the vault — ONCE per batch. + + Normalization (`compass.vault.url_dedup.normalize_url`) collapses + case/scheme/trailing-slash/utm-params variants so the same job seen via + Google and via LinkedIn dedups to one. Without it, the pipeline used to + write duplicate JobNotes for cosmetically-different URLs. A malformed frontmatter file is logged (NOT silently dropped from the set). Without the log, a corrupt note silently causes a duplicate write next run. """ + from compass.vault.url_dedup import normalize_url + urls: set[str] = set() for path in list_job_notes(): try: @@ -111,11 +137,23 @@ def _vault_url_set() -> set[str]: continue url = post.metadata.get("url") if isinstance(url, str): - urls.add(url) + urls.add(normalize_url(url)) return urls -def _initial_state(job: RawJob) -> CompassState: +def _thread_id_for(job_url: str, batch_started_at: datetime) -> str: + """Deterministic 16-char SHA-1 of (url, batch start, pid) — same batch + same job = same thread. + + PID is included to disambiguate cross-process collisions (e.g. local CLI + and Modal cron both starting at the same wall-clock microsecond in 1.B.3). + """ + import os + + raw = f"{job_url}|{batch_started_at.isoformat()}|{os.getpid()}" + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] + + +def _initial_state(job: RawJob, thread_id: str | None = None) -> CompassState: return { "raw_jobs": [], "current_job": job, @@ -123,6 +161,7 @@ def _initial_state(job: RawJob) -> CompassState: "score_result": None, "in_scope": None, "role_family": None, + "agent_signal_count": None, "human_approved": None, "human_feedback": None, "tailored_paragraph": None, @@ -130,61 +169,151 @@ def _initial_state(job: RawJob) -> CompassState: "jobs_processed": 0, "jobs_written": 0, "errors": [], + "thread_id": thread_id, + "score_threshold": None, } +_langfuse_client_initialized = False + + def _langfuse_config() -> dict: """Return a LangGraph config dict with the Langfuse callback if usable. + Langfuse 4.x split credentials away from `CallbackHandler.__init__` — the + callback now reads from a process-global `Langfuse(...)` singleton (or + LANGFUSE_* env vars). Earlier code passed `host=`/`secret_key=` directly + to `CallbackHandler`, which raises TypeError on 4.x and silently disabled + all tracing. + + Pattern: + 1. Initialize the Langfuse client once (idempotent — singleton). + 2. Construct a CallbackHandler that picks up the singleton's config. + Returns {} when Langfuse env is unset or langfuse fails to import — the - pipeline never blocks on observability. This wires traces from Phase 0.B - onward so by 2.B (public-trace polish) we have history to show. + pipeline never blocks on observability. """ + global _langfuse_client_initialized from compass.config import LANGFUSE_HOST, LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY if not (LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY): return {} try: + from langfuse import Langfuse from langfuse.langchain import CallbackHandler - handler = CallbackHandler( - host=LANGFUSE_HOST, - public_key=LANGFUSE_PUBLIC_KEY, - secret_key=LANGFUSE_SECRET_KEY, - ) + if not _langfuse_client_initialized: + Langfuse( + host=LANGFUSE_HOST, + public_key=LANGFUSE_PUBLIC_KEY, + secret_key=LANGFUSE_SECRET_KEY, + ) + _langfuse_client_initialized = True + handler = CallbackHandler() return {"callbacks": [handler]} except Exception as e: logger.warning("langfuse: failed to init callback, continuing without traces — %s", e) return {} -async def _process_one(graph, job: RawJob, sem: asyncio.Semaphore) -> CompassState: +async def _process_one( + graph, + job: RawJob, + sem: asyncio.Semaphore, + thread_id: str, +) -> tuple[CompassState, bool]: + """Invoke the graph for one job. Returns (final_state, was_paused).""" + config = { + "configurable": {"thread_id": thread_id}, + **_langfuse_config(), + } async with sem: try: - return await graph.ainvoke(_initial_state(job), config=_langfuse_config()) + result = await graph.ainvoke(_initial_state(job, thread_id=thread_id), config=config) except Exception as e: - # logger.exception preserves the traceback to stderr; the string version - # below is still aggregated into state for the run summary. logger.exception("pipeline: graph crashed on %s", job.url) - return {**_initial_state(job), "errors": [f"graph: {type(e).__name__}: {e}"]} + return ( + { + **_initial_state(job, thread_id=thread_id), + "errors": [f"graph: {type(e).__name__}: {e}"], + }, + False, + ) + + # When interrupt() fires, ainvoke returns with state containing the + # __interrupt__ marker AND without progressing past the hitl node. + # vault_written stays False, jobs_written stays 0 — that's our signal. + interrupts = result.get("__interrupt__") + if interrupts: + payload = interrupts[0].value if hasattr(interrupts[0], "value") else interrupts[0] + if isinstance(payload, dict) and payload.get("kind") == "approval_request": + await state_store.add_pending( + thread_id=thread_id, + job_url=payload["job_url"], + company=payload["company"], + title=payload["title"], + score=float(payload["score"]), + score_reasoning=payload["score_reasoning"], + matched_skills=list(payload["matched_skills"]), + missing_skills=list(payload["missing_skills"]), + ) + logger.info( + "pipeline: paused %s for approval (thread_id=%s, score=%.2f)", + job.url, + thread_id, + payload["score"], + ) + return (result, True) + # Unknown interrupt shape — DO NOT silently swallow. Phase 0 bug pattern: + # a future interrupt() added elsewhere in the graph would otherwise + # disappear into a "succeeded with jobs_paused=0" black hole. Log loudly + # and still count as paused so the caller's bookkeeping reflects reality. + logger.error( + "pipeline: graph paused at UNKNOWN interrupt kind for %s — payload=%r", + job.url, + payload, + ) + return (result, True) + return (result, False) async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: - """Scrape (or accept) jobs, dedup, run per-job graph, regenerate gap plan.""" + """Scrape (or accept) jobs, dedup, run per-job graph under a single + AsyncSqliteSaver, regenerate gap plan.""" + from compass.config import HITL_CHECKPOINT_DB + start_monotonic = time.monotonic() start_wall = datetime.now() # captured alongside monotonic; immune to NTP/DST mid-run if raw_jobs is None: raw_jobs = await _scrape_all() + from compass.vault.url_dedup import normalize_url as _norm_url + seen_urls = _vault_url_set() - fresh = [j for j in raw_jobs if j.url not in seen_urls] + fresh = [j for j in raw_jobs if _norm_url(j.url) not in seen_urls] dropped = len(raw_jobs) - len(fresh) if dropped: logger.info("pipeline: dropping %d/%d jobs already in vault", dropped, len(raw_jobs)) - graph = build_graph() - sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) - results = await asyncio.gather(*[_process_one(graph, j, sem) for j in fresh]) + HITL_CHECKPOINT_DB.parent.mkdir(parents=True, exist_ok=True) + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() + # Enable WAL on the checkpoint DB once per process. Cheap if already set. + # See state_store._connect for the rationale. + try: + await checkpointer.conn.execute("PRAGMA journal_mode=WAL") + await checkpointer.conn.execute("PRAGMA busy_timeout=5000") + except Exception: + logger.debug("checkpoint: WAL pragma set already or unsupported; continuing") + graph = build_graph(checkpointer=checkpointer) + sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) + coros = [ + _process_one(graph, j, sem, thread_id=_thread_id_for(j.url, start_wall)) for j in fresh + ] + results = await asyncio.gather(*coros) + + paused_count = sum(int(p) for _, p in results) + final_states = [s for s, _ in results] aggregate: CompassState = { "raw_jobs": raw_jobs, @@ -194,13 +323,17 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: "human_approved": None, "human_feedback": None, "tailored_paragraph": None, - "vault_written": any(r.get("vault_written") for r in results), + "vault_written": any(r.get("vault_written") for r in final_states), "jobs_processed": len(fresh), - "jobs_written": sum(int(bool(r.get("vault_written"))) for r in results), - "errors": [e for r in results for e in r.get("errors", [])], + "jobs_written": sum(int(bool(r.get("vault_written"))) for r in final_states), + "errors": [e for r in final_states for e in r.get("errors", [])], "in_scope": None, "role_family": None, + "thread_id": None, + "score_threshold": None, } + # `jobs_paused` is informational only — not in CompassState TypedDict. + aggregate["jobs_paused"] = paused_count # type: ignore[typeddict-unknown-key] if aggregate["jobs_written"] > 0: gap_aggregator.regenerate(write=True) @@ -209,9 +342,10 @@ async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: _append_run_log(aggregate, duration_s) unknown_count = _count_unknown_skills_seen_this_run(start_wall) logger.info( - "pipeline: processed=%d written=%d errors=%d unknown_skills_seen=%d duration=%.1fs", + "pipeline: processed=%d written=%d paused=%d errors=%d unknown_skills_seen=%d duration=%.1fs", aggregate["jobs_processed"], aggregate["jobs_written"], + paused_count, len(aggregate["errors"]), unknown_count, duration_s, @@ -223,22 +357,64 @@ def _append_run_log(state: CompassState, duration_s: float) -> None: """Append one row per run to `_meta/pipeline-runs.md` for debugging cron failures.""" log_path = VAULT_PATH / "_meta" / "pipeline-runs.md" log_path.parent.mkdir(parents=True, exist_ok=True) + expected_header = "| Timestamp | Processed | Written | Paused | Errors | Duration |" + expected_separator = "|---|---|---|---|---|---|" + if not log_path.exists(): log_path.write_text( - "# Pipeline Run Log\n\n" - "| Timestamp | Processed | Written | Errors | Duration |\n" - "|---|---|---|---|---|\n", + f"# Pipeline Run Log\n\n{expected_header}\n{expected_separator}\n", encoding="utf-8", ) + else: + # One-time migration: pre-1.B.1 logs are 5-column. Insert a Paused=0 + # column into existing rows so Dataview parses the table correctly. + text = log_path.read_text(encoding="utf-8") + if expected_header not in text: + _migrate_run_log(log_path, text, expected_header, expected_separator) + ts = datetime.now().isoformat(timespec="seconds") + paused = state.get("jobs_paused", 0) # type: ignore[typeddict-item] row = ( f"| {ts} | {state['jobs_processed']} | {state['jobs_written']} | " - f"{len(state['errors'])} | {duration_s:.1f}s |\n" + f"{paused} | {len(state['errors'])} | {duration_s:.1f}s |\n" ) with log_path.open("a", encoding="utf-8") as f: f.write(row) +def _migrate_run_log(log_path, text: str, expected_header: str, expected_separator: str) -> None: + """One-time migration: insert Paused=0 column into existing 5-col rows. + + Pre-1.B.1 rows: | ts | processed | written | errors | duration | + Post-1.B.1 rows: | ts | processed | written | paused | errors | duration | + """ + new_lines = [] + in_table = False + migrated_header = False + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("| Timestamp"): + new_lines.append(expected_header) + in_table = True + migrated_header = True + continue + if in_table and stripped.startswith("|---"): + new_lines.append(expected_separator) + continue + if in_table and stripped.startswith("|") and not stripped.startswith("| Timestamp"): + cells = [c.strip() for c in stripped.strip("|").split("|")] + if len(cells) == 5: + ts, processed, written, errors, duration = cells + new_lines.append(f"| {ts} | {processed} | {written} | 0 | {errors} | {duration} |") + continue + elif len(cells) == 6: + new_lines.append(line) + continue + new_lines.append(line) + if migrated_header: + log_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8") + + def _count_unknown_skills_seen_this_run(start_wall: datetime) -> int: """Count rows appended to the unknown-skills log since `start_wall`. @@ -257,24 +433,62 @@ def _count_unknown_skills_seen_this_run(start_wall: datetime) -> int: ) -async def _scrape_all() -> list[RawJob]: - """Scrape all configured sources concurrently, interleave round-robin, cap. +MAX_POSTING_AGE_DAYS = 30 # Drop JDs older than this — postings >30d are usually filled or stale. + - Interleaving prevents a single high-volume source from exhausting - MAX_JOBS_PER_RUN before quieter sources get a chance. +async def _scrape_all() -> list[RawJob]: + """Scrape all configured sources concurrently, drop stale postings, sort + each board by recency, then interleave round-robin and cap. + + Targeting source of truth: `_profile/target-companies.yaml`. The YAML + drives which boards get hit — the legacy static `ASHBY_BOARDS` / + `GREENHOUSE_BOARDS` / `LEVER_COMPANIES` config lists are used only when + the YAML is missing or empty (fallback for tests + safety net). + + Order matters: + 1. Drop postings with `date_posted` older than MAX_POSTING_AGE_DAYS — stale + roles aren't worth LLM cost or vault clutter. + 2. Sort each board's results by date_posted DESC (None last) so when the + interleave-cap below fires, each board contributes its FRESHEST jobs. + 3. Round-robin interleave so a single high-volume board doesn't exhaust + MAX_JOBS_PER_RUN before quieter boards get a chance. + 4. Cap to MAX_JOBS_PER_RUN. """ from compass.config import ASHBY_BOARDS, GREENHOUSE_BOARDS, LEVER_COMPANIES, MAX_JOBS_PER_RUN from compass.scrapers.ashby import scrape_ashby_many from compass.scrapers.greenhouse import scrape_greenhouse_many from compass.scrapers.lever import scrape_lever_many + from compass.scrapers.workday import scrape_workday_many + + yaml_slugs = _yaml_scraper_slugs() + gh_slugs = yaml_slugs.get("greenhouse") or list(GREENHOUSE_BOARDS) + lv_slugs = yaml_slugs.get("lever") or list(LEVER_COMPANIES) + ash_slugs = yaml_slugs.get("ashby") or list(ASHBY_BOARDS) + wd_slugs = yaml_slugs.get("workday") or [] + + if yaml_slugs: + logger.info( + "scrape: YAML-driven targeting — greenhouse=%d lever=%d ashby=%d workday=%d boards", + len(gh_slugs), + len(lv_slugs), + len(ash_slugs), + len(wd_slugs), + ) - gh, lv, ash = await asyncio.gather( - scrape_greenhouse_many(GREENHOUSE_BOARDS), - scrape_lever_many(LEVER_COMPANIES), - scrape_ashby_many(ASHBY_BOARDS), + gh, lv, ash, wd = await asyncio.gather( + scrape_greenhouse_many(gh_slugs), + scrape_lever_many(lv_slugs), + scrape_ashby_many(ash_slugs), + scrape_workday_many(wd_slugs), ) + + gh = _filter_and_sort_by_recency(gh) + lv = _filter_and_sort_by_recency(lv) + ash = _filter_and_sort_by_recency(ash) + wd = _filter_and_sort_by_recency(wd) + interleaved: list[RawJob] = [] - iters = [iter(gh), iter(lv), iter(ash)] + iters = [iter(gh), iter(lv), iter(ash), iter(wd)] while iters: next_iters = [] for it in iters: @@ -287,10 +501,62 @@ async def _scrape_all() -> list[RawJob]: return interleaved[:MAX_JOBS_PER_RUN] +def _filter_and_sort_by_recency(jobs: list[RawJob]) -> list[RawJob]: + """Drop >30d-old postings (when date_posted is known); sort by date_posted + DESC with None-dated jobs sinking to the bottom. Postings without a + `date_posted` aren't dropped — many ATSes don't expose it consistently and + we can't tell stale from undated.""" + from datetime import date, timedelta + + cutoff = date.today() - timedelta(days=MAX_POSTING_AGE_DAYS) + kept = [j for j in jobs if j.date_posted is None or j.date_posted >= cutoff] + kept.sort(key=lambda j: (j.date_posted is None, -_date_to_ordinal(j.date_posted))) + return kept + + +def _yaml_scraper_slugs() -> dict[str, list[str]]: + """Read `_profile/target-companies.yaml` and group apply-now + opportunistic + boards by ATS provider. Returns {} when the YAML is missing — caller falls + back to the static config lists. + + Tiers below `opportunistic` (backend-prep, stretch) are NOT scraped: the + 3-month pivot defers those companies, and scraping them wastes LLM cost. + """ + from compass.vault.target_companies import list_yaml_companies, refresh_yaml + + refresh_yaml() + by_provider: dict[str, list[str]] = { + "greenhouse": [], + "lever": [], + "ashby": [], + "workday": [], + } + for tier in ("apply-now", "opportunistic"): + for entry in list_yaml_companies(tier_filter=tier): + ats = entry.get("ats") or {} + provider = (ats.get("provider") or "").lower() + slug = ats.get("slug") + if provider not in by_provider or not slug: + continue + if slug not in by_provider[provider]: + by_provider[provider].append(slug) + return {k: v for k, v in by_provider.items() if v} + + +def _date_to_ordinal(d: object) -> int: + """date.toordinal() wrapper that treats None as 0 — only used for sort key.""" + from datetime import date + + if isinstance(d, date): + return d.toordinal() + return 0 + + if __name__ == "__main__": result = asyncio.run(run_pipeline()) print( f"Processed: {result['jobs_processed']} | " f"Written: {result['jobs_written']} | " + f"Paused: {result.get('jobs_paused', 0)} | " f"Errors: {len(result['errors'])}" ) diff --git a/compass/pipeline/nodes/hitl.py b/compass/pipeline/nodes/hitl.py index 3fc29c4..743ec68 100644 --- a/compass/pipeline/nodes/hitl.py +++ b/compass/pipeline/nodes/hitl.py @@ -1,10 +1,10 @@ -""" -hitl_node — Phase 0.B auto-approve based on SCORE_THRESHOLD. +"""hitl_node — Phase 1.B.1 real human-in-the-loop via LangGraph interrupt(). -This is INTENTIONALLY auto-approve for 0.B. Real LangGraph `interrupt()` + -`AsyncSqliteSaver` checkpointing + an MCP-tool-driven approval surface ship in -Phase 1.B per the spec. Auto-approve unblocks the end-to-end pipeline so we -can collect real eval data first. +Behaviour: + * score < SCORE_THRESHOLD or score missing -> auto-reject, no interrupt fires + * score >= SCORE_THRESHOLD -> interrupt() with the approval payload; the + orchestrator catches the interrupt and registers the thread in + compass.hitl.state_store. A human resumes via Command(resume={...}). """ from __future__ import annotations @@ -12,6 +12,8 @@ import logging from typing import TYPE_CHECKING +from langgraph.types import interrupt + from compass.config import SCORE_THRESHOLD if TYPE_CHECKING: @@ -21,20 +23,44 @@ async def hitl_node(state: CompassState) -> dict: - """Auto-approve if score >= SCORE_THRESHOLD, else reject. No human interaction in 0.B.""" score = state.get("score_result") - if score is None: - logger.info("hitl: no score_result, rejecting by default") + job = state.get("current_job") + + # Prefer the threshold captured at score time (sticky across pause/resume). + # Falls back to live config only for backward-compat with paused threads + # checkpointed before this field existed. + threshold = state.get("score_threshold") + if threshold is None: + threshold = SCORE_THRESHOLD + + if score is None or score.score < threshold: + logger.info( + "hitl: auto-reject %s (score=%s, threshold=%.2f)", + job.url if job else "(unknown)", + getattr(score, "score", None), + threshold, + ) return {"human_approved": False} - approved = score.score >= SCORE_THRESHOLD - job = state.get("current_job") - job_id = job.url if job else "(unknown)" + payload = { + "kind": "approval_request", + "job_url": job.url if job else "", + "company": job.company if job else "", + "title": job.title if job else "", + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": list(score.matched_skills), + "missing_skills": list(score.missing_skills), + } logger.info( - "hitl: auto-%s job %s (score=%.2f, threshold=%.2f)", - "approve" if approved else "reject", - job_id, - score.score, - SCORE_THRESHOLD, + "hitl: interrupting for approval — %s (score=%.2f)", payload["job_url"], score.score ) - return {"human_approved": approved} + decision = interrupt(payload) + + if not isinstance(decision, dict): + logger.warning("hitl: malformed resume value %r — defaulting to reject", decision) + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } diff --git a/compass/pipeline/nodes/intake_filter.py b/compass/pipeline/nodes/intake_filter.py index 718ae0d..dea8871 100644 --- a/compass/pipeline/nodes/intake_filter.py +++ b/compass/pipeline/nodes/intake_filter.py @@ -15,11 +15,85 @@ from __future__ import annotations import logging +import re from datetime import datetime from typing import TYPE_CHECKING import compass.config as cfg from compass.pipeline.role_family import keyword_classify, llm_classify, upgrade_family +from compass.vault.reader import load_reject_rules + +# Body-level signal that the JD actually describes agentic work in production — +# mirrors the "AND" clause in _profile/target-roles.md::JD-master-boolean. +# +# Tiered to avoid false positives: +# +# STRONG signals are unambiguous — they only appear in JDs that actually +# describe agent/LLM-platform work. At least ONE strong signal is required +# for the gate to pass on an agent-oriented title. +# +# WEAK signals (just "agent" / "agents" / "agentic" on their own) are noisy: +# every corporate JD has phrases like "be a change agent", "user-agent header", +# "outage management agent". Weak hits are tracked for the auto-tag (so the +# dashboard can surface JDs with rich agentic language) but DON'T satisfy the +# gate on their own. +_AGENT_STRONG_TERMS = [ + r"\bagent[-\s]?orchestration\b", + r"\bmulti[-\s]?agent\b", + r"\bagentic\s+ai\b", + r"\bai\s+agent[s]?\b", + r"\bllm\s+agent[s]?\b", + r"\bmcp\b", + r"\bmodel context protocol\b", + r"\blanggraph\b", + r"\blangchain\b", + r"\bpydantic\s+ai\b", + r"\bagents\s+sdk\b", + r"\blangfuse\b", + r"\blangsmith\b", + r"\bbraintrust\b", + r"\btool[-\s]?calling\b", + r"\bfunction[-\s]?calling\b", + r"\bdurable\s+execution\b", + r"\bsub[-\s]?agents?\b", + r"\bagent[-\s]?reliability\b", + r"\bagent\s+evaluation\b", + r"\bagent\s+platform\b", + r"\bagent\s+framework\b", + r"\bagent\s+development\b", + r"\bagent\s+system\b", + r"\bautonomous\s+agent\b", +] +_AGENT_WEAK_TERMS = [ + r"\bagentic\b", + r"\bagents?\b", +] +_AGENT_STRONG_RE = re.compile("|".join(_AGENT_STRONG_TERMS), re.IGNORECASE) +_AGENT_WEAK_RE = re.compile("|".join(_AGENT_WEAK_TERMS), re.IGNORECASE) + + +def _agent_signal_count(body: str) -> int: + """Combined unique-hit count across strong + weak terms — preserved as the + `agent_signal_count` exposed in state for downstream tagging. The gate + uses `_has_strong_agent_signal` instead for the actual drop decision.""" + if not body: + return 0 + hits = {m.group(0).lower() for m in _AGENT_STRONG_RE.finditer(body)} + hits |= {m.group(0).lower() for m in _AGENT_WEAK_RE.finditer(body)} + return len(hits) + + +def _has_strong_agent_signal(body: str) -> bool: + """True iff the body contains at least one STRONG agent term. This is the + gate criterion — weak terms ("agent" / "agentic" alone) don't satisfy. + + Without this, JDs containing phrases like "be a change agent in our team" + or "set the User-Agent header" survive the gate on weak signal alone. + """ + if not body: + return False + return bool(_AGENT_STRONG_RE.search(body)) + if TYPE_CHECKING: from compass.pipeline.state import CompassState @@ -48,10 +122,37 @@ async def intake_filter_node(state: CompassState) -> dict: body = job.description or "" + # Hard rejects from preferences.md — runs BEFORE LLM-stage classification so + # senior/staff/principal/lead titles and "5+ years" / "PhD required" JDs are + # dropped at zero LLM cost. On a 41-board scrape this is ~40-60% of volume. + rules = load_reject_rules() + title_lc = (job.title or "").lower() + body_lc = body.lower() + for needle in rules["title"]: + if needle and needle in title_lc: + _log_filtered(job.company, job.title, f"title rejects: {needle!r}") + logger.info( + "intake_filter: dropped %s — %s (title rule: %r)", + job.company, + job.title, + needle, + ) + return {"in_scope": False, "role_family": "out-of-scope", "agent_signal_count": 0} + for needle in rules["jd"]: + if needle and needle in body_lc: + _log_filtered(job.company, job.title, f"jd rejects: {needle!r}") + logger.info( + "intake_filter: dropped %s — %s (jd rule: %r)", + job.company, + job.title, + needle, + ) + return {"in_scope": False, "role_family": "out-of-scope", "agent_signal_count": 0} + decided, family = keyword_classify(job.title) if decided is True: upgraded = upgrade_family(family, body) - return {"in_scope": True, "role_family": upgraded} + return _gated_by_agent_signal(job.company, job.title, body, upgraded) if decided is False: _log_filtered(job.company, job.title, f"title keyword → {family}") logger.info("intake_filter: dropped %s — %s (keyword)", job.company, job.title) @@ -74,4 +175,48 @@ async def intake_filter_node(state: CompassState) -> dict: return {"in_scope": False, "role_family": result.role_family} upgraded = upgrade_family(result.role_family, body) - return {"in_scope": True, "role_family": upgraded} + return _gated_by_agent_signal(job.company, job.title, body, upgraded) + + +def _gated_by_agent_signal(company: str, title: str, body: str, role_family: str) -> dict: + """Final gate after title-based role_family classification. + + User's target market (from _profile/target-roles.md) requires JDs that + talk about agents in production — not just titles that say "AI Engineer." + A JD body with ZERO agent-related terms is almost always a generic + ML/RAG/data-science role mis-titled, which the user doesn't want. + + Applied to in-scope agent-engineer / applied-ai / infra-llm classifications + only; SWE family roles (swe-backend, swe-fullstack, swe-frontend) pass + through unfiltered because they may be agent-eng IC roles posted under + generic SWE titles at AI-native companies (Sierra "Software Engineer, + Product" is real). + + Sets `agent_signal_count` in state so downstream nodes (vault_write, + score) can tag/weight by signal strength. + """ + signal_count = _agent_signal_count(body) + has_strong = _has_strong_agent_signal(body) + agent_oriented = role_family in {"agent-engineer", "applied-ai", "infra-llm"} + # Gate: an agent-oriented title with NO strong signal is dropped. A title + # like "AI Engineer" whose body only says "be a change agent" counts as + # zero strong signal even though signal_count>=1. This is intentional — + # "agent" alone is too noisy in corporate JD prose. + if agent_oriented and not has_strong: + _log_filtered(company, title, "no-strong-agent-signal in body") + logger.info( + "intake_filter: dropped %s — %s (agent-oriented title but body has no strong agent signal; signal_count=%d)", + company, + title, + signal_count, + ) + return { + "in_scope": False, + "role_family": "out-of-scope", + "agent_signal_count": signal_count, + } + return { + "in_scope": True, + "role_family": role_family, + "agent_signal_count": signal_count, + } diff --git a/compass/pipeline/nodes/score.py b/compass/pipeline/nodes/score.py index 817c383..7647eae 100644 --- a/compass/pipeline/nodes/score.py +++ b/compass/pipeline/nodes/score.py @@ -1,8 +1,9 @@ """ score_node — score a job against the candidate profile. -Reads resume.md + skill-inventory.md from the vault and passes them as context -to the LLM. Returns a JobScore (0.0–5.0) with matched/missing/tailoring breakdown. +Profile context is the full resume plus top-k chunks retrieved from the +skill-inventory Chroma index, queried with the JD's skills + summary. +Returns a JobScore (0.0–5.0) with matched/missing/tailoring breakdown. Model: SCORE_MODEL (default google/gemini-2.5-flash). """ @@ -11,9 +12,12 @@ import logging +from compass.config import SCORE_THRESHOLD from compass.llm import make_agent -from compass.pipeline.state import CompassState, JobRequirements, JobScore -from compass.vault.reader import read_resume, read_skill_inventory +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob +from compass.rag.retriever import retrieve as rag_retrieve +from compass.vault.reader import read_resume +from compass.vault.target_companies import get_company_meta logger = logging.getLogger(__name__) @@ -36,6 +40,22 @@ - missing_skills: skills from the JD's required+nice-to-have list that the candidate lacks (level < 2) - tailoring_notes: ONE sentence suggesting how to frame the application (skip if score < 3.0) +SECONDARY SIGNAL — company targeting context: +The job's company carries a tier classification from the candidate's strategic +target list (apply-now / opportunistic / backend-prep / stretch / skip), an +interview-loop difficulty (hackerrank / case / lc-easy / lc-medium / lc-hard / +takehome), and the candidate's notes about why the company matters. When that +context is provided in the JOB BLOCK below, use it as ONE input to the score — +not the dominant one. Skill match is still primary. But: +- An `apply-now` company with an `lc-easy` or `hackerrank` or `case` loop is a + realistic landing — don't under-score those just because the candidate has + gaps; tailoring + interview prep can close them. +- An `opportunistic` or `backend-prep` company with an `lc-hard` loop is a + stretch — score those honestly even if skill match looks strong, since the + loop will be the actual blocker. +- The `notes` field tells you why the candidate cares — use it for the + tailoring_notes if the score is >= 3. + HARD CONSTRAINTS on matched_skills and missing_skills: 1. Every skill in matched_skills MUST appear in the JD's required or nice_to_have list. Do NOT list skills from the candidate's profile that the JD did not ask for. 2. Every skill in missing_skills MUST appear in the JD's required or nice_to_have list. Do NOT list every skill in the canonical taxonomy that the candidate lacks. @@ -50,9 +70,30 @@ def _build_agent(): return make_agent("score", output_type=JobScore, system_prompt=_SYSTEM_PROMPT) -def _format_prompt(req: JobRequirements, profile_text: str) -> str: +def _company_context_block(job: RawJob | None) -> str: + """Render the company's YAML targeting metadata as a prompt block. Empty + string when the company isn't in the user's target list — the LLM scores + purely on skill match in that case (likely a JD that landed via fallback + config, not the YAML).""" + if job is None: + return "" + meta = get_company_meta(job.company) + if not meta: + return "" + return ( + "# COMPANY TARGETING CONTEXT\n" + f"company: {job.company}\n" + f"tier: {meta.get('tier') or 'unknown'}\n" + f"interview_difficulty: {meta.get('interview_difficulty') or 'unknown'}\n" + f"section: {meta.get('section') or 'unknown'}\n" + f"notes: {meta.get('notes') or '(none)'}\n\n" + ) + + +def _format_prompt(req: JobRequirements, profile_text: str, job: RawJob | None) -> str: return ( f"# CANDIDATE PROFILE\n{profile_text}\n\n" + f"{_company_context_block(job)}" f"# JOB REQUIREMENTS\n" f"required: {', '.join(req.required_skills) or '(none)'}\n" f"nice-to-have: {', '.join(req.nice_to_have_skills) or '(none)'}\n" @@ -63,10 +104,10 @@ def _format_prompt(req: JobRequirements, profile_text: str) -> str: ) -async def _score(req: JobRequirements, profile_text: str) -> JobScore: +async def _score(req: JobRequirements, profile_text: str, job: RawJob | None = None) -> JobScore: # Tests patch this function; the underlying pydantic-ai Agent is harder to stub. agent = _build_agent() - result = await agent.run(_format_prompt(req, profile_text)) + result = await agent.run(_format_prompt(req, profile_text, job)) return result.output @@ -79,8 +120,10 @@ def _reasoning_complete(text: str) -> bool: return len(t) >= 20 and t[-1] in '.!?"' -async def _score_with_retry(req: JobRequirements, profile_text: str) -> JobScore: - result = await _score(req, profile_text) +async def _score_with_retry( + req: JobRequirements, profile_text: str, job: RawJob | None = None +) -> JobScore: + result = await _score(req, profile_text, job) if _reasoning_complete(result.reasoning): return result logger.warning( @@ -88,14 +131,30 @@ async def _score_with_retry(req: JobRequirements, profile_text: str) -> JobScore len(result.reasoning or ""), (result.reasoning or "")[-40:], ) - retry = await _score(req, profile_text) + retry = await _score(req, profile_text, job) if not _reasoning_complete(retry.reasoning): logger.warning("score_node: retry still produced incomplete reasoning — accepting anyway") return retry -def _profile_text() -> str: - return f"## RESUME\n{read_resume()}\n\n## SKILL INVENTORY\n{read_skill_inventory()}" +async def _profile_text(req: JobRequirements) -> str: + """Build candidate-profile context for the score prompt. + + Resume stays inline; the prior full skill-inventory inject is now top-k + chunks retrieved against the JD's skills + summary. + """ + query_parts = [*req.required_skills, *req.nice_to_have_skills] + if req.summary: + query_parts.append(req.summary) + query = " ".join(query_parts).strip() + + chunks = await rag_retrieve(query, k=8) if query else [] + + profile = f"## RESUME\n{read_resume()}" + if chunks: + ranked = "\n\n".join(c.document for c in chunks) + profile += f"\n\n## RELEVANT SKILLS (top-{len(chunks)} by similarity)\n{ranked}" + return profile async def score_node(state: CompassState) -> dict: @@ -107,15 +166,20 @@ async def score_node(state: CompassState) -> dict: } try: - result = await _score_with_retry(req, _profile_text()) + profile = await _profile_text(req) + result = await _score_with_retry(req, profile, state.get("current_job")) except Exception as e: logger.exception("score_node: LLM call failed") return { "score_result": None, "errors": [*state.get("errors", []), f"score_node: {type(e).__name__}: {e}"], + "score_threshold": SCORE_THRESHOLD, } - return {"score_result": _constrain_to_jd_skills(result, req)} + return { + "score_result": _constrain_to_jd_skills(result, req), + "score_threshold": SCORE_THRESHOLD, + } def _constrain_to_jd_skills(score: JobScore, req: JobRequirements) -> JobScore: diff --git a/compass/pipeline/nodes/tailor.py b/compass/pipeline/nodes/tailor.py index d4bc589..8e57dd1 100644 --- a/compass/pipeline/nodes/tailor.py +++ b/compass/pipeline/nodes/tailor.py @@ -71,6 +71,26 @@ async def tailor_node(state: CompassState) -> dict: if score is None or job is None: return {} + # Cost gate — tailor uses Sonnet (~$0.05/call). Don't burn it on jobs that + # scored below the threshold even if the human approved (mis-click in the + # HiTL UI, score drift between pause and resume, etc.). The intended gate + # is score-based; without this, an accidental approval on a 1.0-scored + # job costs as much as a real 4.5-scored one. + from compass.config import SCORE_THRESHOLD + + threshold = state.get("score_threshold") + if threshold is None: + threshold = SCORE_THRESHOLD + if score.score < threshold: + logger.info( + "tailor_node: skipping low-score job %s (score=%.2f < threshold=%.2f) " + "despite human_approved=True", + job.url, + score.score, + threshold, + ) + return {} + profile = f"{read_resume()}\n\n{read_profile_section('role-clarifications')}" try: diff --git a/compass/pipeline/nodes/vault_write.py b/compass/pipeline/nodes/vault_write.py index 71d0b48..c9339bd 100644 --- a/compass/pipeline/nodes/vault_write.py +++ b/compass/pipeline/nodes/vault_write.py @@ -21,7 +21,7 @@ from __future__ import annotations import logging -from datetime import date +from datetime import date, datetime from typing import TYPE_CHECKING from compass.vault.schemas import CompanyNote, JobNote @@ -33,6 +33,76 @@ logger = logging.getLogger(__name__) +def _build_auto_tags( + *, + tier: str, + match_score: float, + role_family: str, + hitl_decision: str | None, + agent_signal_count: int | None = None, +) -> list[str]: + """Generate Obsidian tag-pane-filterable tags from JobNote fields. + + Tag taxonomy is intentionally shallow (one slash) so Obsidian's nested-tag + view groups them. Composed filters like `#fit/strong AND #role/agent-engineer` + happen naturally in the tag pane and in Dataview `contains(tags, "...")` calls. + """ + tags: list[str] = [] + if tier: + tags.append(f"#tier/{tier}") + if match_score >= 4.0: + tags.append("#fit/strong") + elif match_score >= 3.0: + tags.append("#fit/decent") + elif match_score >= 2.0: + tags.append("#fit/stretch") + else: + tags.append("#fit/weak") + if role_family: + tags.append(f"#role/{role_family}") + if hitl_decision: + tags.append(f"#decision/{hitl_decision}") + # Body-level agentic signal (computed by intake_filter from JD body keyword + # scan). Distinguishes "AI Engineer role at a real agent-eng team" from + # "AI Engineer role that means RAG-and-prompts." + if agent_signal_count is not None: + if agent_signal_count >= 3: + tags.append("#signal/agent-strong") + elif agent_signal_count >= 1: + tags.append("#signal/agent-mention") + return tags + + +def _derive_hitl_decision(state: CompassState) -> tuple[str | None, datetime | None]: + """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" + from compass.config import SCORE_THRESHOLD + from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX + + approved = state.get("human_approved") + if approved is None: + # hitl never reached (e.g. extract/score errored). Leave fields null. + return (None, None) + + feedback = (state.get("human_feedback") or "").lower() + score = state.get("score_result") + score_value = score.score if score is not None else 0.0 + + # Prefer state-captured threshold (sticky); fall back to config for old checkpoints. + threshold = state.get("score_threshold") + if threshold is None: + threshold = SCORE_THRESHOLD + + if approved is True: + decision = "approved" + elif feedback.startswith(HITL_TIMEOUT_FEEDBACK_PREFIX): + decision = "timed_out" + elif score_value < threshold: + decision = "auto_rejected" + else: + decision = "rejected" + return (decision, datetime.now()) + + async def vault_write_node(state: CompassState) -> dict: job = state.get("current_job") score = state.get("score_result") @@ -57,12 +127,16 @@ async def vault_write_node(state: CompassState) -> dict: # still gates tailor (Sonnet cost control) inside hitl_node. Removing it here # lets stretch-role gaps drive the master gap plan — see spec § 2. - from compass.vault.target_companies import get_tier + from compass.vault.target_companies import ( + get_interview_difficulty, + get_tier, + ) # JobNote.tier is a per-posting snapshot — always use the currently-resolved # tier so a later edit to target-companies.md doesn't retroactively change # what the snapshot said when the job was first seen. company_tier = get_tier(job.company) + interview_difficulty = get_interview_difficulty(job.company) # CompanyNote.tier — read-before-write to preserve human edits in Obsidian. # If an existing CompanyNote has a non-default tier, pass "unknown" on @@ -86,6 +160,15 @@ async def vault_write_node(state: CompassState) -> dict: company_tier_for_write = "unknown" break + hitl_decision, hitl_at = _derive_hitl_decision(state) + role_family = state.get("role_family") or "" + auto_tags = _build_auto_tags( + tier=company_tier, + match_score=score.score, + role_family=role_family, + hitl_decision=hitl_decision, + agent_signal_count=state.get("agent_signal_count"), + ) note = JobNote( company=job.company, title=job.title, @@ -100,14 +183,18 @@ async def vault_write_node(state: CompassState) -> dict: remote=("remote" if job.remote else None), seniority=req.seniority, years_required=req.years_experience, - role_family=state.get("role_family") or "", + role_family=role_family, tier=company_tier, + interview_difficulty=interview_difficulty, # type: ignore[arg-type] + tags=auto_tags, skills_required=req.required_skills, skills_nice_to_have=req.nice_to_have_skills, skills_matched=score.matched_skills, skills_missing=score.missing_skills, jd_summary=req.summary, tailored_paragraph=state.get("tailored_paragraph"), + hitl_decision=hitl_decision, + hitl_at=hitl_at, ) write_job_note(note, full_description=job.description) diff --git a/compass/pipeline/role_family.py b/compass/pipeline/role_family.py index 7fac4a3..17d3dd5 100644 --- a/compass/pipeline/role_family.py +++ b/compass/pipeline/role_family.py @@ -28,21 +28,42 @@ IN_TITLE_KEYWORDS: dict[str, list[str]] = { "agent-engineer": [ "agent engineer", + "ai agent engineer", "agentic engineer", + "agentic ai engineer", "agent platform", "agent orchestration", "agent reliability", + "software engineer, agents", + "software engineer - agents", + "software engineer - agentic", + "software engineer, agentic", + "ai native engineer", + # MTS = Member of Technical Staff. Frontier-startup flat-hierarchy + # signal, not literal seniority. Sierra / Decagon / Cognition / Cursor + # / Mistral / xAI all use it for agent-eng ICs. The MTS title alone + # routes to agent-engineer; if the JD body is research-flavored the + # body-signal upgrader can move it elsewhere. + "member of technical staff", + "mts - agents", + "mts, agents", ], "applied-ai": [ "applied ai", "applied ml", "ai engineer", + "ai/ml engineer", "ml engineer", "machine learning engineer", + "genai engineer", + # AI Enablement = Cognition (Devin/Windsurf), Cursor — FDE-lite path + # explicitly named in target-roles.md as in-range. + "ai enablement engineer", ], "infra-llm": [ "llm platform", "ai infrastructure", + "ai platform engineer", "inference engineer", "eval engineer", "evaluation engineer", @@ -82,17 +103,36 @@ # pre-sales / solutions "presales", "pre-sales", + "solutions engineer", + "solution engineer", + # NOTE: "solutions architect" intentionally NOT in this list — that title + # straddles pre-sales and hands-on infra depending on company; the LLM + # body-check stage decides per-JD. # CS "customer success", "customer experience", "customer support", "technical csm", - # PM + # PM / product ops "product manager", "product management", "group pm", "agent pm", - # design (note: "designer", "motion graphics", etc. — substring is safe because they're distinctive) + "product operations", + "operations specialist", + "program manager", + # management track (Akash isn't pursuing management — per role-clarifications) + "engineering manager", + "engineering lead", + "director of engineering", + "head of engineering", + "vp of engineering", + "vp engineering", + # security (separate engineering discipline, not agentic-AI) + "security engineer", + "application security", + "infrastructure security", + # design (note: "designer" is distinctive enough for substring match) "designer", "motion graphics", "web designer", @@ -177,12 +217,18 @@ "huggingface", ] +# `upgrade_family` only promotes families in this set to `agent-engineer` / +# `applied-ai` based on JD body signals. Mobile and frontend are intentionally +# excluded — a React Native job at an LLM-native startup whose JD mentions +# "LangGraph" once is still a mobile job, not an agent-engineering job. The +# user isn't a mobile or frontend specialist; promoting these clutters the +# vault. Backend / fullstack / founding / other-eng are kept eligible because +# "Software Engineer, Product" at Sierra is a real agent-eng role posted +# under a generic SWE title. GENERIC_FAMILIES = { "swe-backend", - "swe-frontend", "swe-fullstack", "swe-founding", - "swe-mobile", "other-eng", } diff --git a/compass/pipeline/state.py b/compass/pipeline/state.py index 5053d65..f589580 100644 --- a/compass/pipeline/state.py +++ b/compass/pipeline/state.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field -Source = Literal["greenhouse", "lever", "ashby", "jobspy", "smoke", "manual"] +Source = Literal["greenhouse", "lever", "ashby", "workday", "jobspy", "smoke", "manual"] Seniority = Literal["junior", "mid", "senior", "staff", "unknown"] RemotePolicy = Literal["remote", "hybrid", "onsite", "unknown"] @@ -62,6 +62,11 @@ class CompassState(TypedDict): in_scope: bool | None role_family: str | None + # Count of distinct agent-related terms hit in the JD body by intake_filter. + # 0 means the body has no agentic signal (and the role was likely dropped + # for that reason if the title was agent-oriented). 1+ means real signal. + # Used by vault_write_node to emit a `#signal/agent-strong|mention` tag. + agent_signal_count: int | None human_approved: bool | None human_feedback: str | None @@ -72,3 +77,7 @@ class CompassState(TypedDict): jobs_written: int errors: list[str] + + thread_id: str | None + + score_threshold: float | None diff --git a/compass/rag/indexer.py b/compass/rag/indexer.py new file mode 100644 index 0000000..e899a3c --- /dev/null +++ b/compass/rag/indexer.py @@ -0,0 +1,145 @@ +"""Chroma index of _profile/skill-inventory.md. + +One chunk per `## SkillName` section. Collection metric is cosine — the +retriever's similarity score formula depends on this. +""" + +from __future__ import annotations + +import asyncio +import logging +import re + +logger = logging.getLogger(__name__) + +_COLLECTION_NAME = "skill_inventory" +_SECTION_RE = re.compile(r"^## +(.+)$", re.M) +# Auto-generated assessor output; not a skill — exclude from the index. +_EXCLUDED_HEADINGS = {"Assessor-current grades"} + + +def _client(): + from chromadb import PersistentClient + + import compass.config as cfg + + cfg.CHROMA_PATH.mkdir(parents=True, exist_ok=True) + return PersistentClient(path=str(cfg.CHROMA_PATH)) + + +def _collection(client): + """Return the skill_inventory collection pinned to cosine distance. + + `get_or_create_collection` does NOT update an existing collection's + metadata — so a pre-existing L2 collection would silently keep the wrong + metric and break the retriever's score formula. Drop and recreate if so. + """ + existing = {c.name for c in client.list_collections()} + if _COLLECTION_NAME in existing: + col = client.get_collection(_COLLECTION_NAME) + if (col.metadata or {}).get("hnsw:space") == "cosine": + return col + client.delete_collection(_COLLECTION_NAME) + return client.create_collection( + name=_COLLECTION_NAME, + metadata={"hnsw:space": "cosine"}, + ) + + +def _kebab(name: str) -> str: + # Drop parenthetical asides and em-dash commentary — they're not identity. + # `Fine-Tuning (awareness only — ...)` -> `fine-tuning` + # `MCP — your strongest cluster` -> `mcp` + primary = re.sub(r"\s*(?:\(|[—–]\s).*$", "", name).strip() + return re.sub(r"[^a-z0-9]+", "-", primary.lower()).strip("-") or "untitled" + + +def _parse_inventory(text: str) -> list[tuple[str, str]]: + """Return [(skill_name, section_text), ...] — one entry per ## heading. + + Skips headings in _EXCLUDED_HEADINGS (auto-generated metadata that isn't + a real skill and would otherwise pollute retrieval). + """ + matches = list(_SECTION_RE.finditer(text)) + sections = [] + for i, m in enumerate(matches): + name = m.group(1).strip() + if any(name.startswith(prefix) for prefix in _EXCLUDED_HEADINGS): + continue + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + sections.append((name, text[m.start() : end].strip())) + return sections + + +def _embed(documents: list[str]) -> list[list[float]]: + from sentence_transformers import SentenceTransformer + + import compass.config as cfg + + model = SentenceTransformer(cfg.EMBEDDING_MODEL) + arr = model.encode(documents, convert_to_numpy=True, show_progress_bar=False) + return [vec.tolist() for vec in arr] + + +async def build_index(force_rebuild: bool = False) -> int: + """Embed each `## Section` of skill-inventory.md and upsert into Chroma. + + Idempotent — repeat calls upsert by stable id. Pass `force_rebuild=True` + to drop the collection first (use when the inventory shrinks; otherwise + deleted skills' stale chunks linger). + """ + import compass.config as cfg + + if not cfg.SKILL_INVENTORY_PATH.exists(): + logger.warning("rag: skill-inventory.md not found at %s", cfg.SKILL_INVENTORY_PATH) + return 0 + + sections = _parse_inventory(cfg.SKILL_INVENTORY_PATH.read_text(encoding="utf-8")) + if not sections: + logger.info("rag: 0 ## sections in skill-inventory.md; nothing to index") + return 0 + + client = _client() + if force_rebuild and _COLLECTION_NAME in {c.name for c in client.list_collections()}: + client.delete_collection(_COLLECTION_NAME) + collection = _collection(client) + + documents = [body for _, body in sections] + new_ids = [_kebab(name) for name, _ in sections] + embeddings = await asyncio.to_thread(_embed, documents) + + collection.upsert( + ids=new_ids, + documents=documents, + embeddings=embeddings, + metadatas=[{"skill": name, "source": "skill-inventory.md"} for name, _ in sections], + ) + + # ORPHAN CLEANUP: upsert only adds/updates — it never deletes. If a skill + # was renamed ("LangGraph" → "LangGraph (async)") or removed from + # skill-inventory.md, the old chunk's id stays in the collection forever + # and pollutes future RAG retrievals. After every build, drop any id in + # the collection that's NOT in the current section list. + existing = collection.get(include=[]).get("ids", []) + orphan_ids = [eid for eid in existing if eid not in set(new_ids)] + if orphan_ids: + collection.delete(ids=orphan_ids) + logger.info("rag: pruned %d orphaned chunks (skills removed/renamed)", len(orphan_ids)) + + logger.info("rag: indexed %d sections at %s", len(sections), cfg.CHROMA_PATH) + return len(sections) + + +def _main() -> None: + import argparse + + import compass.config as cfg + + parser = argparse.ArgumentParser(description="Rebuild Chroma index of skill-inventory.md") + parser.add_argument("--force", action="store_true", help="Drop+rebuild the collection") + n = asyncio.run(build_index(force_rebuild=parser.parse_args().force)) + print(f"Indexed {n} sections from {cfg.SKILL_INVENTORY_PATH}") + + +if __name__ == "__main__": + _main() diff --git a/compass/rag/retriever.py b/compass/rag/retriever.py new file mode 100644 index 0000000..8b51a50 --- /dev/null +++ b/compass/rag/retriever.py @@ -0,0 +1,41 @@ +"""Top-k retrieval over the skill-inventory Chroma index. + +Lazy-init on first call so scoring works before any explicit indexer run. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from compass.rag import indexer + + +@dataclass +class RetrievedChunk: + skill: str + document: str + score: float # cosine similarity in [0, 1]; 1.0 = identical + + +async def retrieve(query: str, k: int = 8) -> list[RetrievedChunk]: + if not (query or "").strip(): + return [] + + collection = indexer._collection(indexer._client()) + if collection.count() == 0 and await indexer.build_index() == 0: + return [] + + [query_emb] = await asyncio.to_thread(indexer._embed, [query]) + res = collection.query( + query_embeddings=[query_emb], + n_results=min(k, collection.count()), + ) + + # Collection is pinned to cosine (see indexer._collection); distance ∈ [0, 2]. + return [ + RetrievedChunk(skill=m["skill"], document=d, score=max(0.0, 1.0 - dist / 2.0)) + for m, d, dist in zip( + res["metadatas"][0], res["documents"][0], res["distances"][0], strict=True + ) + ] diff --git a/compass/scrapers/ashby.py b/compass/scrapers/ashby.py index 4a03c34..24e7eed 100644 --- a/compass/scrapers/ashby.py +++ b/compass/scrapers/ashby.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import html import logging import re from datetime import date, datetime @@ -16,6 +17,18 @@ from compass.pipeline.state import RawJob +_SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE) +_TAG_RE = re.compile(r"<[^>]+>") + + +def _strip_html(raw: str) -> str: + """HTML→text fallback for Ashby boards that don't populate descriptionPlain.""" + text = _SCRIPT_STYLE_RE.sub(" ", raw) + text = _TAG_RE.sub(" ", text) + text = html.unescape(text) + return re.sub(r"\s+", " ", text).strip() + + logger = logging.getLogger(__name__) ASHBY_BASE = "https://api.ashbyhq.com/posting-api/job-board" @@ -65,9 +78,14 @@ def _parse_date(value: str | None) -> date | None: def _to_rawjob(slug: str, raw: dict) -> RawJob | None: try: description = (raw.get("descriptionPlain") or "").strip() + if not description: + # Some Ashby boards return descriptionPlain=null but populate the + # HTML-rendered `description` field. Mirror the Lever fallback + # rather than silently dropping. Pre-fix, these jobs were lost. + description = _strip_html((raw.get("description") or "").strip()) if not description: logger.warning( - "ashby %s: empty description for %r — dropping (posting may use external HTML)", + "ashby %s: empty description for %r — dropping", slug, raw.get("title", "?"), ) diff --git a/compass/scrapers/workday.py b/compass/scrapers/workday.py new file mode 100644 index 0000000..e109b86 --- /dev/null +++ b/compass/scrapers/workday.py @@ -0,0 +1,270 @@ +""" +Workday public JSON-endpoint scraper. + +Endpoint: POST https://{tenant}.{subdomain}.myworkdayjobs.com/wday/cxs/{tenant}/{site}/jobs + +Workday's careers UI is a JavaScript SPA, but every tenant exposes an +unauthenticated JSON endpoint behind it that serves the same paginated list. +No auth, no cookies, no anti-bot — the SPA hits this same URL. + +The slug format the rest of Compass uses is `{subdomain}/{tenant}/{site}` — +e.g. `wd5/citi/2`, `wd1/wf/WellsFargoJobs`. The YAML's `ats.slug` for a +Workday-provider entry must be in this `wdN/tenant/site` form. The slug is +parsed and joined back into the full URL at request time. + +Tenant/site discovery is per-company manual work — public careers pages all +embed the URL in their initial JS bundle. There's no programmatic way to +enumerate them. The verified set as of 2026-05-19: + wf/WellsFargoJobs/wd1 Wells Fargo + citi/2/wd5 Citi + adobe/external_experienced/wd5 Adobe + ms/External/wd5 Morgan Stanley + blackrock/BlackRock_Professional/wd1 BlackRock +""" + +from __future__ import annotations + +import asyncio +import html +import logging +import re +from datetime import date + +import httpx + +from compass.pipeline.state import RawJob +from compass.scrapers._remote_parser import infer_remote_policy + +logger = logging.getLogger(__name__) + +_REQUEST_TIMEOUT = 25.0 +_USER_AGENT = "compass-job-scraper/0.1" +# Workday paginates at 20 by default; ask for 50 per request to reduce round-trips. +_PAGE_SIZE = 50 +# Cap iterations so a runaway pagination loop (or a board that returns +# `total` lower than the actual count and never converges) can't burn +# unbounded requests. 10 pages × 50 = 500 jobs per board, more than enough. +_MAX_PAGES = 10 + +_SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE) +_TAG_RE = re.compile(r"<[^>]+>") + + +def _strip_html(raw: str) -> str: + """Same approach as greenhouse/lever: strip script/style blocks then tags.""" + text = _SCRIPT_STYLE_RE.sub(" ", raw) + text = _TAG_RE.sub(" ", text) + text = html.unescape(text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def _parse_slug(slug: str) -> tuple[str, str, str] | None: + """Parse `subdomain/tenant/site` (e.g. `wd5/citi/2`) into a 3-tuple. + + Returns None for malformed slugs so the orchestrator can log + skip + without raising. The subdomain (wd1/wd3/wd5/wd12/...) is per-tenant — + don't try to auto-detect; the YAML carries it. + """ + parts = [p for p in slug.split("/") if p] + if len(parts) != 3: + return None + subdomain, tenant, site = parts + if not subdomain.startswith("wd"): + return None + return subdomain, tenant, site + + +def _parse_posted_on(value: str | None) -> date | None: + """Workday's `postedOn` is a free-text relative phrase like + 'Posted Yesterday', 'Posted 5 Days Ago', 'Posted 30+ Days Ago'. The JSON + endpoint does NOT include an ISO timestamp — `postedOn` is the only signal + we have. Convert to an approximate date so the recency sort in graph.py + can still rank. + """ + if not value: + return None + today = date.today() + v = value.lower() + if "today" in v: + return today + if "yesterday" in v: + from datetime import timedelta + + return today - timedelta(days=1) + m = re.search(r"(\d+)\+?\s*day", v) + if m: + from datetime import timedelta + + return today - timedelta(days=int(m.group(1))) + m = re.search(r"(\d+)\+?\s*month", v) + if m: + from datetime import timedelta + + return today - timedelta(days=int(m.group(1)) * 30) + return None + + +async def _fetch_job_detail( + client: httpx.AsyncClient, base: str, ext_path: str +) -> tuple[str | None, str | None]: + """Fetch the full JD body + apply URL for one job. Returns (body, apply_url). + + Workday's list endpoint returns short summaries; the actual JD lives at + /jobs/{external_path}. That endpoint is also unauthenticated JSON. + """ + url = f"{base}{ext_path}" + try: + r = await client.get(url) + if r.status_code != 200: + return None, None + data = r.json() + body_html = (data.get("jobPostingInfo") or {}).get("jobDescription") or "" + apply_url = (data.get("jobPostingInfo") or {}).get("externalUrl") + return _strip_html(body_html), apply_url + except (httpx.HTTPError, ValueError) as e: + logger.warning("workday detail %s: %s", url, e) + return None, None + + +def _to_rawjob( + company_label: str, + raw: dict, + body: str | None, + apply_url: str | None, + base_url: str = "", +) -> RawJob | None: + """Build a RawJob from a Workday list-page entry + (optional) detail body. + + `base_url` is the Workday tenant origin (e.g. `https://citi.wd5.myworkdayjobs.com`) + used to absolutize relative `externalPath` values when the detail endpoint + didn't return an `externalUrl`. Without this, the `RawJob.url` would be a + bare path like `/job/abc` that `normalize_url` then mangles into the + invalid string `https:///job/abc`, breaking dedup and apply-link clicks. + """ + try: + title = raw.get("title") + if not title: + return None + if not body: + # Without a body the score node will hallucinate from title alone. + # Workday rate-limits detail fetches more aggressively than list + # fetches, so we sometimes get list-only entries. Drop them. + return None + # Resolve the canonical URL: prefer the detail endpoint's `externalUrl`; + # fall back to absolutizing `externalPath` against the tenant origin. + url = apply_url + if not url: + ext_path = raw.get("externalPath") or "" + if ext_path and base_url and ext_path.startswith("/"): + url = f"{base_url}{ext_path}" + elif ext_path.startswith("http"): + url = ext_path + if not url: + # No usable URL at all — skip rather than write a broken vault row. + logger.warning( + "workday: no usable URL for %r at %s (skipping)", + title, + company_label, + ) + return None + location_str = raw.get("locationsText") or None + return RawJob( + company=company_label, + title=title, + url=url, + source="workday", # type: ignore[arg-type] + location=location_str, + remote=infer_remote_policy(location_str), + salary_min=None, + salary_max=None, + description=body, + date_posted=_parse_posted_on(raw.get("postedOn")), + ) + except (KeyError, TypeError) as e: + logger.warning("workday: malformed entry skipped: %s", e) + return None + + +async def scrape_workday(slug: str, company_label: str | None = None) -> list[RawJob]: + """Scrape all open jobs from a Workday tenant/site. + + `slug` is `subdomain/tenant/site` (e.g. `wd5/citi/2`). `company_label` + overrides the displayed company name — useful when the tenant token is + cryptic (e.g. `ms` → Morgan Stanley, `wf` → Wells Fargo). + + Returns [] on HTTP error — never raises. + """ + parsed = _parse_slug(slug) + if parsed is None: + logger.warning("workday: malformed slug %r (expected subdomain/tenant/site)", slug) + return [] + subdomain, tenant, site = parsed + base = f"https://{tenant}.{subdomain}.myworkdayjobs.com" + list_url = f"{base}/wday/cxs/{tenant}/{site}/jobs" + label = company_label or tenant + + jobs: list[RawJob] = [] + offset = 0 + async with httpx.AsyncClient( + timeout=_REQUEST_TIMEOUT, headers={"User-Agent": _USER_AGENT} + ) as client: + for _ in range(_MAX_PAGES): + try: + resp = await client.post( + list_url, + json={ + "limit": _PAGE_SIZE, + "offset": offset, + "searchText": "", + "appliedFacets": {}, + }, + ) + if resp.status_code != 200: + logger.warning("workday %s: status=%s", label, resp.status_code) + break + data = resp.json() + except (httpx.HTTPError, ValueError) as e: + logger.warning("workday %s: %s", label, e) + break + + postings = data.get("jobPostings") or [] + if not postings: + break + + # Fetch detail bodies in parallel (modest concurrency to be a + # good citizen — Workday will rate-limit aggressive scrapers). + detail_results = await asyncio.gather( + *[_fetch_job_detail(client, base, p.get("externalPath", "")) for p in postings], + return_exceptions=True, + ) + + for raw, detail in zip(postings, detail_results, strict=True): + if isinstance(detail, BaseException): + continue + body, apply_url = detail + rj = _to_rawjob(label, raw, body, apply_url, base_url=base) + if rj is not None: + jobs.append(rj) + + offset += _PAGE_SIZE + total = data.get("total") + if total is not None and offset >= total: + break + + return jobs + + +async def scrape_workday_many(slugs: list[str]) -> list[RawJob]: + """Scrape multiple Workday tenants concurrently. Each slug is the full + `subdomain/tenant/site` form.""" + if not slugs: + return [] + results = await asyncio.gather(*[scrape_workday(s) for s in slugs], return_exceptions=True) + out: list[RawJob] = [] + for r in results: + if isinstance(r, list): + out.extend(r) + else: + logger.warning("workday_many: unexpected exception: %s", r) + return out diff --git a/compass/vault/learning_bridge.py b/compass/vault/learning_bridge.py index f418996..d9b765a 100644 --- a/compass/vault/learning_bridge.py +++ b/compass/vault/learning_bridge.py @@ -31,13 +31,33 @@ class EvidenceArtifact: def _parse_uri(uri: str) -> tuple[Path, str | None]: + """Parse a `learning-vault://path/to/file.md[#section]` URI. + + SECURITY: jails the resolved path under LEARNING_VAULT_PATH. A URI like + `learning-vault://../../.env` would otherwise escape the vault directory + and leak arbitrary host files (verified pre-fix on 2026-05-19). Path is + resolved + compared against the vault root; mismatch raises ValueError. + + The threat model assumes the URI may come from LLM output (the assessor + cites evidence URIs and prompt-injection in JD bodies could in theory + influence cited URIs). Defense-in-depth: validate every URI even though + most come from the user's own files. + """ if not uri.startswith(URI_PREFIX): raise ValueError(f"not a learning-vault URI: {uri}") rest = uri[len(URI_PREFIX) :] anchor: str | None = None if "#" in rest: rest, anchor = rest.split("#", 1) - return LEARNING_VAULT_PATH / rest, anchor + candidate = (LEARNING_VAULT_PATH / rest).resolve() + root = LEARNING_VAULT_PATH.resolve() + try: + candidate.relative_to(root) + except ValueError as e: + raise ValueError( + f"learning-vault URI {uri!r} resolves outside the vault root {root}" + ) from e + return candidate, anchor def _kind_for(path: Path) -> str: diff --git a/compass/vault/reader.py b/compass/vault/reader.py index 0e1ce9a..6ec6538 100644 --- a/compass/vault/reader.py +++ b/compass/vault/reader.py @@ -4,9 +4,11 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING import frontmatter +import yaml from compass.config import VAULT_PATH @@ -50,3 +52,38 @@ def list_job_notes() -> list[Path]: if not jobs_dir.exists(): return [] return sorted(jobs_dir.glob("*.md")) + + +# ── reject rules (preferences.md) ───────────────────────────────────────────── + +# Match a labelled YAML block: `<label>:\n - item\n - item`. +_REJECT_BLOCK = re.compile( + r"(?m)^(reject_if_title_contains|reject_if_jd_contains):[ \t]*\n((?:[ \t]+-[^\n]*\n?)+)" +) + + +def load_reject_rules() -> dict[str, list[str]]: + """Parse the `reject_if_title_contains` + `reject_if_jd_contains` YAML blocks + from `_profile/preferences.md`. Returns lower-cased strings for cheap + substring matching. + + Returns: + {"title": [...], "jd": [...]} — empty lists when preferences.md is + missing or the block doesn't exist. + """ + text = read_profile_section("preferences") + out: dict[str, list[str]] = {"title": [], "jd": []} + for m in _REJECT_BLOCK.finditer(text): + label = m.group(1) + try: + items = yaml.safe_load(m.group(2)) or [] + except yaml.YAMLError: + continue + if not isinstance(items, list): + continue + rules = [str(s).strip().lower() for s in items if isinstance(s, str | int)] + if label == "reject_if_title_contains": + out["title"] = rules + elif label == "reject_if_jd_contains": + out["jd"] = rules + return out diff --git a/compass/vault/schemas.py b/compass/vault/schemas.py index 09035f8..e5151c5 100644 --- a/compass/vault/schemas.py +++ b/compass/vault/schemas.py @@ -10,7 +10,19 @@ from pydantic import BaseModel, ConfigDict, Field -Tier = Literal["apply-now", "6-month", "stretch", "skip", "unknown"] +# Tier values must stay in sync with `_profile/preferences.md::tier_weights`. +# `6-month` is the legacy name retained for backwards-compat with existing +# JobNotes; new code should prefer the 3-month-pivot tiers (apply-now, +# opportunistic, backend-prep, stretch, skip). +Tier = Literal[ + "apply-now", + "opportunistic", + "backend-prep", + "6-month", + "stretch", + "skip", + "unknown", +] SkillLevel = Literal[0, 1, 2, 3, 4, 5] SkillCategory = Literal[ "language", @@ -32,7 +44,18 @@ "voice", "fine-tuning", ] -Source = Literal["greenhouse", "lever", "ashby", "jobspy", "smoke", "manual"] +Source = Literal["greenhouse", "lever", "ashby", "workday", "jobspy", "smoke", "manual"] +HitlDecision = Literal["approved", "rejected", "auto_rejected", "timed_out"] +InterviewDifficulty = Literal[ + "hackerrank", + "case", + "lc-easy", + "lc-medium", + "lc-medium-hard", + "lc-hard", + "takehome", + "unknown", +] class JobNote(BaseModel): @@ -52,6 +75,7 @@ class JobNote(BaseModel): years_required: int | None = None role_family: str = "" tier: Tier = "unknown" + interview_difficulty: InterviewDifficulty = "unknown" tags: list[str] = [] skills_required: list[str] = [] skills_nice_to_have: list[str] = [] @@ -59,7 +83,7 @@ class JobNote(BaseModel): skills_missing: list[str] = [] jd_summary: str = "" tailored_paragraph: str | None = None - hitl_decision: str | None = None + hitl_decision: HitlDecision | None = None hitl_at: datetime | None = None applied_at: datetime | None = None @@ -99,6 +123,7 @@ class CompanyNote(BaseModel): why_interesting: str = "" known_stack: list[str] = [] interview_format_notes: str = "" + interview_difficulty: InterviewDifficulty = "unknown" tags: list[str] = [] diff --git a/compass/vault/target_companies.py b/compass/vault/target_companies.py index 1dc521a..b12e55c 100644 --- a/compass/vault/target_companies.py +++ b/compass/vault/target_companies.py @@ -1,20 +1,15 @@ -"""Parse _profile/target-companies.md into a company→tier map. - -The file is human-edited but follows a stable section structure: - ## Tier `apply-now` - | Company | ... | - |---|---| - | Sierra | ... | - ... - ## Tier `6-month` - ... - -Parser walks the file once at module import (and on refresh()) and builds a -dict keyed by normalized company name. Naive lookup; no fuzzy matching. - -This is the source of truth for JobNote.tier and CompanyNote.tier during -pipeline runs. Human edits to a CompanyNote's tier are still preserved by -write_company_note (which we don't touch here). +"""Parse _profile/target-companies.md into a company→tier map AND +_profile/target-companies.yaml into a company→full-metadata map. + +Two files cooperate: +- target-companies.md — human-readable narrative + tier tables. Source of truth + for `get_tier` lookups (legacy, tested). +- target-companies.yaml — machine-readable per-company metadata: ATS coords, + geos, interview_difficulty, role_family_hints, notes. + Source of truth for `get_company_meta` lookups (added 2026-05-19). + +Both are kept in sync by hand. Eventually the YAML may absorb the md tier +column outright, but for the 3-month pivot they coexist. """ from __future__ import annotations @@ -23,10 +18,23 @@ import re from typing import Literal +import yaml + logger = logging.getLogger(__name__) -Tier = Literal["apply-now", "6-month", "stretch", "skip", "unknown"] -TIER_ORDER: list[Tier] = ["apply-now", "6-month", "stretch", "skip"] +Tier = Literal[ + "apply-now", "opportunistic", "backend-prep", "6-month", "stretch", "skip", "unknown" +] +# Order: most-preferred first. Used by `get_tier` to break bidirectional-match +# ties (prefer the strongest tier when a company name matches multiple entries). +TIER_ORDER: list[Tier] = [ + "apply-now", + "opportunistic", + "backend-prep", + "6-month", + "stretch", + "skip", +] # NOTE: _TIER_HEADING is intentionally un-anchored so "Tier `apply-now` (in range)" # still captures "apply-now". Don't tighten it without updating the real vault file. @@ -107,15 +115,19 @@ def refresh() -> None: def get_tier(company: str) -> Tier: """Resolve a tier for a company name. - Lookup strategy: - 1. Exact normalized match. - 2. Bidirectional substring fallback — handles the common case where a - scraper board_token is a single word (e.g. "gleanwork", "nvidia") but - target-companies.md uses a longer descriptive name in one cell - (e.g. "Glean", "NVIDIA Agentic AI", "Vapi, Retell, Wispr Flow"). - Either direction (query ⊂ key OR key ⊂ query) counts as a match. - 3. If multiple bidirectional matches with different tiers, the highest - tier (apply-now > 6-month > stretch > skip) wins. + Lookup precedence (highest first): + 1. YAML exact match — `target-companies.yaml` is the 3-month-pivot source + of truth and carries newer entries (banks/consulting/manual-provider) + that the legacy markdown hasn't been updated to reflect. + 2. Markdown exact match — legacy lookup; still useful for entries that + exist only in `target-companies.md`. + 3. Bidirectional substring fallback against the markdown (handles the + common case where a scraper board_token is a single word but the + markdown uses a longer descriptive name). + 4. If multiple substring matches with different tiers, the highest tier + (apply-now > opportunistic > backend-prep > ... > skip) wins. + + Returns "unknown" when no source resolves the company. """ if _company_to_tier is None: refresh() @@ -125,12 +137,19 @@ def get_tier(company: str) -> Tier: if not query: return "unknown" - # Exact match + # 1. YAML exact match — newest source of truth. + yaml_meta = get_company_meta(company) + if yaml_meta is not None: + yaml_tier = yaml_meta.get("tier") + if yaml_tier in TIER_ORDER: + return yaml_tier # type: ignore[return-value] + + # 2. Markdown exact match. direct = _company_to_tier.get(query) if direct is not None: return direct - # Bidirectional substring fallback (guarded by min-length to avoid noise) + # 3. Bidirectional substring fallback (guarded by min-length to avoid noise). if len(query) < _MIN_FUZZY_LEN: return "unknown" @@ -150,3 +169,167 @@ def get_tier(company: str) -> Tier: # import-time parse usually returns {}. Tests must call refresh() after their # fixture monkeypatches cfg.VAULT_PATH. refresh() + + +# ── YAML-driven per-company metadata ───────────────────────────────────────── + +_yaml_meta: dict[str, dict] | None = None +_yaml_mtime: float | None = None # last-seen mtime of target-companies.yaml + + +def _yaml_path(): + """Resolve VAULT_PATH at call time — required so tests that monkeypatch + `compass.config.VAULT_PATH` don't get the module-import-time value.""" + import compass.config as cfg + + return cfg.VAULT_PATH / "_profile" / "target-companies.yaml" + + +def _parse_yaml() -> dict[str, dict]: + """Load _profile/target-companies.yaml into {normalized_company: meta_dict}. + + Returns {} when the file is missing — callers fall through to the legacy + markdown-only behavior. + """ + path = _yaml_path() + if not path.exists(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as e: + logger.warning("target-companies.yaml is malformed: %s", e) + return {} + out: dict[str, dict] = {} + for entry in data.get("companies") or []: + if not isinstance(entry, dict): + continue + name = entry.get("company") + if not name: + continue + # Index by primary name AND any aliases (e.g. JPMorgan ↔ JPMC, BofA ↔ BankofAmerica) + keys = [_normalize(name)] + aliases = entry.get("aliases") or [] + if isinstance(aliases, list): + keys.extend(_normalize(a) for a in aliases if isinstance(a, str)) + for k in keys: + if k: + out[k] = entry + return out + + +def refresh_yaml() -> None: + """Re-parse target-companies.yaml. Call from tests or after manual edits. + Cheap — small file, < 1ms read.""" + global _yaml_meta, _yaml_mtime + _yaml_meta = _parse_yaml() + path = _yaml_path() + _yaml_mtime = path.stat().st_mtime if path.exists() else None + logger.info("target_companies (yaml): parsed %d entries", len(_yaml_meta)) + + +def _maybe_reload_yaml() -> None: + """Reload _yaml_meta if the file on disk has been modified since last + parse. Called by every YAML-reading accessor so a long-running MCP + server picks up edits within ~one mtime check. + + Cheap — a single stat() call per accessor invocation. + """ + global _yaml_meta, _yaml_mtime + if _yaml_meta is None: + refresh_yaml() + return + path = _yaml_path() + if not path.exists(): + if _yaml_meta: + _yaml_meta = {} + _yaml_mtime = None + return + current_mtime = path.stat().st_mtime + if _yaml_mtime is None or current_mtime > _yaml_mtime: + refresh_yaml() + + +def get_company_meta(company: str) -> dict | None: + """Return the full YAML metadata dict for a company, or None if absent. + + Lookup uses the same normalize + bidirectional-substring strategy as + `get_tier`. Returns None — not a default — so callers can distinguish + "company not in target list" from "company has empty metadata". + """ + _maybe_reload_yaml() + assert _yaml_meta is not None # mypy: _maybe_reload_yaml ensures non-None + + query = _normalize(company) + if not query: + return None + + direct = _yaml_meta.get(query) + if direct is not None: + return direct + + if len(query) < _MIN_FUZZY_LEN: + return None + + for key, meta in _yaml_meta.items(): + if len(key) < _MIN_FUZZY_LEN: + continue + if query in key or key in query: + return meta + return None + + +_VALID_DIFFICULTIES = { + "hackerrank", + "case", + "lc-easy", + "lc-medium", + "lc-medium-hard", + "lc-hard", + "takehome", + "unknown", +} + + +def get_interview_difficulty(company: str) -> str: + """Return one of `_VALID_DIFFICULTIES`. YAML typos collapse to "unknown" + so JobNote/CompanyNote schema validation never fails on human edits.""" + meta = get_company_meta(company) + if meta is None: + return "unknown" + raw = str(meta.get("interview_difficulty") or "unknown").strip().lower() + return raw if raw in _VALID_DIFFICULTIES else "unknown" + + +def get_ats(company: str) -> tuple[str, str] | None: + """Return (provider, slug) or None if not in YAML or not scrapable.""" + meta = get_company_meta(company) + if meta is None: + return None + ats = meta.get("ats") or {} + provider = ats.get("provider") + slug = ats.get("slug") + if not provider or not slug: + return None + return (str(provider), str(slug)) + + +def list_yaml_companies(tier_filter: str | None = None) -> list[dict]: + """Return the raw company entries from the YAML, optionally filtered to + one tier. Used by seed scripts + scraper positive-filtering. + + `_yaml_meta` indexes each entry by primary name AND all aliases — so a + naive `list(_yaml_meta.values())` returns N+1 copies for a company with + N aliases (JPMorgan has 5 aliases ⇒ 6 entries). Deduplicate by object + identity so callers (seed_companies_from_yaml, gap_aggregator, MCP tools) + see each company exactly once. + """ + _maybe_reload_yaml() + assert _yaml_meta is not None + # dict-by-id deduplication preserves order of first encounter + unique = list({id(e): e for e in _yaml_meta.values()}.values()) + if tier_filter is not None: + unique = [e for e in unique if e.get("tier") == tier_filter] + return unique + + +refresh_yaml() diff --git a/compass/vault/taxonomy.py b/compass/vault/taxonomy.py index 36e6dad..2d2f563 100644 --- a/compass/vault/taxonomy.py +++ b/compass/vault/taxonomy.py @@ -152,7 +152,7 @@ def _synonym_index() -> dict[str, str]: _CASE_SENSITIVE_CANONICALS: frozenset[str] = frozenset({"ReAct", "Go"}) -def normalize(raw_skill: str) -> str | None: +def normalize(raw_skill: str | None) -> str | None: """Map an arbitrary skill string to its canonical form, or None if unknown. Strict synonym match only. The previous substring fallback produced false @@ -165,7 +165,14 @@ def normalize(raw_skill: str) -> str | None: `_CASE_SENSITIVE_CANONICALS` blocks specific collisions: an input "React" (capital R only) won't map to canonical "ReAct" because the case patterns differ. The LLM has to emit the exact canonical case for those skills. + + Defensive: `None` / empty / non-str inputs return `None` rather than + crash. A JobNote with a YAML list-entry written as `- ` parses as + `[None]`; without this guard, gap_aggregator.aggregate() crashes the + whole regenerate() call on a single malformed JobNote. """ + if not raw_skill or not isinstance(raw_skill, str): + return None key = _norm_key(raw_skill) if not key: return None @@ -191,3 +198,21 @@ def category_for(canonical: str) -> str | None: def all_canonicals() -> list[str]: return list(load_taxonomy().keys()) + + +def refresh_taxonomy() -> None: + """Clear the LRU caches so the next access re-reads `skill-taxonomy.md`. + + Both `load_taxonomy` and `_synonym_index` are `@lru_cache(maxsize=1)` — + long-running processes (the MCP server, the pipeline) cache them at + first call and never see file edits. Tests that monkeypatch + `TAXONOMY_PATH` get stale data too. + + Call this: + - From the MCP server before any analysis tool (so user edits to + skill-taxonomy.md are picked up between invocations) + - From test fixtures that swap the taxonomy file + - After any script that rewrites the canonical taxonomy + """ + load_taxonomy.cache_clear() + _synonym_index.cache_clear() diff --git a/compass/vault/url_dedup.py b/compass/vault/url_dedup.py new file mode 100644 index 0000000..3729bd3 --- /dev/null +++ b/compass/vault/url_dedup.py @@ -0,0 +1,95 @@ +"""URL normalization for deduplication. + +The pipeline dedups jobs by URL — but raw URLs vary in case, scheme, trailing +slashes, and tracking params. Two scrapers (or the same scraper across runs) +can produce different strings for the same job: + + https://jobs.ashbyhq.com/sierra/abc + https://jobs.ashbyhq.com/sierra/abc/ + https://jobs.ashbyhq.com/sierra/abc?utm_source=google + http://JOBS.ASHBYHQ.COM/sierra/abc + +Without normalization the dedup at `_vault_url_set()` and the URL-match in +`write_job_note()` treat all of these as distinct, leading to duplicate +JobNotes for the same role. +""" + +from __future__ import annotations + +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +# Query params that are tracking-only and don't change which job a URL points +# to. Strip them before dedup so the same JD via Google + LinkedIn refer counts +# once. Conservative list — anything not here is preserved. +_TRACKING_PARAMS = frozenset( + { + "utm_source", + "utm_medium", + "utm_campaign", + "utm_term", + "utm_content", + "gclid", + "fbclid", + "msclkid", + "ref", + "ref_src", + "referer", + "referrer", + "src", + "source", + "campaign", + } +) + + +def normalize_url(url: str) -> str: + """Return a canonical form of `url` for dedup purposes. + + Normalization rules: + - Lowercase the scheme + host (URL components below the path are + case-insensitive per RFC 3986). + - Strip default ports (`:80` for http, `:443` for https). + - Strip trailing slash from the path (treat /abc and /abc/ as same). + - Remove known tracking query params (utm_*, gclid, etc.). + - Sort remaining query params alphabetically. + - Drop the URL fragment (#section anchors are client-side only). + - HTTP and HTTPS variants of the same URL collapse to https. + + NOT normalized: + - User-info, port (non-default). + - Path case (servers may treat /Foo and /foo differently). + - Non-tracking query params. + + Returns the original `url` unchanged if parsing fails — dedup degrades + gracefully to per-string matching for that one URL rather than crashing + the pipeline. + """ + if not url: + return url + try: + parts = urlsplit(url.strip()) + except ValueError: + return url + scheme = parts.scheme.lower() or "https" + # http and https collapse — same job, both schemes, one canonical. + if scheme == "http": + scheme = "https" + host = (parts.hostname or "").lower() + # Reconstruct netloc — drop user-info (rare in JD URLs, no signal) and + # drop default ports. + if parts.port and not ( + (scheme == "https" and parts.port == 443) or (scheme == "http" and parts.port == 80) + ): + netloc = f"{host}:{parts.port}" + else: + netloc = host + path = parts.path.rstrip("/") + # Strip tracking params, sort the rest. + kept = sorted( + (k, v) + for k, v in parse_qsl(parts.query, keep_blank_values=True) + if k.lower() not in _TRACKING_PARAMS + ) + query = urlencode(kept) + # Drop fragment — anchors are client-side, not a different resource. + return urlunsplit((scheme, netloc, path, query, "")) diff --git a/compass/vault/writer.py b/compass/vault/writer.py index 579d029..af7cc31 100644 --- a/compass/vault/writer.py +++ b/compass/vault/writer.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib +import html import logging import re from datetime import datetime @@ -18,24 +19,54 @@ import frontmatter from compass.config import AGENT_LOG_PATH, VAULT_PATH -from compass.vault.schemas import ApplicationNote, CompanyNote, JobNote, SkillCategory, SkillNote -from compass.vault.taxonomy import category_for if TYPE_CHECKING: from pathlib import Path from pydantic import BaseModel + from compass.vault.schemas import ApplicationNote, CompanyNote, JobNote + logger = logging.getLogger(__name__) _FILENAME_BAD = re.compile(r"[^\w\-.]+") +_SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE) +_HTML_TAG_RE = re.compile(r"<[^>]+>") def _safe_segment(s: str) -> str: return _FILENAME_BAD.sub("_", s).strip("_") +def _looks_like_html(text: str) -> bool: + """Cheap detector — `<p>`, `</div>`, `<span ...>` style tags. Avoids the + rare case where a JD contains a literal `<` (e.g. ASCII art, code snippet).""" + return bool( + re.search(r"</[a-z]+>|<[a-z]+ [^>]*>|<(p|div|span|strong|br|h\d)\b", text, re.IGNORECASE) + ) + + +def _normalize_full_jd(text: str) -> str: + """Safety-net HTML strip applied to JD bodies at vault-write time. + + Each ATS scraper already strips HTML at its boundary (greenhouse/lever + via `_strip_html`, ashby via `descriptionPlain`). This is belt-and- + suspenders: if a future scraper or a hand-built RawJob bypasses that, + the vault still gets clean text instead of `</span></strong></p>` cruft. + + Only fires when the input visibly looks like HTML — JDs containing + literal `<` (code snippets, ASCII art) pass through untouched. + """ + if not _looks_like_html(text): + return text + out = _SCRIPT_STYLE_RE.sub(" ", text) + out = _HTML_TAG_RE.sub(" ", out) + out = html.unescape(out) + out = re.sub(r"\s+", " ", out).strip() + return out + + def _job_filename(note: JobNote) -> str: """JobNote filename includes a short URL hash so titles that sanitize to the same string (e.g. "Engineer / Backend" vs "Engineer (Backend)") never @@ -57,6 +88,38 @@ def _to_metadata(model: BaseModel) -> dict: return model.model_dump(mode="json", by_alias=True) +def _wikilink(skill: str) -> str: + """Render a skill as an Obsidian wikilink pointing at skills/<safe>.md. + + SkillNotes are stored under filenames produced by `_safe_segment`. When the + skill name contains characters that get rewritten (e.g. "AWS Bedrock" → + "AWS_Bedrock", "C++" → "C__"), emit the alias form `[[target|display]]` so + the link resolves AND the display matches what the user typed. + """ + target = _safe_segment(skill) + return f"[[{target}|{skill}]]" if target != skill else f"[[{skill}]]" + + +def _render_skills_section(note: JobNote) -> str: + """Render `## Skills` body block. Empty categories are omitted entirely.""" + rows: list[tuple[str, list[str]]] = [ + ("Required", note.skills_required), + ("Nice to have", note.skills_nice_to_have), + ("Matched", note.skills_matched), + ("Missing", note.skills_missing), + ] + lines = ["## Skills", ""] + any_row = False + for label, skills in rows: + if not skills: + continue + any_row = True + lines.append(f"**{label}:** " + " · ".join(_wikilink(s) for s in skills)) + if not any_row: + return "" + return "\n".join(lines) + "\n" + + def write_job_note(note: JobNote, full_description: str | None = None) -> Path: """Write a JobNote to vault/jobs/. Idempotent on URL — same URL overwrites the same file. @@ -67,21 +130,28 @@ def write_job_note(note: JobNote, full_description: str | None = None) -> Path: jobs_dir = VAULT_PATH / "jobs" jobs_dir.mkdir(parents=True, exist_ok=True) + from compass.vault.url_dedup import normalize_url + + note_url_norm = normalize_url(note.url) target: Path | None = None for existing in jobs_dir.glob("*.md"): try: post = frontmatter.load(existing) except Exception: continue - if post.metadata.get("url") == note.url: + existing_url = post.metadata.get("url") + if isinstance(existing_url, str) and normalize_url(existing_url) == note_url_norm: target = existing break if target is None: target = jobs_dir / _job_filename(note) body = f"# {note.company} — {note.title}\n\n{note.jd_summary}\n" + skills_block = _render_skills_section(note) + if skills_block: + body += f"\n{skills_block}" if full_description: - body += f"\n## Full JD\n\n{full_description.strip()}\n" + body += f"\n## Full JD\n\n{_normalize_full_jd(full_description).strip()}\n" post = frontmatter.Post(content=body) post.metadata = _to_metadata(note) target.write_text(frontmatter.dumps(post) + "\n", encoding="utf-8") @@ -89,31 +159,6 @@ def write_job_note(note: JobNote, full_description: str | None = None) -> Path: return target -def update_skill_note(canonical_skill: str, job_url: str) -> Path: - """Increment appears_in_jobs on a skill note. Creates a minimal note if missing.""" - skills_dir = VAULT_PATH / "skills" - skills_dir.mkdir(parents=True, exist_ok=True) - path = skills_dir / f"{_safe_segment(canonical_skill)}.md" - - if path.exists(): - post = frontmatter.load(path) - post.metadata["appears_in_jobs"] = int(post.metadata.get("appears_in_jobs", 0)) + 1 - else: - category: SkillCategory = category_for(canonical_skill) or "language" # type: ignore[assignment] - if category_for(canonical_skill) is None: - logger.warning( - "update_skill_note: %r not in canonical taxonomy; defaulting category=language", - canonical_skill, - ) - skill = SkillNote(skill=canonical_skill, category=category, appears_in_jobs=1) - post = frontmatter.Post(content=f"# {canonical_skill}\n") - post.metadata = _to_metadata(skill) - - path.write_text(frontmatter.dumps(post) + "\n", encoding="utf-8") - append_agent_log(f"vault_write skill {canonical_skill} += 1 (from {job_url})") - return path - - def write_company_note(note: CompanyNote) -> Path: """Write or update a company note. @@ -144,7 +189,15 @@ def write_company_note(note: CompanyNote) -> Path: # against that by ignoring invalid tier values (they get reset to # whatever the pipeline computed). existing_tier = existing.get("tier", "unknown") - _valid_tiers = {"apply-now", "6-month", "stretch", "skip", "unknown"} + _valid_tiers = { + "apply-now", + "opportunistic", + "backend-prep", + "6-month", + "stretch", + "skip", + "unknown", + } if existing_tier not in _valid_tiers: logger.warning( "write_company_note: %s has invalid tier=%r (expected one of %s); " diff --git a/docs/HOW_COMPASS_WORKS.md b/docs/HOW_COMPASS_WORKS.md new file mode 100644 index 0000000..9e61607 --- /dev/null +++ b/docs/HOW_COMPASS_WORKS.md @@ -0,0 +1,441 @@ +# How Compass Works + +A plain-English walkthrough of what's actually in this project, how the pieces +fit together, and how data flows from a scraped job posting to a JobNote you +can apply against. Written 2026-05-19 after 14 commits past the +`phase-1b2-rag` tag. + +--- + +## 1. What Compass is + +An agentic career coach. You point it at ~50 company career boards, it +scrapes job postings, identifies which ones match your skill profile, scores +them against you with reasoning, tailors a paragraph for each, generates +cover letters on demand, regrades your skills against the live agent-eng +market, and tells you what to study next. + +The whole thing runs locally on your machine. There's a daily-cron mode +(Modal) but it's not required — you can run it manually whenever. + +**The interview story** is the headline: "I built the system I'm using to +run my own job search, and I built an agent inside it that grades my skills +against the live job market and tells me what to study next." + +--- + +## 2. The three vaults (data lives in three places) + +| Vault | Path | Owner | What's in it | +|---|---|---|---| +| **compass-vault** | `~/Documents/compass-vault/` | Compass writes here; you edit some files | JobNotes, SkillNotes, CompanyNotes, applications, master gap plan, dashboard, the `_profile/` files (resume, skill-inventory, preferences, target-companies) | +| **learning-vault** | `~/Documents/learning-vault/` | You own; Compass reads only via `learning-vault://` URIs | Decision logs, debug notes, courses, leetcode practice, daily notes — your second brain. Compass's skill_assessor cites these as evidence. | +| **compass repo** | `~/Documents/compass/` | Code only | Python source, tests, scripts, this doc | + +The two vaults are intentionally separated. compass-vault is what the agent +writes (it can be wiped and rebuilt). learning-vault is what you write +(personal notes; the agent never modifies it). Skills get graded by reading +your learning-vault evidence URIs. + +--- + +## 3. The data flow (one job, end to end) + +``` + ┌───────────────────────────────┐ + │ YAML target companies │ + │ _profile/target-companies. │ + │ yaml — 49 entries with ATS │ + │ slugs │ + └───────────────┬───────────────┘ + │ + ┌────────────────┴────────────────┐ + ↓ ↓ + ┌──────────────────────┐ ┌───────────────────┐ + │ scrape_greenhouse_ │ │ scrape_workday_ │ + │ many / lever / ashby │ │ many (4 banks + │ + │ (37 boards) │ │ Adobe) │ + └──────────┬────────────┘ └─────────┬─────────┘ + ↓ ↓ + └────────────────┬───────────────┘ + ↓ + ┌────────────────────────────────┐ + │ _scrape_all (graph.py) │ + │ - drop date_posted > 30d old │ + │ - sort each board by recency │ + │ - round-robin interleave │ + │ - cap to MAX_JOBS_PER_RUN=50 │ + └────────────────┬───────────────┘ + ↓ + ┌────────────────────────────────┐ + │ _vault_url_set + dedup │ + │ Normalize URLs (case / scheme │ + │ / utm / trailing slash / │ + │ fragment) — same job via two │ + │ URL variants collapses to one │ + └────────────────┬───────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ PER-JOB graph (5 concurrent via │ + │ semaphore; LangGraph with │ + │ AsyncSqliteSaver checkpointer) │ + │ │ + │ START → intake → intake_filter → │ + │ │ │ + │ (out-of-scope)→ END │ + │ (in-scope) ↓ │ + │ extract → score → reflect → hitl → │ + │ │ │ + │ (approved) → tailor → │ + │ ↓ │ + │ (rejected/auto) ─→ vault_write│ + │ ↓ │ + │ END │ + └────────────────┬─────────────────────────┘ + ↓ + ┌────────────────────────────────┐ + │ Post-batch: │ + │ gap_aggregator.regenerate() │ + │ - sync SkillNote counters │ + │ - rebuild backlinks │ + │ - sync CompanyNote roles_seen │ + │ - rewrite master-gap-plan.md │ + │ (atomic via os.replace) │ + └────────────────────────────────┘ +``` + +### The 8 pipeline nodes — what each does to a single job + +1. **intake** — packages the RawJob into the LangGraph state. No I/O. +2. **intake_filter** — three gates BEFORE any LLM call: + - **Reject rules from preferences.md**: `Senior` / `Staff` / `Principal` in + title → drop. `5+ years` / `PhD required` in JD body → drop. + - **`role_family.keyword_classify(title)`**: maps titles to families + (`agent-engineer`, `applied-ai`, `infra-llm`, `swe-backend`, etc.) + using IN/OUT keyword lists. + - **Agent-body-signal gate**: AI-titled JDs (`agent-engineer` / + `applied-ai` / `infra-llm`) MUST contain at least one strong signal + ("LangGraph", "MCP", "multi-agent", "tool calling", etc.) — weak + mentions of just "agent" aren't enough. +3. **extract** — Pydantic-AI LLM call (default Gemini Flash). Returns + `JobRequirements{required_skills, nice_to_have_skills, years_experience, + seniority, remote_policy, summary}`. Post-processing canonicalizes skill + names against `_meta/skill-taxonomy.md` and drops skills the JD body + doesn't actually mention (anti-hallucination guard). +4. **score** — Pydantic-AI LLM call (default Gemini Flash). Compares + `JobRequirements` against your candidate profile (resume + RAG-retrieved + skill-inventory chunks + YAML targeting context). Returns + `JobScore{score 0-5, reasoning, matched_skills, missing_skills, + tailoring_notes}`. Post-processing retries on truncated reasoning + drops + matched/missing outside the JD's skill universe (anti-hallucination). +5. **reflect** — currently a no-op placeholder for future critique-revise. +6. **hitl** — calls `interrupt({kind: "approval_request", ...})` when score + meets threshold. Resumes via MCP `approve_job` or Modal cron timeout. + The `interrupt()` checkpoints state via `AsyncSqliteSaver`, so the + pipeline can pause for hours and resume cleanly. +7. **tailor** — Sonnet LLM call. Only fires on approved jobs above + threshold (cost gate). Returns a 3-5 sentence application paragraph + anchored on specific projects from `_profile/role-clarifications.md`. +8. **vault_write** — persists JobNote to `compass-vault/jobs/`, upserts + CompanyNote, adds auto-tags (`#tier/...`, `#fit/...`, `#role/...`, + `#signal/agent-strong`, `#decision/...`). Idempotent on URL (uses + normalize_url for dedup). + +--- + +## 4. The components — what each module does + +``` +compass/ +├── pipeline/ +│ ├── graph.py ← LangGraph orchestration; run_pipeline entry point +│ ├── state.py ← CompassState TypedDict (what each node reads/writes) +│ ├── intake.py +│ ├── role_family.py ← Title classifier (IN/OUT keywords + LLM fallback) +│ ├── add_url.py ← Manual URL→RawJob for sites Compass can't auto-scrape +│ ├── cover_letter.py ← On-demand cover letter draft (Sonnet) +│ └── nodes/ +│ ├── intake_filter.py ← reject rules + role_family + agent-signal gate +│ ├── extract.py +│ ├── score.py +│ ├── reflect.py +│ ├── hitl.py ← interrupt() pause-for-approval +│ ├── tailor.py +│ └── vault_write.py +│ +├── scrapers/ +│ ├── greenhouse.py ← Public Greenhouse API (boards-api.greenhouse.io) +│ ├── lever.py ← Lever public API +│ ├── ashby.py ← Ashby public API +│ ├── workday.py ← Workday JSON endpoint (banks + Adobe — 5 tenants) +│ └── _remote_parser.py +│ +├── vault/ +│ ├── writer.py ← write_job_note / write_company_note / write_application_note +│ ├── reader.py ← read_resume / read_profile_section / load_reject_rules +│ ├── schemas.py ← Pydantic models for every note type +│ ├── taxonomy.py ← Skill canonicalization (sync against _meta/skill-taxonomy.md) +│ ├── target_companies.py ← YAML + markdown parsers; get_tier / get_company_meta +│ ├── learning_bridge.py ← Resolves learning-vault:// URIs (path-jailed) +│ └── url_dedup.py ← normalize_url for scheme/case/utm/etc. +│ +├── rag/ +│ ├── indexer.py ← Builds Chroma index from _profile/skill-inventory.md +│ └── retriever.py ← Top-k cosine retrieval for score_node context +│ +├── analysis/ +│ ├── gap_aggregator.py ← Sync counters, rebuild backlinks, rewrite master plan +│ └── skill_assessor.py ← Adversarial-grader assessing skills against evidence URIs +│ +├── hitl/ +│ ├── state_store.py ← SQLite tracking pending HiTL approvals + atomic claim +│ ├── resume.py ← Single source-of-truth for resuming a paused thread +│ └── timeout_checker.py ← Modal-cron consumer for stale pending approvals +│ +├── evals/ +│ ├── dataset.py ← EvalRecord schema + JSON dataset round-trip +│ ├── metrics.py ← Pure-function MAE / RMSE / bias / recall / precision +│ ├── judge.py ← LLM-as-judge (no hand labels needed) +│ └── runner.py ← Two modes: --labels (rigorous) / --judge (cheap baseline) +│ +├── applications/ +│ └── lifecycle.py ← Application status tracker +│ +├── mcp_server/ +│ └── server.py ← FastMCP exposing the pipeline + analysis + vault as tools +│ +├── llm.py ← make_agent() — pydantic-ai factory + OpenRouter routing +└── config.py ← Env-var loading; per-node model selection +``` + +--- + +## 5. The MCP tools (what Claude Code can invoke) + +The MCP server in `compass/mcp_server/server.py` exposes 12 tools: + +| Tool | Use case | +|---|---| +| `score_jd(jd_text)` | Score a pasted JD without writing to vault — cheap sanity check | +| `add_job_from_url(url)` | Auto-fetch + score + write a JobNote (greenhouse/lever/ashby/workday/generic HTML) | +| `add_job_from_text(company, title, url, jd_text)` | For JS-rendered sites Compass can't scrape (JPM Oracle Cloud, LinkedIn) | +| `generate_cover_letter(jobnote_filename)` | Sonnet-quality 250-400 word cover letter draft for a specific JobNote | +| `run_evals(mode, limit)` | Run the eval harness — measures extract recall + score MAE | +| `search_jobs(query, limit)` | Substring search over JobNote bodies + frontmatter | +| `get_skill_gaps(job_id)` | Matched + missing skills for one JobNote | +| `get_profile(section)` | Read a `_profile/` file (path-jailed) | +| `read_learning_artifact(uri)` | Resolve a `learning-vault://` URI (path-jailed) | +| `assess_skills(scope)` | Regrade skills via adversarial grader against evidence URIs | +| `regenerate_gap_plan()` | Rebuild `study-plans/master-gap-plan.md` | +| `get_master_gap_plan()` | Read current top gaps | +| `suggest_evidence(skill, search_terms)` | Surface candidate learning-vault files to cite | +| `list_canonical_skills()` | Enumerate the taxonomy | +| `add_application(...)` | Mark a JobNote as applied; create applications/ note | + +--- + +## 6. The skill-assessor meta-loop (the differentiated bit) + +This is the loop that makes Compass interesting as a portfolio artifact: + +``` + ┌─────────────────────────────────────────┐ + │ You add a new evidence URI to a │ + │ SkillNote (e.g. point LangGraph at │ + │ compass/pipeline/graph.py via the │ + │ decision-log note) │ + └──────────────────┬──────────────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ Run `assess_skills(scope=["LangGraph"])│ + │ via MCP or as a Modal cron │ + └──────────────────┬──────────────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ skill_assessor: │ + │ - resolve_many(evidence_uris) → reads │ + │ each learning-vault:// URI (path- │ + │ jailed; raises on traversal attempt) │ + │ - construct adversarial-grader prompt │ + │ with 5-level rubric + dissenting-view │ + │ requirement │ + │ - Pydantic-AI Agent returns │ + │ SkillAssessment{proposed_level, │ + │ cited_evidence, reasoning, │ + │ dissenting_view, confidence, │ + │ requires_hitl} │ + └──────────────────┬──────────────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ Promotion rules: │ + │ - single-level change → auto-apply │ + │ - 2+ level change → requires_hitl=True; │ + │ user reviews via MCP before applying │ + │ - grade_override set → ignore assessor │ + └──────────────────┬──────────────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ Updates: │ + │ - SkillNote.my_level │ + │ - SkillNote body: ## Latest assessment │ + │ notes section appended/updated │ + │ - _profile/skill-inventory.md table │ + │ regenerated │ + │ - _meta/agent-log.md gets a one-line │ + │ audit entry │ + └──────────────────┬──────────────────────┘ + ↓ + ┌─────────────────────────────────────────┐ + │ Next pipeline run's score_node sees │ + │ the new level via RAG retrieval — so │ + │ JD scoring AUTOMATICALLY reflects your │ + │ evolving skill graph as you ship work │ + └─────────────────────────────────────────┘ +``` + +This is the part the spec calls the "differentiated angle" — Compass doesn't +just find jobs, it grades you against the live market and tells you what to +study next, with documented evidence chains from your `learning-vault/`. + +--- + +## 7. The vault layout (what gets written where) + +``` +compass-vault/ +├── jobs/ ← JobNotes (one per JD) +│ └── YYYY-MM-DD-{company}-{title}-{8charhash}.md +├── companies/ ← CompanyNotes (one per company; human-editable fields preserved) +│ └── {company}.md +├── skills/ ← SkillNotes (95 seeded from taxonomy) +│ └── {skill}.md +├── applications/ ← One per submitted application +│ └── YYYY-MM-DD-{company}-{title}-{hash}.md +├── cover-letters/ ← Generated cover letters +│ └── YYYY-MM-DD-{company}-{title}-{hash}.md +├── study-plans/ +│ ├── master-gap-plan.md ← Auto-regenerated; atomic write (os.replace) +│ └── *.archive-pre-pivot-*.md ← Old plans (preserved on wipe) +├── _profile/ ← YOU edit these — Compass reads them +│ ├── resume.md +│ ├── skill-inventory.md +│ ├── preferences.md +│ ├── target-companies.md ← Human-readable narrative +│ ├── target-companies.yaml ← Machine-readable; drives the scraper +│ ├── target-roles.md +│ ├── role-clarifications.md +│ └── interview-prep.md +├── _meta/ ← Compass-managed metadata +│ ├── skill-taxonomy.md ← Canonical skill list + synonyms + categories +│ ├── agent-log.md ← Append-only audit trail of every vault write +│ ├── pipeline-runs.md ← Per-run forensic summary row +│ ├── filtered-jobs.md ← Audit log of every dropped JD with reason +│ ├── unknown-skills-log.md ← Skills the LLM emitted but taxonomy doesn't know +│ └── *.archive-pre-pivot-*.md +├── dashboard.md ← Dataview-powered: 5 panels (apply-now / velocity / etc.) +└── jobs.archive-pre-pivot-*/ ← Old JobNotes (preserved on wipe) +``` + +--- + +## 8. How a single full pipeline run looks (timeline) + +For a manual `uv run python -m compass.pipeline.graph` invocation: + +``` +0:00.0 start_wall captured +0:00.1 _scrape_all begins — 4 scrapers run concurrently +0:01.5 greenhouse_many returns ~150 JDs from 17 boards +0:02.0 ashby_many returns ~200 JDs from 24 boards +0:03.0 lever_many returns 0 (no lever targets in YAML) +0:05.0 workday_many returns ~80 JDs from 5 banks (slower; 2 fetches per JD) +0:05.5 _filter_and_sort_by_recency drops ~40% as >30 days old → 270 jobs +0:05.6 round-robin interleave + cap to MAX_JOBS_PER_RUN=50 → 50 jobs +0:05.7 _vault_url_set normalizes URLs from existing JobNotes; dedup → 48 fresh +0:05.8 AsyncSqliteSaver checkpoint DB opens; build_graph compiles +0:06.0 asyncio.gather over 48 _process_one calls, bounded by semaphore=5 + Each _process_one: + intake → intake_filter (reject rules + role_family + agent gate) + ~50% drop here at zero LLM cost + extract (Flash) → score (Flash) → reflect (no-op) → hitl + hitl auto-rejects below SCORE_THRESHOLD=3.5 + hitl interrupts for human approval on score >= threshold + non-interrupt path: vault_write + interrupt-path: state_store.add_pending; main loop continues +0:30.0 ~24 jobs reach vault_write (other 24 dropped at intake_filter) +0:30.5 ~6 jobs paused for HiTL (score >= 3.5); thread_ids logged +0:30.6 asyncio.gather completes +0:30.7 gap_aggregator.regenerate() syncs counters, rebuilds backlinks, + writes master-gap-plan.md (atomic via os.replace) +0:31.0 pipeline-runs.md gets a one-line summary row +0:31.1 process exits +``` + +For each HiTL-paused job, the user later runs `approve_job(thread_id)` or +the Modal cron `check_and_resume_timeouts` fires after the 4-hour window +and auto-rejects. Resume goes through `claim_pending` to prevent +double-resume races, then continues from the checkpoint past `hitl_node` +into `tailor` (if approved) and `vault_write`. + +--- + +## 9. Cost expectations (approximate, OpenRouter pricing) + +| Cost item | Per-unit | Typical batch | +|---|---|---| +| extract (Gemini Flash) | ~$0.0003/JD | $0.015 / 50-JD batch | +| score (Gemini Flash) | ~$0.0005/JD | $0.025 / 50-JD batch | +| tailor (Sonnet) | ~$0.05/JD | $0.30 / 6-approved batch | +| cover_letter (Sonnet, on demand) | ~$0.08/letter | $0.08 / app | +| assess_skills (Sonnet, on demand) | ~$0.02/skill | $0.40 / full re-grade | +| eval --judge (Flash) | ~$0.002/JD | $0.04 / 20-record run | + +**A typical day's run** (manual refresh + tailoring 5 approved jobs + +cover-letters for the 3 you apply to + 1 eval run) costs roughly $0.50. +At 7 runs/week that's $3.50/week. Modal compute is free at this scale. + +--- + +## 10. What's NOT in the pipeline (deferred / out of scope) + +- **Modal cron deploy** — daily scrape + weekly skill-assessor. + Not deployed. Manual runs work fine for the 2-month sprint. +- **Cisco-internal manual tracker** — AI Hub / Outshift / Webex AI / + ThousandEyes AI. Need to come from a manager referral, not the + ATS scrape. +- **Banks/consulting beyond the 5 Workday tenants** — JPM Oracle Cloud, + Capital One Workday (non-discoverable site), etc. Use `add_job_from_url` + per specific role. +- **Auto-apply** — explicit non-goal. Compass is research + preparation. +- **LinkedIn integration** — explicit non-goal (ToS + rate-limit pain). + +--- + +## 11. The honest summary + +What Compass does well: +- Scrapes 41 verified ATS boards in one command +- Filters out senior/staff/principal + non-agentic JDs cheaply (no LLM) +- Scores remaining JDs against your profile with reasoning + matched/ + missing skills +- Pauses for human approval on strong-fit jobs; auto-rejects weak-fit +- Tailors a paragraph + cover letter per approved JD +- Regrades your skills against documented evidence as you ship work +- Surfaces gaps with weighted ranking (tier × frequency × match score) +- Surfaces realistic apply-now roles with easy-loop filter for the + 8-week sprint +- Audit-trails every decision in `_meta/` + +What it doesn't do (yet): +- Run on a daily cron (Modal undeployed) +- See Oracle Cloud / iCIMS / SmartRecruiters job boards (~30% of the + apply-now market) without manual paste +- Auto-update applications/ from interview emails +- Suggest specific cover-letter A/B variants +- Track interview prep progress against per-skill study plans + +The architecture is intentionally separable. Each scraper is independent; +each pipeline node is testable in isolation; the vault is the +serialization boundary between code and user. If you wanted to swap the +LLM provider tomorrow it's an env-var change. If you wanted to add a new +ATS, it's one scraper file matching the existing interface. diff --git a/docs/KNOWN_DATA_QUALITY_ISSUES.md b/docs/KNOWN_DATA_QUALITY_ISSUES.md new file mode 100644 index 0000000..90d6526 --- /dev/null +++ b/docs/KNOWN_DATA_QUALITY_ISSUES.md @@ -0,0 +1,179 @@ +# Known Data-Quality Issues — Pipeline Behavior + +> Surfaced during the Phase 1.B.2 deep adversarial review against real vault data. These are pre-existing Phase 0.B / 1.A LLM-behavior defects that distort scoring + skill matching. Fix scope: **Phase 2.A's eval harness work** — each needs a regression test against labeled JDs before prompt tuning, otherwise we whack-a-mole. + +**Tag context:** discovered at `phase-1b2-rag`. Vault snapshot: 23 JobNotes across Anthropic / Decagon / Cognition / Sierra / Ramp. + +--- + +## B1. `extract_node` under-extracts `skills_required` on best-fit roles + +**Symptom:** roles that are obvious matches for the candidate end up with empty or near-empty `skills_required` lists, which artificially caps the score because the scorer has nothing to match against. + +**Evidence:** +- `jobs/2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f.md` → `skills_required: []`, `skills_matched: []`, `match_score: 3.0`. The JD describes building an agent SDK + runtime + evals — a near-perfect fit for the candidate's MCP/agent cluster (level 3-4). The score reasoning even acknowledges the alignment, but the score caps at 3.0 because there are no required-skill entries to flip into matched. **A fair human grade is ~4.0.** +- `jobs/2026-05-06-cognition-Software_Engineer-7c26fdaf.md` → `skills_required: [Python]` only, despite the JD emphasizing sub-agent orchestration, tool use, agent infrastructure. + +**Hypothesis:** the extract prompt is being too literal about "explicit requirements only" and missing skills implied by responsibility statements. JDs at agentic startups frequently lead with mission text ("build the orchestration engine"), with the canonical skill list buried or implied. + +**Phase 2.A fix surface:** +- Tighten extract prompt to also pull canonical skills from responsibility statements (`build orchestration engine` → LangGraph, `subagent coordination` → Multi-Agent). +- Add a labeled JD eval where the ground-truth skill list includes implied skills; assert extract recall ≥ 0.7. + +--- + +## B2. `extract_node` mis-reads OR-lists as AND-lists + +**Symptom:** "languages such as A, B, C, or similar" is interpreted as four ANDed requirements. Candidate gets dinged on missing alternatives the JD didn't actually require. + +**Evidence:** +- `jobs/2026-04-15-cognition-AI_Support_Engineer-25133cbd.md` → JD: "Strong ability to read and reason about code in **multiple languages, such as Python, TypeScript, Java, Go, or similar**". Extracted: `skills_required: [Python, TypeScript, Go, Docker]`. Scorer marked TS+Go as missing → 2.0 → auto-rejected. **A fair human assigns ~3.0-3.5.** The role was incorrectly auto-rejected by extraction artefact. + +**Phase 2.A fix surface:** +- Extract prompt addendum: explicit OR-list detection ("languages such as X, Y, or Z" → store as `nice_to_have_skills` or a separate `language_alternatives` field, not as `skills_required`). +- Eval coverage: at least 3 OR-list JDs in the labeled set with ground-truth scores. + +--- + +## B3. `score_node` occasionally marks candidate-strong skills as missing + +**Symptom:** scorer LLM lists a skill in `missing_skills` despite the candidate having a non-trivial level in the profile. + +**Evidence:** +- `jobs/2026-04-07-ramp-Security_Engineer_Cloud-a25113ae.md` → `skills_required: [Python]`, `skills_missing: [Python]`, `skills_matched: []`. The candidate has Python at level 3 (skill-inventory.md). RAG would have surfaced the Python chunk on any query containing "Python". The scorer LLM didn't consult the retrieved profile faithfully. +- Low-fit branch failure mode: when the overall role is wildly off (security, in this case), the scorer appears to short-circuit and declare every required skill missing without verifying the candidate's level. + +**Phase 2.A fix surface:** +- Score prompt assertion: "Before adding a skill to missing_skills, verify it is NOT in the candidate's profile at level ≥ 2." +- Defensive post-filter: in `score.py`, augment `_constrain_to_jd_skills` to also check `skills_missing` against the candidate's profile and downgrade false-misses to nothing. (Requires access to canonical skill levels at score time — currently only the chunks are passed.) +- Eval regression test: "if candidate has skill X at level ≥ 3 AND skill X is in `skills_required`, then X MUST appear in `skills_matched`, never in `skills_missing`." + +--- + +## B4. ✅ FIXED in `phase-1b2-rag` — `intake_filter` title-keyword stage was leaky + +Resolved at this tag: added `engineering manager`, `engineering lead`, `director/head/vp of engineering`, `solutions engineer`, `solution engineer`, `security engineer`, `application security`, `infrastructure security`, `operations specialist`, `product operations`, `program manager` to `OUT_SUBSTRING_KEYWORDS`. The four false-negatives observed (Decagon EM, Ramp AI Ops Specialist, Decagon Senior Solutions Eng, Sierra Security Eng) now correctly drop at intake. + +--- + +## B5. Anti-claim gating is inconsistent on post-training/research roles + +**Symptom:** the candidate's `role-clarifications.md` declares SFT/LoRA/RLHF/DPO as level 0 (explicit anti-claim). Sometimes the scorer catches this, sometimes it doesn't. + +**Evidence:** +- Cognition Research Post-Training role → correctly scored 0.0, reasoning cited RLHF=0. +- `jobs/2025-08-22-decagon-Senior_Research_Engineer-5030d3e2.md` → JD requires "Prior experience post-training and deploying LLMs in production". Scored 3.0. The disqualifier was missed. + +**Phase 2.A fix surface:** +- The anti-claim signal should be a hard rule, not a contextual hint. Add an explicit pre-score check: if JD `required_skills` contains any of {fine-tuning, RLHF, post-training, SFT, LoRA, DPO} AND candidate role-clarification anti-claims include them, then score is capped at 1.0 regardless of LLM output. + +--- + +--- + +## B6. Derived-field staleness asymmetry — `role_family` is set once and never reconciled + +**Symptom:** when the keyword OUT list expands (as in commit `3828d8f`), existing JobNotes that should now be classified out-of-scope keep their old `role_family`. `gap_aggregator.regenerate()` reads ALL JobNotes regardless of `role_family`, so stale entries keep contributing to the master gap plan. + +**Evidence:** +- `jobs/2026-03-17-ramp-AI_Operations_Specialist_Agentic_Workflows-f489c32b.md` has `role_family: agent-engineer` but `keyword_classify()` now correctly returns `out-of-scope` for that title. The JobNote was written before "operations specialist" was in the OUT list. Its skills still feed gap-plan weights. + +**Pattern:** counters (`appears_in_jobs`, `roles_seen`) are derived at every gap_aggregator run — Phase 0 bug #12 / Phase 1.A bug #1 enforced this discipline. But `role_family` is treated as ground truth once written. **Same asymmetry could repeat for any future "set-once classification" field** (e.g. `tier`, `seniority`). + +**Phase 2.A fix surface:** +- Either: `gap_aggregator.load_jobs()` re-runs `keyword_classify` on each JobNote and skips out-of-scope. +- Or: a one-time migration script that re-classifies and updates `role_family` in place (similar to the Phase 1.A `cleanup_stale_jobnotes.py` pattern). +- Either way: add a regression test that asserts "no JobNote in the vault has a `role_family` that current code would classify as `out-of-scope`." + +--- + +## B7. `gap_aggregator` includes `auto_rejected` / `timed_out` / `null hitl_decision` jobs at full weight + +**Symptom:** the gap plan is computed from all 23 JobNotes — currently 1 approved, 10 auto_rejected, 8 null (pre-1.B.1), 4 timed_out — with no `hitl_decision` filter. A `timed_out` job with score 4.0 contributes the same gap signal as an `approved` job. + +**Trade-off:** the original spec rationale was "JD-market signal is independent of personal action" — even rejected jobs reveal what skills the market wants. Defensible. But it conflates: +- "Skills the market demands generally" (all in-scope JDs) +- "Skills I should study to convert near-misses to applies" (high-score auto_rejected = stretch signals) +- "Skills the human is committed to pursuing" (approved only) + +**Phase 2.A fix surface:** +- Add an optional filter mode to `gap_aggregator.regenerate(filter_decision=None | "approved_or_above_threshold")`. +- Document the chosen weighting decision in `_profile/preferences.md` so the user can tune. + +--- + +## B8. `tailor_node` has no programmatic constraint against hallucination + +**Symptom:** unlike `score_node`'s `_constrain_to_jd_skills`, `tailor_node`'s prompt says "Mention real projects and concrete numbers when the profile provides them" but enforces nothing. The agent could invent project names and they'd ship to the JobNote. + +**Evidence:** the one tailored paragraph in the vault (Cognition Special Projects) was spot-checked — every concrete claim verified against `_profile/resume.md`. **No hallucination in this sample**, but n=1 with no code-level defense. + +**Risk:** Sonnet on a sparse JD ("Founder mindset, 2+ years exp") with a longer profile could easily invent specifics. + +**Phase 2.A fix surface:** +- Post-generation regex pass that flags proper-noun project mentions not present in `resume.md` (similar to extract's JD-substring validation in Phase 0). +- Eval coverage: hand-label 5 generated tailoring paragraphs against ground-truth resume facts; assert zero unsupported claims. + +--- + +## Portfolio-claim risks (operational, not code bugs) + +These are NOT data-quality bugs but portfolio narrative risks: the spec advertises features whose code is built but real data is empty. + +### PR1. Skill assessor loop is theoretical — zero evidence URIs + +`grep '^evidence:' ~/Documents/compass-vault/skills/*.md` returns 95/95 `evidence: []`. The "unique angle" — the agent that grades candidate skills against `learning-vault://` evidence — has never operated on real input. The MCP `assess_skills` tool returns immediately with no work. + +**The README narrative says:** "an agent inside it that grades my skills against the live job market and tells me what to study next." Currently it can't, because no evidence is wired up. The infrastructure exists; the demo isn't recorded. + +**Mitigation paths:** +- Manually wire 3-5 `learning-vault://` URIs to a sample of skills (e.g. MCP, LangGraph, RAG) — proves the loop end-to-end. +- Record one assessor run with grade-change output as a portfolio screenshot. + +### PR2. Application lifecycle has never been used + +`~/Documents/compass-vault/applications/` is empty. All 23 JobNotes have `applied_at: null`. The Dashboard panels "In-flight applications" and "Today's next actions" will both always be empty until someone runs `add_application(job_id)`. + +**Mitigation:** apply to one job for real, exercise the full lifecycle (`add_application` → `update_application_status` → next-action reminder). Provides a portfolio screenshot. + +### PR3. Concurrent pipeline runs not protected + +Two parallel `run_pipeline()` invocations share `HITL_CHECKPOINT_DB`. SQLite raises `database is locked` and one process may leave a thread without a saved checkpoint. Also two parallel `gap_aggregator.regenerate()` calls race on `MASTER_GAP_PLAN_PATH.write_text()` — atomic per call but last-writer-wins across processes. + +**Phase 1.B.3 fix surface:** Modal cron + human-MCP race is the real-world trigger. Use SQLite advisory lock or `flock` on a sentinel file. + +--- + +## What's working correctly (positive findings) + +These were spot-checked and are NOT a problem: + +- **`jd_summary`** extraction is faithful across 8 spot-checks. No hallucinated facts. ATS boilerplate correctly excluded. +- **`_constrain_to_jd_skills`** filter (Phase 0 fix #5) is intact. Would catch over-extraction; current bug class is under-extraction. +- **LLM-stage `intake_filter`** is well-calibrated. Catches Japanese pre-sales architect, fellowship roles, GTM partnerships, HR coordinator — all correct. No false positives observed. +- **Pre-RAG vs post-RAG (Phase 1.B.1 vs 1.B.2)** comparison shows no quality regression. Top-k=8 chunks produce score reasoning at comparable depth to the prior full-inventory inject. +- **Audit-trail integrity** (`hitl_decision`, `hitl_at`, `tailored_paragraph`, `score_threshold`) is correct across the 23 JobNotes inspected. + +--- + +## Why these are deferred to Phase 2.A + +The spec's Phase 2.A is the **eval harness** phase: 30+ hand-labeled JDs with ground-truth scores + skill lists, run nightly, alert on MAE drift. That's the right place to attack B1/B2/B3/B5 because: + +1. Each bug class needs a regression test, not a one-off prompt tweak. +2. Prompt tuning without measurement is whack-a-mole — fixing B1 might break B2 silently. +3. The labeled dataset doubles as the portfolio artifact ("eval-driven LLM development"). + +Phase 1.B.2 closed the architectural RAG portfolio claim. Phase 1.B.3 closes the cron+observability claim. Phase 2.A makes the LLM behavior measurably correct. + +--- + +## Stop-gap mitigation pre-2.A + +If a particular roll-out is needed before the eval harness: + +1. **B3 has the highest cost-to-value for a manual fix** — adding a one-line "verify candidate level ≥ 2 before declaring missing" assertion in the score prompt is low-risk. +2. **B2 has high impact when it fires** but is rare. Manual JD-by-JD review until 2.A is cheap (only ~20 jobs/week reach scoring). +3. **B1 affects every best-fit JD.** This is the single most consequential prompt-tuning target. + +**Recommendation:** do not attempt B1-B3 ad-hoc. Build 2.A first. diff --git a/docs/NEXT_SESSION.md b/docs/NEXT_SESSION.md new file mode 100644 index 0000000..a24811c --- /dev/null +++ b/docs/NEXT_SESSION.md @@ -0,0 +1,317 @@ +# Next Session Handoff — 2026-05-20 morning + +> You're a fresh agent picking this up. Read this top-to-bottom first, then +> follow Day-1-tomorrow. Everything you need to start in 90 seconds is here. + +--- + +## TL;DR + +**Branch:** `phase-1b2-rag` · **HEAD:** `7971630` · **Tests:** 411 passing · **Ruff:** clean +**Vault:** wiped clean for fresh refresh; companies/skills seeded from YAML +**Pipeline:** never run end-to-end; tomorrow is first real execution +**Today's first action:** the 10-minute resume + skill-inventory pass with the user, THEN run the refresh + +--- + +## What was done in the previous session (long, intense night) + +Started at the Day-1 Obsidian P1 work; ended 14 commits later with three full +adversarial review waves complete. Concretely: + +- **Days 1-2 of the sprint:** Obsidian P1/P2/P3 (JobNote wikilinks + SkillNote + backlinks + dashboard panels + auto-tags) ALL shipped +- **Vault wipe** — frontier-startup-only legacy JobNotes archived under + `jobs.archive-pre-pivot-2026-05-19/`; gap plan reset; ready for fresh data +- **Strategic re-tier in YAML** — `target-companies.yaml` rewritten: + - 49 auto-scrapable apply-now/opportunistic companies (Greenhouse/Ashby/Workday) + - 23 manual-add entries for banks/consulting (JPM/Capital One/GS/BofA/Deloitte/Accenture/etc.) + - 5 Workday tenants verified (Wells Fargo, Citi, Morgan Stanley, BlackRock, Adobe) + - 8 Austin/local startups added (Self Financial, AlertMedia, Diligent, Apptronik, Roboflow, Maven Clinic, Maven AGI, Vapi) + - Aliases on banks for tenant-slug ↔ name resolution (JPMC↔JPMorgan, BofA↔BankofAmerica, etc.) +- **Audit findings 1-8** all shipped (see commit `b13e1a2`) +- **Workday scraper** built (`compass/scrapers/workday.py`) — 5 confirmed + tenants scraping ~80 jobs/day +- **add_job_from_url + add_job_from_text MCP tools** — for JPM Oracle Cloud, + iCIMS, LinkedIn, anywhere Compass can't auto-scrape +- **Cover-letter generator** — Sonnet, 250-400 words, MCP tool +- **Score-node sees company tier + interview_difficulty** from YAML +- **Eval harness** (Phase 2.A) shipped — dataset.json + metrics + judge + + runner + scripts/label_jd.py interactive CLI +- **Three adversarial review waves** with 4 parallel agents each: + - Wave 1 (initial probe): 8 bugs, 8 fixed (URL dedup, agent-signal tiers, + path traversal in cover_letter, YAML cache mtime, aliases, URL scheme, + eval filename collision) + - Wave 2 (HiTL/RAG/pipeline/scrapers): 12 flagged, 7 fixed (HiTL race fix + via claim_pending atomic — the long-deferred Phase 1.B.3 spec item ships; + RAG stale chunks; Workday relative URL; swe-mobile/frontend not + upgrade-eligible; tailor score gate; judge skill normalization; + list_yaml_companies dedup; Ashby HTML fallback) + - Wave 3 (security/atomicity/cache/cold-start): 12 flagged, 9 fixed + (**learning_bridge path traversal — actively leaked .env contents before + fix**, get_profile path traversal, normalize(None) crash, + master-gap-plan + save_dataset atomic writes, add_job_from_text URL scheme, + seed_companies empty-company, taxonomy LRU cache invalidation, + get_model_id reads cfg) + - **One CRITICAL agent claim DEBUNKED** by direct probe: wave-3 agent said + "HiTL `__interrupt__` reads from wrong object — entirely broken" with + confidence 95. LangGraph actually DOES populate `__interrupt__` in + ainvoke return state. Current code works. **Always verify before fixing.** +- **Evidence URIs wired** for 4 SkillNotes (RAG / LangGraph / Eval_harness / + Pydantic_AI) pointing at new anchored sections in + `learning-vault/projects/compass/decisions.md`. Skill assessor ran; + proposed LangGraph: 1→3 (pending HiTL), RAG: 1→2 (auto-applied). +- **`HOW_COMPASS_WORKS.md`** written (441 lines) — the canonical + "explain Compass to me" doc. Read this before everything else. +- **State reset**: checkpoints.db purged, hitl.db purged, Chroma index + rebuilt against current skill-inventory, master-gap-plan regenerated empty. + +**Total: 259 → 411 tests passing (+152 tests). 24 real bugs fixed across 3 waves.** + +--- + +## The single thing NOT done that blocks tomorrow + +**The user has not done the resume + skill-inventory pass yet.** + +They committed last night to doing it "tomorrow morning before the run." +Until they do, the score node will miscalibrate every JD because the +candidate-profile context is partially stale. + +### What needs to happen in the 10-minute resume + inventory pass + +User-facing edits to two files in `~/Documents/compass-vault/_profile/`: + +1. **`resume.md`** — currently lists Technical Skills with Languages first, + Agent frameworks third. Reorder so **Agent frameworks / MCP come first**: + every JD they're targeting reads "LangGraph / MCP / agentic AI" as the + primary keyword line. Free ATS-keyword + recruiter-scan improvement. + +2. **`skill-inventory.md`** — confirm the Cisco scope description matches + `role-clarifications.md` (test development engineer, not security). + Verify the MCP cluster at level 4 reflects the production Cisco work + + Minx 4 servers. Consider bumping LangGraph 1→3 to match the just-applied + assessor proposal (user can also approve via MCP). + +This is YOUR job tomorrow morning to walk them through. Don't run the +refresh until this is done — it directly affects every score. + +--- + +## Pre-flight check (run before anything) + +```bash +cd /Users/akmini/Documents/compass +git status # expect: clean on phase-1b2-rag +git log --oneline -3 # confirm HEAD = 7971630 +uv run pytest -q 2>&1 | tail -3 # expect: 411 passed +uv run ruff check # expect: All checks passed +ls ~/Documents/compass-vault/jobs/ | wc -l # expect: 0 (empty, ready for refresh) +ls ~/Documents/compass-vault/companies/ | wc -l # expect: 76 +ls ~/Documents/compass-vault/skills/ | wc -l # expect: 98 +ls ~/.compass/checkpoints.db 2>&1 # expect: absent (will be created on first run) +du -sh ~/.compass/chroma # expect: ~1.0M (just rebuilt) +``` + +If anything is red, **STOP** and diagnose before proceeding. + +--- + +## Tomorrow's plan (in order, ~3 hours to first applications) + +| Time | Step | What to do | +|---|---|---| +| 0:00 | **Pre-flight check** (above) | Verify the state | +| 0:05 | **Resume + skill-inventory pass** | Walk user through the two edits above | +| 0:15 | **First refresh** | `uv run python -m compass.pipeline.graph` — expect ~$0.50 LLM cost, 50 jobs through pipeline, ~25 surviving intake_filter, ~6 paused for HiTL, ~19 written with auto_rejected status | +| 0:35 | **Inspect vault state** | Open Obsidian. Verify the 5 things in HOW_COMPASS_WORKS.md section 2: graph view edges, wikilinks resolve, SkillNote backlinks populate, tag pane has #tier/#fit/#signal tags, dashboard panels populate | +| 0:45 | **Eval --judge baseline** | `uv run python -m compass.evals.runner --judge --limit 20` — costs ~$0.04, produces first measurement of extract recall + score MAE/bias | +| 0:55 | **Inspect eval results** | `cat compass/evals/results-judge-*.json \| jq '.metrics, .per_record[0:3]'` — look for extract_skill_recall and per-record missed_skills. If recall < 60% the B1 bug is real and needs the Phase 2.A prompt-tuning loop (Day 9 work). If recall > 80% you're good to apply. | +| 1:00 | **Approve HiTL-paused jobs** | Each paused thread_id needs an MCP `approve_job(thread_id, decision)` call. Currently no MCP tool for this — uses `compass.hitl.resume.resume_pending` programmatically. If you find this is awkward, that's the signal to ship an `approve_job` MCP tool. | +| 1:30 | **Label 10 JobNotes** | `uv run python -m scripts.label_jd <filename>` for the 10 strongest-fit JobNotes — the agent's extract output displays, user provides expected_score (their gut read) + expected_skills (keep agent's or override). One-keystroke labeling for cases where the agent got it right. | +| 2:15 | **Rigorous eval baseline** | `uv run python -m compass.evals.runner --labels` — now you have measured numbers vs hand-labels. Record in `docs/EVAL_BASELINE.md` | +| 2:30 | **First applications** | Generate cover letters for 3 strongest fits via `generate_cover_letter(filename)` MCP tool. Apply via company portals. Track in `applications/` via `add_application` MCP tool (codepath untested in production — first real use). Cisco internal first (highest EV, no LC). | +| 3:00 | **Done for day 1** | First measurement + first applications + first real Compass run on disk | + +--- + +## What's UNDER YOUR THUMB tomorrow + +These are the things the user needs YOU to do, not them: + +- **Walk them through resume + inventory pass.** Don't just say "do it" — sit + with the file open, show diffs, suggest reorderings. They've deferred this + 3 times. +- **Run the refresh together.** Watch the output stream. Note which boards + rate-limit (Workday will probably 429 once or twice). Note which JDs hit + the agent-signal gate (logged to `_meta/filtered-jobs.md`). +- **Inspect the FIRST JobNote together.** The wave-1/2/3 fixes are unverified + against real data. Open one JobNote in Obsidian. Walk through: + - `## Skills` block has wikilinks? + - `[[Python]]` resolves to `skills/Python.md`? + - Tags include `#tier/...`, `#fit/...`, `#signal/agent-strong`? + - `## Full JD` body has no HTML cruft (Greenhouse leak)? + - JobNote.tier matches the company's YAML tier? +- **If the first refresh blows up,** read the logs at + `_meta/agent-log.md` + `_meta/pipeline-runs.md` + `_meta/filtered-jobs.md` + before changing code. Most failures will be data-driven, not code bugs. + +--- + +## What to NOT do tomorrow + +| Don't | Why | +|---|---| +| Add new features before the first refresh runs | Three waves of adversarial review already; remaining bugs are runtime-discovery class. Code review can't catch them. | +| Build Modal cron deployment | Manual runs work fine; defer until you've verified daily-cron makes sense at this volume | +| Build full-resume-rewrite-per-JD tool | Cover letter + tailored_paragraph cover the practical use case. Defer until you see whether recruiters respond to current outputs. | +| Tune extract prompt before measurement | Phase 2.A loop: eval baseline → identify B1 patterns → prompt tweak. NOT prompt-tweak-then-measure. | +| Add more Workday tenants speculatively | First refresh will reveal which tenants are reliable. Add more after seeing day-1 yield. | +| Re-run a wave-4 adversarial review | Diminishing returns curve hit hard at wave 3 (false-alarm rate rising). Marginal value < running Compass. | + +--- + +## Communication style (the user's preferences, verbatim) + +From their CLAUDE.md: + +- **Be direct and concise.** Don't restate what they said. +- **Lead with the answer or action.** Not the reasoning. +- **Don't add comments / docstrings / type annotations to code you didn't change.** +- **Don't create files unless necessary.** Prefer editing existing ones. +- **When you save to the vault: tell them path + why.** +- Want **honest harsh assessments when asked**, real adversarial reviews + that find real bugs, direct recommendations. +- Don't want long menus of options when one is obviously right, excessive + caveats, AI-slop code, apologetic preambles. + +What they DON'T want from you: +- Restating the plan back to them +- Asking permission for things this doc already authorized +- More features before the refresh runs + +Pattern they've used several times: +- "Be harsh and honest." → They mean it. Don't soften the assessment. +- "Should we do X?" — when they ask this, they want a YES/NO with one + paragraph of reasoning, not a 5-option menu. + +--- + +## Known data-quality issues to watch for in the first refresh + +These are documented in `docs/KNOWN_DATA_QUALITY_ISSUES.md`. The most likely +to surface tomorrow: + +1. **B1 — extract under-extracts on best-fit JDs** (Sierra, Decagon, etc.). + Symptom: a JobNote for "Software Engineer, Agents" at Sierra shows only + 3 skills in the `## Skills` block when the JD body listed 12. Fix path: + eval baseline → see the per-record `missed_skills` set → prompt-tune + extract_node. Day 9 work in the sprint plan. + +2. **Workday rate-limit during detail fetch** — 50 parallel detail + requests per page. May 429 on Citi (highest volume = 1470 jobs in their + tenant). If you see "workday detail timeout/429" in logs, drop + `_PAGE_SIZE` from 50 to 20 in `compass/scrapers/workday.py`. + +3. **Score may over-rank ML/Vision** at banks — the RAG retriever ranks + "ML/Vision (kept here for reference)" highly for Capital One-style ML + JDs because the keywords match. The section is marked "not in JD-keyword + spine" but the embedding doesn't know that. Watch for inflated Capital + One scores. Fix path: demote reference-only sections during retrieval + (~30 min). Do AFTER measurement. + +4. **Anthropic/Sierra/Decagon may dominate vault attention** even though + they're `opportunistic` tier. Their JDs survive intake_filter (strong + agent signal). Just monitor — if the dashboard "apply-now top 5" is + filled with frontier startups, the apply-now anchors (banks/consulting/ + Datadog) aren't surfacing enough. Fix by reviewing the YAML + `apply-now` vs `opportunistic` assignments. + +5. **Adobe JDs may be too broad** — Adobe is a 10k+ employee company + scraped via Workday. The scraper will pull every open role across the + company. Expect ~50 Adobe JDs in the raw scrape, most filtered out by + role_family but the few survivors may not all be agent-eng. Watch + the role_family classifications. + +--- + +## Cost expectations for tomorrow + +Approximate, OpenRouter pricing: + +| Activity | Cost | +|---|---| +| First refresh (50 jobs through extract+score) | ~$0.04 | +| Tailor calls on ~6 approved jobs (Sonnet) | ~$0.30 | +| `--judge --limit 20` eval baseline | ~$0.04 | +| `--labels` eval after 10 hand-labels | ~$0.04 | +| 3 cover letters (Sonnet) | ~$0.24 | +| **Total day 1** | **~$0.70** | + +Modal compute: $0 (not deployed). + +--- + +## Key docs in priority reading order + +1. **This doc** (you just read it) — start state + plan +2. **`docs/HOW_COMPASS_WORKS.md`** (441 lines) — what Compass is, current architecture +3. **`/Users/akmini/.claude/CLAUDE.md`** (user's global preferences) +4. **`/Users/akmini/Documents/compass/CLAUDE.md`** (repo conventions) +5. **`docs/KNOWN_DATA_QUALITY_ISSUES.md`** — deferred bugs with severity + fix path +6. **`docs/TWO_WEEK_SPRINT.md`** — the previous sprint plan (mostly executed in last session, but useful for sequencing context) +7. **`compass-vault/_profile/target-companies.yaml`** — the targeting source of truth (96 companies tracked) +8. **`compass-vault/_profile/target-roles.md`** — JD master boolean + title decoder + +--- + +## Things deferred (NOT priorities — don't start these tomorrow) + +| Deferred | Why it's deferred | +|---|---| +| Modal cron deploy | Wait until daily-cron makes sense at observed volume | +| Cisco internal tracker | User getting info from former boss; manual checklist for now | +| `claim_pending` callers fully exercised | The atomic-claim infrastructure ships but resume.py is the only caller; MCP `approve_job` tool not yet wired (tomorrow's HiTL approval needs this OR direct programmatic resume) | +| Chunk-per-skill RAG | Current per-category chunking works for realistic queries. Defer until eval shows it's a bottleneck. | +| Strip markdown table noise from chunks | Same — defer until measurement | +| README rewrite + Mermaid diagram | Phase 2.B portfolio polish | +| Public Langfuse trace URL in README | Same | +| Blog post | Phase 2.C | +| Cover-letter A/B variants | Defer until you see whether the v1 cover letter actually lands screens | +| Full-resume-rewrite-per-JD | Same — defer until cover letter signal is in | + +--- + +## The single most likely scenario for tomorrow's session + +User opens Claude Code, says something like "let's run compass" or "let's go +over my resume." You: + +1. **Read this doc + HOW_COMPASS_WORKS.md** (5 min) +2. **Run the pre-flight check** (1 min) +3. **Suggest the resume + skill-inventory pass FIRST** — open both files, + walk through the reorderings together +4. **Run `uv run python -m compass.pipeline.graph`** — watch the output +5. **Open Obsidian** — verify the 5 things in HOW_COMPASS_WORKS.md +6. **Run the judge eval** — produce the first measurement +7. **Pick the strongest 3 fits** — generate cover letters, apply + +If something blows up at step 4, **read the logs first** (`_meta/agent-log.md`, +`_meta/pipeline-runs.md`, `_meta/filtered-jobs.md`) before changing code. +The data-driven failure modes are most likely class. + +--- + +## Closing reality check + +The code is in the best shape it's been in. 411 tests, 3 adversarial waves, +zero security issues remaining. The bottleneck has fully moved from code +quality to RUNTIME EXECUTION + USER ACTION. + +The user's 2-month timeline is real. Every day spent reviewing code instead +of running Compass + applying to jobs is a day of opportunity cost. Tomorrow +is the day Compass gets used. + +Don't engineer. Don't review. **Run the refresh. Apply to jobs.** + +Go. diff --git a/docs/OBSIDIAN_LEVERAGE.md b/docs/OBSIDIAN_LEVERAGE.md new file mode 100644 index 0000000..27a9649 --- /dev/null +++ b/docs/OBSIDIAN_LEVERAGE.md @@ -0,0 +1,347 @@ +# Obsidian Leverage — Design Proposals + +> The vault is currently a markdown store with YAML frontmatter. Obsidian renders it but the only Obsidian-specific feature we use is Dataview on the dashboard. This doc maps the unused capabilities — wikilinks, graph view, embeds, Bases, Canvas, tags — to concrete project-relevant uses. Pick what to build; the proposals are sized. + +**Current state:** 23 JobNotes, 95 SkillNotes, 9 CompanyNotes, 0 ApplicationNotes. Skills appear in JobNote frontmatter as plain string lists (`skills_required: [Python, LangGraph]`). No bidirectional navigation. Graph view is empty. + +--- + +## P1. Bidirectional skill ↔ job wikilinks (your specific ask) + +**Goal:** click "Python" in a JobNote → see every job requiring Python. Click any skill in graph view → see the constellation of jobs that demand it. Find rare skills (skills appearing in 1-2 jobs) vs commodity ones (appearing in 10+). + +### How Obsidian wikilinks work + +`[[Python]]` in any markdown file: +- Renders as a clickable link +- Creates a backlink on the target file (`skills/Python.md` shows "Linked mentions" of the source) +- Appears as an edge in the graph view + +Obsidian resolves `[[Python]]` to any file named `Python.md` in the vault. Since `compass-vault/skills/Python.md` exists, the link works. (No `skills/` prefix needed unless there's an ambiguous filename collision.) + +### Proposal — non-invasive + +Keep the typed frontmatter (`skills_required: [Python, LangGraph]`) as the source of truth — Dataview queries and Python validation both depend on it. ADD a new section to the JobNote body that renders as wikilinks for human navigation + graph view. + +**Change to `compass/vault/writer.py:write_job_note`** — when writing the JobNote body, prepend a section like: + +```markdown +## Skills + +**Required:** [[Python]] · [[LangGraph]] · [[MCP]] +**Nice to have:** [[FastAPI]] +**Matched:** [[Python]] · [[MCP]] +**Missing:** [[LangGraph]] · [[Sub-agents]] + +--- +``` + +(One line per category. Bullet/list works too — `- [[Python]]` per line — but inline `·`-separated is denser and reads well.) + +**Change to gap_aggregator's `_sync_skill_counters`** (which already updates SkillNote frontmatter): also write a "Jobs Requiring This Skill" section to the SkillNote body. Two options: + +**Option A — static list, rewritten each run** (simpler, no plugin dependency): +```markdown +## Jobs requiring this skill + +- [[2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f|Sierra — Software Engineer, Agent Architecture]] · score 3.0 · apply-now +- [[2026-05-06-cognition-Software_Engineer-7c26fdaf|Cognition — Software Engineer]] · score 3.0 · apply-now +- [[2025-04-18-sierra-Software_Engineer_Agent-8f58539b|Sierra — Software Engineer, Agent]] · score 4.5 · apply-now +``` + +Sortable by score; renders as clickable links. Updated atomically on each gap-aggregator regen. + +**Option B — Dataview query block** (more powerful, requires Dataview plugin which is already used): +````markdown +## Jobs requiring this skill + +```dataview +TABLE company, match_score AS Score, tier +FROM "jobs" +WHERE econtains(skills_required, this.file.name) +SORT match_score DESC +``` +```` + +Always fresh, sortable in the UI, no per-job rewriting. + +**Recommendation: Option A first, Option B if it gets stale.** Option A is more robust because it works even if Dataview is disabled or breaks. Option B is fancier but couples to a plugin. + +### Effort + +- Writer body-section change: **30 min** (one new function, wire into `write_job_note`) +- SkillNote body update: **30 min** (extend `_sync_skill_counters` to write the Jobs section) +- Tests: 2 small unit tests +- One-time backfill: write a script that walks existing JobNotes and re-renders body sections (similar to the B6 migration script) +- **Total: ~2 hours** + +### What you get + +- Click `[[Python]]` in any JobNote → land on `skills/Python.md` → see every JobNote requiring it, ranked by score +- Graph view (Obsidian sidebar → "Open graph view") shows clusters. Skills appearing in many JobNotes become hub nodes. Rare skills are peripheral. +- Visual answer to your specific question — *"which skills are required across many jobs and which are rare/specific"* + +--- + +## P2. Skill rarity dashboard panel + +**Goal:** at a glance, see which skills are commodity (every JD wants them) vs differentiated (rare, niche, harder to source). + +**Implementation:** add a Dataview block to `compass-vault/dashboard.md`: + +````markdown +## Skill rarity — which gaps to prioritize + +```dataview +TABLE WITHOUT ID + file.link AS Skill, + appears_in_jobs AS "in jobs", + my_level AS "my level", + gap_score AS "gap score" +FROM "skills" +WHERE appears_in_jobs > 0 +SORT appears_in_jobs DESC, gap_score DESC +LIMIT 30 +``` + +## Rare/specialized skills (appears in 1-2 jobs only) + +```dataview +TABLE WITHOUT ID + file.link AS Skill, + appears_in_jobs AS "in jobs" +FROM "skills" +WHERE appears_in_jobs > 0 AND appears_in_jobs <= 2 +SORT appears_in_jobs ASC +``` +```` + +### Effort + +- **5 min** to add to dashboard.md +- No code changes + +### What you get + +- Top panel: skills sorted by demand frequency. The "commodity" skills (Python, Docker) at the top. Investing in them helps for many jobs. +- Second panel: skills appearing in 1-2 jobs only. These are differentiators — if you don't have them, only specific roles surface. If you DO have them, you can target those niche roles confidently. + +--- + +## P3. Auto-tags on JobNotes for filter dimensions + +**Goal:** flexible filtering via Obsidian's tag system. Find all "stretch" jobs across companies, all "apply-now-fit" jobs, all jobs requiring TypeScript, etc. + +**Current state:** JobNote has `tags: list[str]` in frontmatter but nothing populates it. + +**Proposal:** `vault_write_node` auto-generates tags based on JobNote fields: + +```python +tags = [] +# tier-derived +if tier == "apply-now": tags.append("#tier/apply-now") +elif tier == "6-month": tags.append("#tier/6mo") +elif tier == "stretch": tags.append("#tier/stretch") +# fit-derived +if match_score >= 4.0: tags.append("#fit/strong") +elif match_score >= 3.0: tags.append("#fit/decent") +elif match_score >= 2.0: tags.append("#fit/stretch") +else: tags.append("#fit/weak") +# role-family-derived +if role_family: tags.append(f"#role/{role_family}") +# hitl-decision-derived +if hitl_decision: tags.append(f"#decision/{hitl_decision}") +``` + +Use Obsidian's tag pane to filter by any combination. + +### Effort + +- **20 min** in `vault_write_node` + a one-time migration to apply tags to existing JobNotes +- 1 unit test verifying tag generation + +### What you get + +- Click any tag in Obsidian's tag pane → see every JobNote with that tag +- Compose tags: `#fit/strong AND #role/agent-engineer` to find strong-fit agent roles +- Tag-based Dataview queries become natural ("show all #decision/timed_out with score > 3.5 — these are the ones the auto-reject lost") + +--- + +## P4. SkillNote embeds candidate level into JobNote view + +**Goal:** when reading a JobNote, immediately see your level on each required skill without flipping pages. + +**How embeds work:** `![[Python#current-level]]` inlines the content under the "Current level" heading from `skills/Python.md`. Live-rendered in Obsidian. + +**Proposal:** in the JobNote body's `## Skills` section: + +```markdown +## Skills with candidate level + +| Skill | Required? | My level | +|---|---|---| +| Python | required | ![[Python#level]] | +| LangGraph | required | ![[LangGraph#level]] | +| MCP | required | ![[MCP#level]] | +``` + +This requires each SkillNote to have a `## Level` heading with the level number visible (e.g. `Level 4 (production)`). The seed script can be extended. + +**Alternative — Dataview-only**: + +````markdown +```dataview +TABLE WITHOUT ID skill AS Skill, my_level AS "My level" +FROM "skills" +WHERE contains(this.skills_required, skill) +``` +```` + +### Effort + +- Embeds path: requires modifying SkillNote body to have a `## Level` section (15 min seed + body-rendering change in writer) +- Dataview path: 5 min, no SkillNote changes +- **Total: 10-30 min depending on approach** + +### What you get + +- Open a JobNote → see "Python: 3, LangGraph: 1, MCP: 4" inline. No clicking. +- Quick scan of your gaps for any role. + +--- + +## P5. Canvas view of the gap plan + +**Goal:** visualize the top-N gaps as a graph showing which jobs each gap would unlock. + +**Obsidian Canvas** is a node-and-arrow visual editor. You drop notes onto a canvas; Obsidian renders them as cards. + +**Proposal:** a script that builds `study-plans/gap-plan.canvas` JSON from the master gap plan + JobNote relationships. Each top-10 gap becomes a center node; arrows fan out to the JobNotes that would benefit. Edge thickness = how many jobs depend on that skill. + +**Effort: 1 hour** for a simple generator. The Canvas JSON format is documented and stable. + +### What you get + +A visual map of "which skills should I learn next, ordered by how many doors they open." More motivating than a ranked list. + +This is portfolio-quality polish — recruiters love screenshots of this kind of thing. + +--- + +## P6. Bases (newer Obsidian feature) for table views + +**Goal:** SQL-like filtered views of JobNotes/ApplicationNotes without writing Dataview queries each time. + +**What Bases are:** `.base` files in Obsidian that define views over frontmatter. Like saved-query database views. Newer feature (released 2025). + +**Proposal:** ship 3 `.base` files in the vault: + +- `applications.base` — all ApplicationNotes, filterable by status/next_action_date +- `jobs-strong-fit.base` — JobNotes with match_score >= 4.0, sorted by date +- `skills-by-rarity.base` — SkillNotes sorted by appears_in_jobs ascending (rare first) + +### Effort + +- **30 min** to design + author 3 .base files +- No code changes +- Requires the user to be on Obsidian 1.7+ (Bases shipped 2025) + +### What you get + +Database-style table views that update automatically. More performant than Dataview for large vaults; first-class UI. + +--- + +## P7. Daily-note linking for organic evidence URIs + +**Goal:** close the loop between "I worked on a skill today" and "skill_assessor sees evidence of work." + +**Current state:** the candidate has `learning-vault/daily/YYYY-MM-DD.md` files but they're not linked to skills. + +**Proposal:** establish a convention in daily notes: + +```markdown +# 2026-05-19 + +Worked on Compass Phase 1.B.2: RAG via Chroma. Shipped the indexer + +retriever. Wrote a [[LangGraph#interrupt and resume]] integration that +checkpoints to SQLite. Tested HITL flow end-to-end. + +Skills practiced: [[LangGraph]] [[Chroma]] [[RAG]] [[MCP]] +``` + +When skill_assessor reads `learning-vault://daily/2026-05-19.md` as evidence for `LangGraph`, the skeptical grader sees: real work, specific artifact, last_modified today. Raises the grade. + +**Implementation:** no code change required. Pure convention. Optional: a small `compass/scripts/sync_daily_to_skills.py` that scans daily notes for `[[SkillName]]` patterns and auto-adds the daily-note URI to each referenced SkillNote's evidence list. **20 min.** + +### What you get + +Daily writing → automatic skill evidence → grade updates on next assessor run. The "build → write → regrade" loop the spec promised, with zero friction. + +--- + +## P8. Tagging interview-stage applications + +**Goal:** when an ApplicationNote moves through stages (applied → screen → onsite → offer), tag transitions show up in Obsidian's tag pane. + +**Implementation:** `compass/applications/lifecycle.py:update_application_status` already updates the `status` frontmatter. Extend it to also append a stage-transition tag to the ApplicationNote body: + +```markdown +## Stage transitions +- 2026-05-19 → #stage/applied +- 2026-05-22 → #stage/screen +- 2026-05-29 → #stage/onsite +``` + +Each tag becomes a backlink target. Click `#stage/onsite` in the tag pane → see all applications in onsite stage. + +### Effort + +- **20 min** in `lifecycle.py` +- The user has zero ApplicationNotes today so no backfill needed + +### What you get + +Visual interview-pipeline tracking in Obsidian. Combined with the existing `list_pending_actions` MCP tool, you get both temporal (next_action_date) and stage-based filtering. + +--- + +## Recommended bundle + +If you want a single focused project to ship next that delivers the biggest visual + workflow win: + +**Ship P1 + P2 + P3 together** as a small "Obsidian leverage" patch on top of `phase-1b2-rag`: + +1. **P1**: JobNote body gets `## Skills` section with `[[skill]]` wikilinks. SkillNote body gets `## Jobs requiring this skill` list. +2. **P2**: dashboard.md gets "Skill rarity" + "Rare/specialized skills" Dataview panels. +3. **P3**: JobNote frontmatter gets auto-generated tags from tier / score / role_family / hitl_decision. + +**Total effort: ~3 hours** including the one-time migration scripts to backfill existing JobNotes + SkillNotes. + +**Visible result:** +- Obsidian graph view becomes meaningful — clusters of jobs around hub skills, peripheral rare skills visible at a glance +- Click any skill in a JobNote → instantly see every related job +- Tag pane gives you `#fit/strong`, `#role/agent-engineer`, etc. for one-click filters +- Dashboard answers your exact question (which skills are commodity vs rare) at the top + +This is also strong portfolio material — Obsidian-as-product-UI for an agentic career coach is the kind of recruiter-facing screenshot that lands. + +--- + +## Things I'd defer + +- **P5 Canvas**: pretty but complex to maintain. Wait until daily use surfaces a need. +- **P6 Bases**: requires Obsidian 1.7+ and is still maturing. Try in a side branch first. +- **P7 daily-note linking**: depends on the user actually using daily notes consistently. Currently 3 daily notes exist (May 17-19). If you build the habit first, the tool follows. + +--- + +## Where this fits in the broader phase plan + +This is a **vault-rendering layer** — doesn't touch the pipeline, LLM, or HITL infrastructure. It's pure additive Obsidian polish. The cleanest place for it: + +- **Phase 1.B.2.x patch** (this branch, before tagging 1.B.3) +- OR **Phase 1.B.3** as a side-quest after Modal cron + Langfuse fix lands + +Recommendation: ship the **P1+P2+P3 bundle** as a small side-quest immediately. It's high-ROI, low-risk, and gives you a daily-use UI improvement before the 1.B.3 work starts (which is mostly invisible plumbing). diff --git a/docs/OBSIDIAN_LEVERAGE_PART_2.md b/docs/OBSIDIAN_LEVERAGE_PART_2.md new file mode 100644 index 0000000..1b09cf9 --- /dev/null +++ b/docs/OBSIDIAN_LEVERAGE_PART_2.md @@ -0,0 +1,728 @@ +# Obsidian Leverage — Part 2: Use-Case Oriented Ideas + +> Part 1 (`OBSIDIAN_LEVERAGE.md`) covered the foundational graph/link layer (P1-P8). This doc goes deeper — organized around the actual problems you face during a tier-2 agentic-AI job search, with Obsidian features mapped to each. Pick what helps; ignore what doesn't. + +--- + +## Problem 1: "What should I study TODAY?" + +You have a master gap plan, but it's a ranked list. You need a daily-flow recommendation that connects "what I worked on yesterday" with "what to do today." + +### Idea 1.1: Daily note → skill evidence auto-pipeline + +You already have `learning-vault/daily/YYYY-MM-DD.md`. Establish a convention: when you log work in a daily note, link the skill. Example: + +```markdown +# 2026-05-19 + +Today I built the Chroma RAG layer for Compass. Decided to use cosine +distance instead of L2 after testing both. Empirically verified the +`with_msgpack_allowlist` API is a no-op when the default is True. + +Worked on: [[LangGraph]] [[Chroma]] [[RAG]] [[MCP_server_authoring]] +``` + +**Automation:** a tiny script `compass/rag/sync_daily_evidence.py` scans `learning-vault/daily/*.md` for `[[SkillName]]` patterns and auto-appends the daily-note URI to each referenced SkillNote's `evidence:` list. Run weekly via Modal cron (Phase 1.B.3). + +**What you get:** daily writing → skill evidence URIs grow → next `assess_skills` run reflects the work. Closes the "build → write → regrade" loop the spec promised. + +**Effort:** ~30 min for the script. Behavioral cost: you have to actually write daily notes. + +### Idea 1.2: Heatmap-calendar plugin (community) + +Install `obsidian-heatmap-calendar`. Render a GitHub-contributions-style grid showing daily skill activity. Days with skills worked on glow green; idle days are gray. + +**Where:** a callout block on `dashboard.md`: + +````markdown +```heatmap-calendar +year: 2026 +colors: greens +entries: + # dataview-derived count of distinct skills worked on per day +``` +```` + +**What you get:** visual reinforcement loop. Bad week → see the gray patch → fix it. Good streak → see momentum. + +### Idea 1.3: "Today's study target" — a dynamic dashboard widget + +Dataview block on `dashboard.md` that computes: "the top-3 gaps from master-gap-plan, intersected with skills the user hasn't touched in the last 7 days." + +````markdown +```dataview +TABLE WITHOUT ID + file.link AS Skill, + gap_score AS Gap, + last_assessed AS "Last touched" +FROM "skills" +WHERE gap_score > 0.5 + AND (last_assessed = null OR date(last_assessed) < date(today) - dur(7d)) +SORT gap_score DESC +LIMIT 3 +``` +```` + +**What you get:** opens dashboard → sees "your top neglected high-gap skills are X, Y, Z." No decision fatigue. + +--- + +## Problem 2: "I have an interview Friday — what do I prepare?" + +You get a recruiter screen for Sierra Agent Engineer. You need to know which of your skills + stories to lead with, which gaps to acknowledge, which to deflect. + +### Idea 2.1: STAR story bank linked to skills + +Create `compass-vault/interview-prep/stories/*.md` — one note per STAR story (Situation, Task, Action, Result). Each story links to the skills it demonstrates: + +```markdown +# Cisco MCP Servers — 4-server architecture + +## Situation +Cisco needed natural-language access to their internal observability stack... + +## Task +Build MCP servers for [[Splunk]], [[Grafana]], [[Datadog]], [[ServiceNow]]... + +## Action +Designed a 4-server pattern with shared auth + tool registry. Used +[[FastMCP]] and [[Pydantic]] for typed surfaces. [[Python]] backend... + +## Result +Adopted across the SRE team. Reduced incident triage time by ~40%. +Talk: [[talks/cisco-mcp-internal-talk]] + +## Skills demonstrated +[[MCP]] [[MCP_server_authoring]] [[Python]] [[Multi-Agent]] [[Production_Concerns]] +``` + +**What you get:** when a JobNote requires `[[MCP]]`, Obsidian's backlinks panel on the MCP SkillNote shows every story you have for it. Or vice versa — open the JobNote, see which skills are required, click each one to find your relevant stories. + +### Idea 2.2: Interview-prep playbook auto-assembled per JobNote + +A Dataview query block embedded in EVERY JobNote (via writer template): + +````markdown +## Stories I have for this role's requirements + +```dataview +LIST +FROM "interview-prep/stories" +WHERE any(skills_demonstrated, (s) => contains(this.skills_required, s)) +SORT file.name DESC +``` + +## Skill gaps to acknowledge + +`= filter(this.skills_required, (s) => link("skills/" + s).my_level < 2)` +```` + +**What you get:** open any JobNote → instantly see which stories to lead with + which weaknesses to prepare a response for. Zero manual lookup. + +### Idea 2.3: Post-interview retro that closes the loop + +After every interview, a retro note in `learning-vault/interview-prep/postmortems/`: + +```markdown +# 2026-05-25 — Sierra Agent Engineer screen + +Result: passed → onsite + +## What worked +- Led with Cisco MCP story for the agent-orchestration question +- "Show me a hard debugging session" → described the C1 audit-trail + divergence we fixed in Compass 1.B.1 + +## What missed +- They asked about [[Evals]] specifically — I conceptually know it but + no shipped artifact. Acknowledged as a current gap. +- [[Modal]] cron experience — only theoretical right now (1.B.3 work + hasn't shipped). Pivoted to general k8s cron experience. + +## Linked +[[applications/2026-05-19-sierra-Software_Engineer_Agent_Architecture]] +``` + +**Automation hook:** add `learning-vault://interview-prep/postmortems/*.md` to every relevant SkillNote's `evidence:` on save. Skills you successfully described in interviews accumulate proof. Skills that got flagged as gaps move up the study priority. + +### Idea 2.4: "Difficult question bank" tagged by skill + +A folder `interview-prep/questions/` with notes per question type. Tag each with the skills it tests: + +```markdown +# How would you build an agentic system for X? + +Tags: #q/system-design #skill/agent-frameworks #skill/multi-agent + +## My current answer outline +1. Clarify constraints: latency, error tolerance, observability +2. Sketch [[LangGraph]] state machine + checkpointing +3. Tool layer via [[MCP]] +4. Eval harness via [[LangSmith]] or [[Langfuse]] +``` + +**What you get:** before any interview, search the tag pane for `#skill/agent-frameworks` → see every question you've prepped + your current answer. Refine answers over time as artifacts grow. + +--- + +## Problem 3: "I keep missing good jobs in the noise" + +The pipeline scrapes ~20 jobs/day. You can't scan all of them. The dashboard's "Apply now (top 5)" panel helps but you're missing context. + +### Idea 3.1: Quick-capture inbox + +`compass-vault/_inbox/` folder. When you see a JD on LinkedIn / X / Slack, paste it into `_inbox/2026-05-19-some-company.md`. The pipeline gains an `inbox_node` (or `_scrape_all` is extended) that picks up `_inbox/*.md` files and runs them through the normal pipeline. + +**What you get:** zero-friction capture. Don't lose interesting jobs to "I'll save the link in a Slack DM and forget." + +**Effort:** ~1 hour to add an inbox scraper. Same RawJob shape as the ATS scrapers; just a different source. + +### Idea 3.2: Custom "this week's best fits" Dataview + +The dashboard has "Apply now (top 5)" but it's score-sorted from all-time. Add a "best fit found this week" panel: + +````markdown +```dataview +TABLE WITHOUT ID + file.link AS Job, + company, + match_score AS Score, + tier +FROM "jobs" +WHERE date_found >= date(today) - dur(7d) + AND match_score >= 3.0 + AND tier != "skip" +SORT match_score DESC +LIMIT 10 +``` +```` + +**What you get:** Monday morning, open dashboard → "10 fresh roles worth a look this week." + +### Idea 3.3: Anti-pattern note → intake_filter feedback + +You keep getting AE/PM/management roles through the filter (or marginally past it). Maintain a personal anti-pattern note: + +```markdown +# _profile/anti-patterns.md + +## Roles I will NEVER apply to (auto-reject) +- Management track (Engineering Manager, Lead, Director, VP) +- Pre-sales / Solutions Engineer / Sales Engineer +- Customer Success / Solutions Architect (sales-track) +- Pure DevRel / Developer Advocate +- Internships + +## Roles I'm UNDECIDED on (route to manual review) +- Research-only (no shipped systems): scoring lower than 3.0 is fine +- Customer-facing FDE roles: prefer SF/NYC, decline remote-only +``` + +`intake_filter._llm_classify` reads this note as context, biasing its classifications. + +**What you get:** the more you reject a class of role, the more aggressively the filter learns to drop it. You teach the system over time. + +### Idea 3.4: Geographic + salary filter dimensions + +JobNote already has `location` + `salary_min` + `salary_max`. Add Dataview panels on the dashboard: + +````markdown +## NYC roles (apply-now tier) +```dataview +LIST WITHOUT ID file.link +FROM "jobs" +WHERE tier = "apply-now" AND (contains(location, "NYC") OR contains(location, "New York")) +SORT match_score DESC +``` + +## Above-target comp +```dataview +TABLE company, salary_min, salary_max, match_score +FROM "jobs" +WHERE salary_min >= 250000 OR salary_max >= 350000 +SORT salary_min DESC +``` +```` + +**What you get:** filter by your real constraints, not just score. + +--- + +## Problem 4: "I want to make better application decisions over time" + +You apply to 5 jobs. Three ghost you. Two screen you out. You don't have a record of WHY you applied and what the pattern is. + +### Idea 4.1: Decision journal + +A note per application with the decision rationale: + +```markdown +# 2026-05-19 — Why I applied to Sierra Agent Architecture + +## Why yes +- Score 3.0 + my MCP/LangGraph fit +- Stretch role — gets me to the agent-engineering interview table +- Sierra is well-funded, founder-mindset language matches +- Has [[Compass-style HITL]] in their JD → talking point + +## Why I almost said no +- Score is at threshold; not a slam-dunk fit +- Onsite in NYC; I'm Austin-based +- Compensation band not posted + +## Decision rule applied +"Above 3.0 + tier=apply-now + role-family=agent-engineer → always apply" + +## Result tracking +- 2026-05-19 — applied +- TBD — screen +- TBD — onsite +- TBD — outcome +``` + +**Pattern recognition:** after 6 months, review every "Why yes" — what predicted callbacks vs ghosting? Refine the decision rule. Update `_profile/preferences.md`. + +### Idea 4.2: Apply-rate dashboard + +Dataview block tracking apply rate per company / tier / role-family: + +````markdown +```dataview +TABLE WITHOUT ID + group AS "Bucket", + length(rows.file) AS "Total seen", + length(filter(rows.status, (s) => s = "applied")) AS "Applied", + length(filter(rows.status, (s) => s = "applied")) / length(rows.file) * 100 AS "Apply %" +FROM "jobs" +GROUP BY tier +``` +```` + +**What you get:** "I see 30 apply-now jobs/month, apply to 4. Conversion: 13%." Useful for setting goals + spotting weeks you're under-applying. + +### Idea 4.3: Offer comparison table + +When you have 2+ offers, a single note `offers/2026-Q2-comparison.md`: + +```markdown +# Q2 Offer Comparison + +| Attribute | Sierra | Decagon | Cognition | +|---|---|---|---| +| Base | $220K | $235K | $210K | +| Equity (4-yr) | $400K | $300K | $600K | +| Bonus | 15% | 20% | 10% | +| Role family | Agent Eng | Agent Eng | Applied AI | +| Tier | apply-now | apply-now | apply-now | +| Notes link | [[applications/sierra-...]] | [[applications/decagon-...]] | [[applications/cognition-...]] | +``` + +Dataview can auto-build this from ApplicationNotes once they exist. + +--- + +## Problem 5: "I lose track of my network and outreach" + +Recruiters DM you. Former coworkers refer roles. Friends-of-friends offer intros. You forget who said what when. + +### Idea 5.1: PersonNote per contact + +`compass-vault/people/Jane_Doe.md`: + +```markdown +--- +type: person +role: Engineering Manager +company: Sierra +relationship: ex-coworker | recruiter | referral | mutual +linkedin: https://linkedin.com/in/janedoe +first_contact: 2026-05-15 +last_contact: 2026-05-19 +status: warm +--- + +# Jane Doe — Sierra + +Met at the AI agents meetup in Austin (April 2026). Now EM of the agent +orchestration team. Offered to refer me when I apply. + +## Conversations +- 2026-05-15 — DM exchange about Compass project. Liked the HITL design. +- 2026-05-19 — Referral offered for [[applications/sierra-...]] + +## Related +- [[applications/2026-05-19-sierra-Software_Engineer_Agent_Architecture]] +- [[companies/sierra]] +``` + +**Graph view payoff:** people → companies → JobNotes → skills. The whole network as a navigable map. + +### Idea 5.2: Outreach cadence reminders + +Tag people with `#contact/follow-up-due`. Dataview panel on dashboard: + +````markdown +```dataview +LIST file.link +FROM "people" +WHERE last_contact != null AND date(last_contact) < date(today) - dur(30d) + AND status = "warm" +SORT last_contact ASC +``` +```` + +**What you get:** monthly nudge to keep the network warm without active tracking effort. + +### Idea 5.3: Referral attribution + +ApplicationNote frontmatter already has a `referral: bool` field (Phase 1.A). Extend to `referral_from: "[[people/Jane_Doe]]"`. Then a dashboard panel: + +````markdown +## Referrals and their outcomes +```dataview +TABLE WITHOUT ID + file.link AS Application, + referral_from AS "Via", + status, + applied_at +FROM "applications" +WHERE referral_from != null +SORT applied_at DESC +``` +```` + +After 6 months: which referrers actually convert. Helps you prioritize who to ask next. + +--- + +## Problem 6: "I want to see my progress over time" + +Are my skills actually improving? Am I applying to better-fitting jobs? Am I closing the gap? + +### Idea 6.1: Skill-level over time chart + +Each `skills/X.md` already has `last_assessed: datetime` and `my_level: int`. Add a `level_history:` list: + +```yaml +level_history: + - {date: 2026-01-15, level: 1, note: "first MCP server"} + - {date: 2026-03-20, level: 3, note: "Cisco 4-server architecture shipped"} + - {date: 2026-05-19, level: 4, note: "Compass MCP server with production HITL"} +``` + +`skill_assessor` appends to this list whenever it changes a grade. Mermaid renders the trajectory: + +````markdown +```mermaid +gantt + title MCP Skill Trajectory + dateFormat YYYY-MM-DD + section Levels + L1 :2026-01-15, 64d + L3 :2026-03-20, 60d + L4 :2026-05-19, 30d +``` +```` + +**What you get:** visual progress per skill. Motivating during plateaus. + +### Idea 6.2: Score trend on JobNotes you've already been scored against + +If you re-score the same JD over time (say, every 6 months), the score should rise as your skills improve. Dataview tracks this: + +````markdown +## JDs I'd score higher on now (vs original) +```dataview +TABLE WITHOUT ID + file.link AS Job, + match_score AS Original, + rescored_at, + rescored_score AS Current +FROM "jobs" +WHERE rescored_at != null AND rescored_score > match_score +SORT (rescored_score - match_score) DESC +``` +```` + +Even better: a "re-score me" MCP tool that lets you periodically refresh select JobNotes and capture the delta. + +### Idea 6.3: Application pipeline funnel + +Mermaid + Dataview combo: + +````markdown +```mermaid +sankey-beta +applied,screen,2 +screen,onsite,1 +onsite,offer,1 +applied,rejected,3 +screen,rejected,1 +``` +```` + +Auto-generated from ApplicationNote status fields. See your funnel conversion at a glance. + +--- + +## Problem 7: "The vault feels like a pile of files, not a product" + +You have 23 JobNotes, 95 SkillNotes, 9 CompanyNotes. The dashboard is the only "UI." Everything else feels like browsing folders. + +### Idea 7.1: Workspaces (saved layouts) + +Obsidian's "Workspaces" feature saves window layouts. Create three: + +- **Morning triage**: left = dashboard, center = pending_approvals output, right = filtered-jobs log +- **Interview prep**: left = JobNote, center = relevant SkillNotes (multiple panes), right = STAR stories +- **Deep work**: left = study-plan, center = today's daily note, right = compass-vault/_meta/agent-log (for forensics) + +Hotkey to switch. Zero code change — pure UI. + +### Idea 7.2: Callouts for high-impact information + +Obsidian renders `> [!callout]` blocks as visually distinct boxes. Use them in JobNotes: + +```markdown +> [!warning] Stretch role +> Score 3.0 is below the apply-now threshold but role family matches. +> Apply only if you can credibly tell a [[LangGraph]] + [[MCP]] story. + +> [!success] Strong fit +> Score 4.5, role-family agent-engineer, candidate has every required skill. +> Apply immediately. + +> [!info] Anti-claim risk +> JD requires [[Fine-Tuning]] which is on the candidate's anti-claim list. +> Acknowledge upfront; pivot to [[Prompt_engineering]] and [[Evals]]. +``` + +`vault_write_node` auto-generates the appropriate callout based on score + role + anti-claim intersection. JobNote at a glance tells you what to do. + +### Idea 7.3: Tier-based CSS + +A snippet in `.obsidian/snippets/compass-tiers.css`: + +```css +.frontmatter[data-tier="apply-now"] { border-left: 4px solid #4caf50; } +.frontmatter[data-tier="6-month"] { border-left: 4px solid #ff9800; } +.frontmatter[data-tier="stretch"] { border-left: 4px solid #2196f3; } +.frontmatter[data-tier="out-of-scope"] { opacity: 0.5; } +``` + +Visual signal. Scan a folder of JobNotes; tier is instant. + +### Idea 7.4: Custom icons via the Iconize plugin + +Folders + tags can have custom icons. Make the vault feel like a product: +- 🎯 `jobs/` +- 🧠 `skills/` +- 🏢 `companies/` +- 📋 `applications/` +- 📝 `_inbox/` +- 🔍 `_meta/` + +Surface-level but reduces cognitive friction during navigation. + +### Idea 7.5: Mermaid pipeline diagram embedded in dashboard + +Auto-update a Mermaid flowchart on the dashboard showing the LATEST pipeline run's path: + +````markdown +```mermaid +flowchart LR + A[scraped: 20] --> B{intake_filter} + B -->|in-scope: 6| C[extract] + B -->|out-of-scope: 14| F[(filtered-jobs.md)] + C --> D[score] + D --> E{hitl} + E -->|paused: 0| P((awaiting human)) + E -->|auto-reject: 4| W[vault] + E -->|approved: 2| T[tailor] + T --> W +``` +```` + +Auto-regenerated by `_append_run_log`. Visual operational dashboard. + +--- + +## Problem 8: "I want the assessor's grading to be transparent" + +The skill_assessor proposes new levels. You see the reasoning, but it's a wall of text per skill. Hard to compare across skills. + +### Idea 8.1: Assessor results dashboard + +After every `assess_skills` run, write the results to `_meta/last-assessor-run.md` as a structured table: + +````markdown +# Last assessor run: 2026-05-19T13:14 + +```dataview +TABLE WITHOUT ID + skill AS Skill, + current_level AS "Now", + proposed_level AS "Proposed", + confidence, + requires_hitl AS HITL? +FROM "_meta/last-assessor-run" +SORT (proposed_level - current_level) DESC +``` +```` + +One glance → all skill movements ranked by delta. Cleanly readable. + +### Idea 8.2: HITL queue for assessor-flagged skills + +The assessor sets `requires_hitl=True` for 2+ level jumps. These should pile up in a "review queue" similar to the JobNote `pending_approvals`. A note `_meta/skill-grade-pending.md` listing them; an MCP tool `apply_skill_grade(skill, accept | reject)`. + +**What you get:** explicit human override on grade changes. The audit trail mirrors HITL on JobNotes. + +### Idea 8.3: Anti-claim explicit on skill page + +`skills/Fine-Tuning.md` has `grade_override: 0` (per `_profile/role-clarifications.md`). Render this prominently with a callout: + +```markdown +> [!danger] Anti-claim — DO NOT claim this skill in applications +> This skill is explicitly disclaimed in `_profile/role-clarifications.md`. +> Compass auto-rejects JDs requiring this skill at score level 0. +``` + +**What you get:** can't accidentally claim something you said you wouldn't. + +--- + +## Problem 9: "I want to capture lessons from building Compass itself" + +Compass IS your portfolio. The decisions you made building it are interview gold. But the decisions live in commit messages, not anywhere you can practice talking about them. + +### Idea 9.1: Promote decision-doc into the learning vault + +Each major architectural decision (the 5 plan-review iterations for the msgpack API, the C1 audit-trail divergence fix, the cosine-pinning silent bug) becomes a `learning-vault/projects/compass/decisions/YYYY-MM-DD-title.md` note. + +Format: +```markdown +# 2026-05-19 — msgpack allowlist API: the 5-iteration silent-bug saga + +## Context +LangGraph 1.2 logged "Deserializing unregistered type" on every paused +thread resume. The fix was supposed to be a one-line allowlist. + +## Decision +Use the constructor form `JsonPlusSerializer(allowed_msgpack_modules= +[(module, classname), ...])`. NOT the post-construction +`with_msgpack_allowlist` method. + +## Alternatives considered +1. Bare module string `['compass.pipeline.state']` — silently fails +2. Class list `[RawJob, ...]` — silently fails +3. Post-construction `serde.with_msgpack_allowlist(...)` — no-op when + default `_allowed_msgpack_modules=True` + +## Why it tipped +Five plan-review iterations with empirical probes finally confirmed only +the constructor form actually suppresses the warning. The deprecation +warning's own suggested fix is misleading. + +## Lesson +"The API does what the docs say" is not a safe assumption for evolving +libraries. Add empirical probes at plan-review time for any new API surface. + +## Skills demonstrated +[[LangGraph]] [[Python]] [[Testing]] [[Production_Concerns]] +``` + +**What you get:** real interview-ready stories. When asked "tell me about a difficult debugging session" — you have 10+ written-up answers, each linked to the skills they demonstrate. + +### Idea 9.2: "What I'd do differently" log + +A single accumulating note `learning-vault/projects/compass/retrospective.md`: + +```markdown +## What I'd do differently in Compass + +1. **Plan-review iterations cost time but save MORE time.** Six iterations + on the 1.B.2 plan caught 4 critical bugs before execution. Without it, + I'd have debugged them at runtime. Default to 3+ iterations. + +2. **Test isolation in pytest.** Raw `module._var = lambda: x` assignments + leak across tests. Use `monkeypatch.setattr` always. Fixed at c4c7fe8; + would have caught earlier if pytest-randomly was installed from day 1. + +3. **LLM behavior bugs need eval harness, not prompt-tuning.** Discovered + in 1.B.2 post-smoke that extract under-extracts on best-fit JDs. Tried + to ad-hoc fix the prompt; rolled back. Building the 30-JD eval set + first (Phase 2.A) is the right order. +``` + +**What you get:** lessons internalized + interview-ready. "What's a hard lesson you've learned recently?" — read directly from this note. + +--- + +## Recommended Layered Rollout + +If P1+P2+P3 from Part 1 is the foundation, here's what to layer on after: + +### Layer 1 (high value, low effort, ~2 hours total) +- **Idea 1.3** "Today's study target" dashboard widget (~10 min) +- **Idea 3.2** "This week's best fits" Dataview panel (~5 min) +- **Idea 7.2** Auto-callouts on JobNotes (~30 min in writer + smoke test) +- **Idea 7.3** Tier-based CSS snippet (~5 min) +- **Idea 8.1** Assessor results dashboard table (~30 min — write structured output) + +### Layer 2 (medium value, more time, ~5 hours) +- **Idea 1.1** Daily note → skill evidence sync script (~30 min) +- **Idea 2.1** STAR story bank scaffolding (~1 hour — establish convention + 1-2 example stories) +- **Idea 4.1** Decision journal convention + dashboard panel (~30 min) +- **Idea 9.1** Promote 3-5 Compass architectural decisions to learning vault (~2 hours) +- **Idea 3.1** Quick-capture inbox + inbox_scraper (~1 hour) + +### Layer 3 (when daily use exposes the need) +- **Idea 5.1** PersonNote system (once your network grows) +- **Idea 6.1** Skill-level over time + Mermaid trajectory +- **Idea 6.3** Application funnel Sankey +- **Idea 7.5** Mermaid pipeline diagram on dashboard +- **Idea 8.2** HITL queue for assessor grade changes + +### Layer 4 (portfolio polish, do before public README) +- **Idea 4.3** Offer comparison table (when you have 2+ offers) +- **Idea 7.1** Workspaces (after layouts stabilize) +- **Idea 9.2** "What I'd do differently" retrospective (interview material) + +--- + +## What I'd actually do first if I were you + +If you only have **30 minutes** today and want concrete progress on the system feeling more useful: + +1. **Idea 7.2 + 7.3** (callouts + CSS) — 35 minutes, immediately visible visual lift on every JobNote +2. **Idea 1.3 + 3.2** (two new dashboard panels) — 15 minutes, daily-flow improvement + +If you have **a full afternoon (~3 hours)** and want a single multi-component improvement: + +- **Ship Part 1's P1+P2+P3 bundle.** That's the foundation. Everything else in this Part 2 doc layers on top of P1's wikilink graph more powerfully than it would standalone. + +If you have **a full day** and want to invest in interview prep: + +- **Idea 2.1** (STAR story bank) + **Idea 9.1** (promote 3-5 Compass decisions) — 4-5 hours combined. By end of day you have 8-10 written-up stories tagged by skill, ready to deploy in a Friday recruiter screen. + +--- + +## Why this matters for the job-search use case specifically + +Most candidates have: +- A resume (one-size-fits-all) +- A list of LinkedIn jobs (chronological, not score-sorted) +- A vague sense of which skills they should learn next +- No record of why they applied where, or what worked in interviews + +What Compass + Obsidian leverage gives you: +- Per-JD score + tailored framing +- Score-sorted recommendation engine +- Empirical gap plan from real JD market signal +- Decision audit trail (which roles, why, what happened) +- Network graph (people → companies → roles) +- Interview prep that maps directly to JobNote requirements +- Skill trajectory tracking + +This is the kind of system a Sierra/Decagon/Cognition hiring manager wants to *see*. Not just hear about — but click through. The Obsidian rendering layer IS the demo. diff --git a/docs/PROJECT_OVERVIEW.md b/docs/PROJECT_OVERVIEW.md new file mode 100644 index 0000000..819f955 --- /dev/null +++ b/docs/PROJECT_OVERVIEW.md @@ -0,0 +1,510 @@ +# Compass — Complete Project Overview + +> A plain-language walkthrough of what Compass is, how it works, what we've built phase by phase, and why each design decision was made. Read this when you want to see the whole project in one place. + +**Current tag:** `phase-1b2-rag` · **Tests:** 259 passing · **Branch lineage:** `phase-0a-foundation` → `phase-0b-pipeline-mvp` → `phase-1a-application-tracking` → `phase-1b1-hitl` → `phase-1b2-rag` + +--- + +## 1. What is Compass? + +### The problem + +Job search at the senior-engineer level is two simultaneous problems: + +1. **Application work** — find roles you fit, decide which ones to apply to, tailor your story for each +2. **Skill-gap work** — figure out what the JD market is actually asking for that you don't yet have, and study toward those gaps + +Most job-search tools solve only the first problem. They surface roles. They don't help you decide what to study to be a stronger candidate next quarter. + +### The solution + +Compass is a tool that does both at once. It: + +- Scrapes job boards (Greenhouse, Lever, Ashby) +- Drops the noise (sales, PM, design roles) +- Scores each engineering role against your profile (0.0 - 5.0) +- Identifies skills the JDs want that you don't yet have, ranked by demand +- Pauses the high-scoring ones so you (the human) can approve before any LLM-generated tailoring is wasted +- Writes everything to an Obsidian markdown vault so you can read/edit it like a regular notes app +- Has an **agent inside it that grades your own skills against evidence you provide**, then re-prioritizes the gap plan based on which skills moved + +It's a research-and-preparation tool, not an auto-apply bot. The human is in the loop for every application decision. + +### The three "sides" of the system + +Compass has three storage layers, each with a different owner: + +``` +┌─────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐ +│ THE PIPELINE │ │ PRODUCT VAULT │ │ LEARNING VAULT │ +│ (the code) │─────▶│ agent-written │◀─────│ human-written │ +│ │ │ markdown │ │ markdown │ +│ scrapes JDs │ │ │ │ │ +│ scores them │ │ jobs/*.md │ │ projects/*/*.md │ +│ writes results │ │ skills/*.md │ │ canon/*.md │ +│ runs gap analysis │ │ applications/*.md │ │ daily/*.md │ +│ │ │ companies/*.md │ │ roadmap/*.md │ +└─────────────────────┘ │ dashboard.md │ └──────────────────────┘ + └──────────────────────┘ + ~/Documents/compass-vault ~/Documents/learning-vault +``` + +**Why three?** Separation of concerns. The agent owns what it writes (jobs, skills, scores) and never touches your personal notes. Your learning vault is private; the agent only *reads* it (via `learning-vault://` URIs) to find evidence that you've actually done the work you claim. + +This is the system's "unique angle": the agent doesn't take your word for what you know. It grades your skills against the evidence in your learning vault, using a deliberately skeptical hiring-manager persona. + +--- + +## 2. The Big Picture: How a Job Flows Through Compass + +A single job posting goes through 8 steps. Each step has a Python module in `compass/pipeline/nodes/`. The whole thing is a LangGraph state machine — meaning each step writes to a shared `state` dictionary, and the graph routes between steps based on what's in that state. + +``` + ┌──────────────────────────────────────────────────────┐ + │ │ + raw JD ───▶ intake ──▶ intake_filter ─out-of-scope──▶ filtered-jobs.md + │ + │ in-scope + ▼ + extract ──▶ score ──▶ reflect + │ + ▼ + hitl + ┌────────┼─────────┐ + below-thresh above-thresh + │ (PAUSE here) + │ ┌─────────────────┐ + │ │ human approves │ + │ │ via MCP tool │ + │ └─────────────────┘ + │ │ + ▼ ▼ + vault_write ◀──── tailor + │ + ▼ + JobNote.md +``` + +Let me walk through each step. + +### Step 1: `intake_node` + +Just packages the raw job data into the shared state. No LLM call. No skill needed. This is the entry point. + +### Step 2: `intake_filter_node` + +The first cost-control gate. Decides if this JD is even worth showing to the LLM. Three stages in order: + +1. **Title keyword filter** — if the title contains `"account executive"`, `"product manager"`, `"sales engineer"`, etc., it's out-of-scope. Cheap, no LLM call. +2. **Body-signal upgrader** — if the title is something generic like "Software Engineer" but the JD body mentions "agent", "MCP", "LangGraph" enough times, promote the role to `agent-engineer`. +3. **LLM fallback** — for genuinely borderline titles (e.g. "Solutions Architect"), call Gemini Flash once to read the first 500 chars of the JD and decide. + +Out-of-scope JDs get logged to `_meta/filtered-jobs.md` and short-circuit to END. They never reach the score step. + +**Why this matters:** without intake_filter, every JD costs ~$0.003 for extract + ~$0.003 for score, even the sales JDs. With it, ~70% of JDs drop here for free. + +### Step 3: `extract_node` + +Calls Gemini 2.5 Flash with a structured-output schema (Pydantic AI). The LLM reads the JD and produces: + +- `required_skills: list[str]` — canonical skill names from the JD +- `nice_to_have_skills: list[str]` +- `seniority: "junior" | "mid" | "senior" | "staff" | "unknown"` +- `years_experience: int | None` +- `remote_policy: "remote" | "hybrid" | "onsite" | "unknown"` +- `summary: str` — a 2-3 sentence summary of the role + +The extracted skills are normalized through the canonical taxonomy (`compass/vault/taxonomy.py`) — so JD-mentions like "Pydantic" and "pydantic-ai" both map to `Pydantic AI`. + +**Defense in depth:** the schema doesn't catch the LLM inventing skills that aren't in the JD. So there's a substring-validation step: every extracted canonical (or synonym) must appear in the JD text. Phase 0 caught a case where the LLM read an Anthropic sales JD and invented "Federated Learning" — schema-valid, completely fabricated. The validation step now drops these silently. + +### Step 4: `score_node` + +The other expensive LLM call. Gemini 2.5 Flash again, structured output. It receives: + +- The job requirements from step 3 +- The candidate's profile context — resume + relevant skill-inventory chunks (retrieved via RAG, see Phase 1.B.2 below) + +And returns: + +- `score: float` (0.0 - 5.0) +- `reasoning: str` (2-3 sentences explaining) +- `matched_skills: list[str]` — skills from the JD the candidate has +- `missing_skills: list[str]` — skills from the JD the candidate doesn't have +- `tailoring_notes: str` (only if score ≥ 3.0) + +**Defense in depth:** the LLM is constrained twice — once via prompt ("matched/missing MUST be subset of JD's required + nice-to-have"), once via post-LLM filter (`_constrain_to_jd_skills`). Phase 0 caught the LLM listing every profile skill as "matched" when the JD had empty required_skills. + +### Step 5: `reflect_node` + +A no-op pass-through for now. Reserved for Phase 2.A where we'll have a second LLM read borderline scores and dissent. + +### Step 6: `hitl_node` — Human-in-the-Loop gate + +The interesting one. Three branches: + +- `score < SCORE_THRESHOLD` (default 3.5) → **auto-reject** with no LLM call. Sets `human_approved=False`. The JD still gets written to the vault (so the gap aggregator sees it), but the expensive Sonnet tailoring step is skipped. +- `score >= SCORE_THRESHOLD` → **`interrupt()`**. LangGraph pauses the graph mid-execution, saves a checkpoint to a SQLite DB, and returns control to the caller. The thread sits idle until a human approves via MCP tool. +- On resume → the LLM's `interrupt()` call returns the human's decision (`{"approved": True/False, "feedback": str}`) and the graph continues to either tailor or vault_write. + +**Why pause instead of always running tailor?** Tailor uses Sonnet (~$0.05/call). Cheap for one job, expensive at 50/day if most of them are mid-fit. Pausing lets the human bulk-approve only the apply-now candidates. + +### Step 7: `tailor_node` + +Only runs if the human approved. Uses Sonnet 4.6 to write a custom paragraph for that JD, referencing the candidate's real projects. The output is recruiter-grade prose, not LLM slop. Example from a real run: + +> "Position yourself as the ideal founder-mindset candidate who has already built and deployed real-world AI agents at scale. Lead with your MCP server portfolio at Cisco where you created natural language interfaces..." + +### Step 8: `vault_write_node` + +Writes the final JobNote markdown file to `~/Documents/compass-vault/jobs/YYYY-MM-DD-Company-Role-<hash>.md`. Updates the company's CompanyNote (`companies/Company.md`). Populates the audit trail (`hitl_decision: approved | rejected | auto_rejected | timed_out`, `hitl_at: <timestamp>`). + +After all jobs in the batch finish, `gap_aggregator.regenerate()` reads every JobNote and produces `study-plans/master-gap-plan.md` — the ranked list of "skills the JD market wants that you don't have, ordered by demand × score × tier-weight." + +--- + +## 3. Phase 0: Building the Foundation + +### Phase 0.A — Foundation (`phase-0a-foundation`) + +The bones. No LLM calls yet. + +- 3 ATS scrapers (Greenhouse, Lever, Ashby) — using each ATS's public unauthenticated API +- Vault reader/writer with Pydantic schemas +- Canonical skill taxonomy (95 skills, with synonyms) +- `learning-vault://` URI bridge + +### Phase 0.B — Pipeline MVP (`phase-0b-pipeline-mvp`) + +The LangGraph pipeline. All 7 nodes implemented. Per-node OpenRouter model routing (Gemini Flash for cheap extract/score, Sonnet for tailor). Batch-level URL dedup. Gap aggregator. Master gap plan auto-regeneration. + +### What Phase 0 taught us — the 16 silent bugs + +This phase is where the project's testing discipline took shape. **Every "ready" claim got rolled back by the next adversarial probe.** Sample bugs: + +| # | What broke | Lesson | +|---|---|---| +| 1 | Greenhouse scraper returned empty `content` field | API list endpoint omits content by default — append `?content=true` | +| 4 | LLM invented "Federated Learning" from an Anthropic sales JD | Validate every extracted skill appears in JD text | +| 6 | Taxonomy parser misread 3-column tables (Voice section had no Synonyms column) | Per-section header column detection | +| 7 | Substring fallback false positives: `"Pythonist" → Python`, `"Goblet" → Go` | Drop the substring fallback; strict synonym match only | +| 8 | `React` vs `ReAct` case collision — JDs mentioning React.js silently became the agentic ReAct pattern | Case-sensitive canonicals set | +| 11 | Filename collisions on similar titles (`Engineer/Backend` and `Engineer (Backend)` both became `Engineer_Backend.md` — silent overwrite) | Append 8-char SHA-1 of URL to filename | +| 12 | Skill counter drift — `appears_in_jobs` accumulated on every overwrite, inflated by URL-dedup reruns | Remove counter increments from pipeline; derive from JobNotes at gap-plan time | + +**The pattern:** tests pass, smoke tests pass, code looks correct — but real data is wrong. Schema-valid, semantically meaningless. Only adversarial probing on real outputs catches this class of bug. + +This pattern became the project's most expensive lesson and is repeated through every subsequent phase. + +--- + +## 4. Phase 1.A: Making It Daily-Usable + +### What shipped (`phase-1a-application-tracking`) + +Two big themes: + +**1. Role-family gating + removing the score-write gate.** Phase 0.B had `SCORE_THRESHOLD=3.5` as a vault-write gate — only jobs scoring above that got written. This was the wrong filter. Akash wants the gap plan to include skills from stretch roles (jobs he'd score 2.0 on), because *those gaps are the most informative for study planning*. Filtering on match score hid exactly the right signal. + +The fix: +- **Role-family gate at intake** (the `intake_filter_node` above) drops out-of-scope JDs cheaply +- **All in-scope JDs reach the vault** regardless of score +- The expensive Sonnet tailor step is still gated on score (so cost stays bounded) + +**2. Application lifecycle.** MCP tools so you can mark a job applied and track status transitions: `add_application(job_id)`, `update_application_status(app_id, status, next_action, next_action_date)`, `list_pending_actions()`. + +### The bugs Phase 1.A caught (20 of them) + +Same pattern as Phase 0. A few highlights: + +- **Bug #1**: parallel writes to `companies/Sierra.md` (5 concurrent jobs reading `roles_seen=0`, incrementing to 1, last writer wins) — fixed by deriving `roles_seen` at gap-plan time, never incrementing +- **Bug #2**: invalid manual `tier` value in Obsidian (user typing `tier: applynow` instead of `apply-now`) crashed the next pipeline run — fixed by tolerating invalid Literal values +- **Bug #B**: body-signal upgrader triple-counted `"agent" ⊂ "agentic" ⊂ "agentic ai"` — fixed by longest-first word-bounded matching +- **Bug #F**: `add_application` silently overwrote in-flight ApplicationNotes — fixed with `force=True` default-off + +--- + +## 5. Phase 1.B.1: Real Human-in-the-Loop + +### What shipped (`phase-1b1-hitl`) + +Phase 0/1.A's `hitl_node` was a placeholder — it auto-approved everything above threshold. Phase 1.B.1 made it real: + +- **`langgraph.types.interrupt()`** — when a high-scoring job hits `hitl_node`, the graph pauses. Mid-execution. The Python coroutine returns, but the graph state is checkpointed. +- **`AsyncSqliteSaver`** — the checkpoint goes to `~/.compass/checkpoints.db`. Survives process restarts. +- **`compass/hitl/state_store.py`** — a separate SQLite table tracking which threads are paused, with company/title/score for display. Powers the `pending_approvals()` MCP tool. +- **`compass/hitl/resume.py`** — re-opens the checkpointer, recompiles the graph, drives `graph.ainvoke(Command(resume={"approved": True, "feedback": "LGTM"}))`. The single entry point for both human approves AND timeout auto-cancels. +- **`compass/hitl/timeout_checker.py`** — auto-cancels pending approvals older than `HITL_TIMEOUT_HOURS` (default 4). Phase 1.B.3 will wire this into a Modal cron. +- **MCP tools** — `pending_approvals()` and `approve(thread_id, approved, feedback)` so the human can see and resolve from Claude Code. +- **Audit trail** — `JobNote.hitl_decision` is now a Literal of `approved | rejected | auto_rejected | timed_out`, populated from state by `vault_write_node`. `JobNote.hitl_at` is the timestamp. + +### Why this is load-bearing for the portfolio claim + +The spec says "real LangGraph `interrupt()` + `AsyncSqliteSaver` checkpointing." This is one of the headline differentiators when interviewing for tier-2 agentic-engineering roles. Phase 1.B.1 delivers it as actual working code, not a slideware claim. + +### The bugs adversarial review caught (8 more) + +The plan went through **6 adversarial review iterations** before execution. Found defects included: + +- **C1 (silent data divergence)**: state_store said "approved" but JobNote said "auto_rejected" for the same thread — because the resume process used the default `SCORE_THRESHOLD=3.5` while the original run used `1.5`, so `hitl_node` short-circuited on resume without consuming the `interrupt()` value. Two-part fix: derive resume status from the FINAL graph state, AND make the threshold check sticky by capturing it in state at score time. +- **C2 (run-log column drift)**: pre-1.B.1 `pipeline-runs.md` header was 5-col, new code wrote 6-col rows. Dataview rendered misaligned. Self-healing migration added. +- **C3 (counter drift on resume paths)**: `gap_aggregator.regenerate()` was only called at end of `run_pipeline()`. Resume paths bypassed it. Cognition CompanyNote showed `roles_seen: 3` while 9 cognition JobNotes existed on disk. Fixed by calling regen at end of `resume_pending`. +- **I2 (thread_id collision)**: `_thread_id_for()` hashed `(url, batch_started_at)` — two cross-process pipelines starting at the same microsecond would collide. Added `os.getpid()` to the hash. + +The plan-review pattern: each adversarial pass rolled back the previous "ready" verdict. Six iterations was the diminishing-returns line. + +--- + +## 6. Phase 1.B.2: RAG + Cleanup (current) + +### What shipped (`phase-1b2-rag`) + +Two themes: + +**1. RAG via Chroma.** Previously `score_node._profile_text()` injected the entire `_profile/skill-inventory.md` (~2,500 tokens) into every score prompt. Phase 1.B.2 replaces that with semantic retrieval: + +- `compass/rag/indexer.py` parses `skill-inventory.md` into one chunk per `## SkillName` section, embeds each via `sentence-transformers all-MiniLM-L6-v2`, stores in a Chroma collection pinned to **cosine** distance +- `compass/rag/retriever.py` exposes `retrieve(query, k=8) -> list[RetrievedChunk]` with lazy index init +- `score_node` builds a query string from the JD's `required_skills + nice_to_have + summary` and retrieves the 8 most relevant skill-inventory chunks; injects those instead of the full file +- Token savings: ~2,500 → ~750 per scored JD +- Real-data verification: the same Sierra JobNote that scored 3.0 pre-RAG scored 3.0 post-RAG — no quality regression, just smaller context + +**2. Phase 1.B.1 carryover fixes.** + +- **msgpack deprecation**: every paused thread logged `Deserializing unregistered type compass.pipeline.state.RawJob from checkpoint. This will be blocked in a future version.` LangGraph 1.2 has a `JsonPlusSerializer(allowed_msgpack_modules=[(module, classname), ...])` constructor that suppresses the warning. Took **5 plan-review iterations** to find the right API form — the obvious-looking `with_msgpack_allowlist` method is a no-op when the default `allowed_msgpack_modules=True`. +- **checkpoint DB bloat**: every paused thread accumulates ~10 checkpoint blobs in `~/.compass/checkpoints.db`, never deleted. New `_purge_thread_checkpoints(thread_id)` in `resume.py` runs after `mark_resolved`. Bounded growth. + +### Why RAG instead of just dumping the inventory + +The spec made RAG a portfolio-claim requirement. But the engineering rationale is real: + +- **Cost**: 2,500-token context costs ~3x what a 750-token one does at Gemini Flash rates. Over 50 jobs/day, that's real money. +- **Focus**: full inventory inject includes irrelevant chunks ("Voice — skip"). Retrieved chunks are the skills actually closest to the JD's vocabulary. +- **Demonstrable**: "I built RAG over my profile and reduced score-prompt tokens 70%" is a concrete interview talking point. + +### Why Chroma + all-MiniLM-L6-v2 + +- **Chroma**: simplest persistent vector DB with a real Python API. PersistentClient writes to a SQLite file. No service to run. Cosine distance pinned at collection-create. +- **all-MiniLM-L6-v2**: 90MB model, downloaded once to `~/.cache/torch/`. Fast enough to embed 20 chunks in <100ms on a laptop. Cosine similarity in [0, 1] is meaningful out of the box. +- **Alternative considered**: `nomic-embed-text` (better quality) — deferred to Phase 2.B as a one-line config change. + +### The bugs adversarial review caught (8 more) + +**6 plan iterations + 2 code-quality iterations.** Highlights: + +- **C1 (the silent metric bug)**: existing `~/.compass/chroma/` collections from before the cosine pin would keep the L2 default. `get_or_create_collection(metadata={"hnsw:space": "cosine"})` does NOT update the metric on an existing collection. The retriever's `1.0 - dist/2.0` formula then produces negative similarities. Fix: inspect existing collection's metadata, drop+recreate if not cosine. +- **`Assessor-current grades` heading pollution**: the assessor writes a `## Assessor-current grades` section to `skill-inventory.md` containing every skill name. If embedded into the index, it would dominate retrieval for any query. Excluded via `_EXCLUDED_HEADINGS`. +- **`_kebab` parenthetical stripping**: heading `## Fine-Tuning (awareness only — explicit anti-claim)` would otherwise become the unreadable ID `fine-tuning-awareness-only-explicit-anti-claim-per-role-clarifications`. Regex strips parentheticals + em-dash commentary before kebab-casing while preserving internal hyphens (`Fine-Tuning` stays `fine-tuning`, not `fine`). +- **AI-slop cleanup** (8 patterns): plan-meta narrative in production docstrings, defensive `try/except: pass`, speculative `_load_model` / `_get_client` helpers, `await ... await ...` on one line, etc. + +### Post-tag findings (deferred to Phase 2.A) + +Adversarial post-smoke review surfaced four LLM-behavior bug classes — documented in [`KNOWN_DATA_QUALITY_ISSUES.md`](KNOWN_DATA_QUALITY_ISSUES.md): + +- **B1**: `extract_node` under-extracts `skills_required` on best-fit roles. Sierra Agent Architecture JD scored 3.0 because `skills_required: []` left no signal for matched_skills. +- **B2**: `extract_node` mis-reads OR-lists ("languages such as Python, TypeScript, Go") as AND-lists. +- **B3**: `score_node` occasionally marks candidate-strong skills as missing (Python listed missing despite candidate's Python=3). +- **B4 (FIXED)**: `intake_filter` title-keyword list was leaky. 5 false-negatives migrated post-fix. + +These are LLM-prompt-tuning bugs that need the Phase 2.A eval harness (30 labeled JDs + score-MAE measurement) to fix correctly. Whack-a-mole prompt tweaking without measurement is worse than waiting. + +--- + +## 7. The Skill Assessor Loop (the "unique angle") + +### What it does + +The candidate marks each skill in `skills/SkillName.md` with `evidence: [learning-vault://...]` URIs pointing to artifacts in the learning vault. The skill assessor agent (`compass/analysis/skill_assessor.py`) reads those URIs, applies a deliberately skeptical hiring-manager rubric, and proposes new grades. + +### The rubric + +``` +Level 0 No exposure. +Level 1 Tutorial-level: course notes, "hello world", or a read paper. +Level 2 Applied in a personal project. Repo or vault note showing real use. +Level 3 Shipped. Deployed, evals exist, OR used by people other than candidate. +Level 4 Production-grade. Shipped WITH observability + cost tracking + recovered from a real failure. +Level 5 Authority. Taught it, merged upstream PR, or fixed a non-trivial bug in the library itself. +``` + +The agent demands artifact citations. Conceptual evidence caps at L1. Jumps of 2+ levels require HITL. + +### Real first run (2026-05-19) + +After wiring 4 skills to actual evidence URIs: + +| Skill | Current | Proposed | Reasoning | +|---|---|---|---| +| MCP | 4 | **2** (requires_hitl=True) | "Empty or template documents in projects/minx/. Minimal content." 2-level downgrade triggers human review. | +| LangGraph | 1 | 1 | "Template files for Compass project, no LangGraph implementation artifacts visible. Decision log empty." | +| Eval_harness | 0 | 0 | "Hamel reading material status=queued, read_count=0. No applied work." | +| HiTL | 1 | 1 | "Decision log template without actual HiTL decisions or artifacts." | + +**This is the value working as intended.** The grader correctly refused to credit Akash's actual MCP/LangGraph/HiTL work — because the *learning vault* doesn't yet document it. The work itself lives in the compass codebase. Filling `learning-vault/projects/compass/decisions.md` with the real architectural decisions from Phases 0-1.B.2 will raise the grades on next run. + +This is the loop: study/build → write it down → cite the writeup as evidence → agent regrades → gap plan reprioritizes. It closes the feedback loop between "what the market wants" and "what I can credibly claim." + +--- + +## 8. Key Design Decisions (and why) + +### Why a markdown vault instead of a database? + +**Decision**: store all agent output as `.md` files with YAML frontmatter. + +**Why**: Obsidian renders it natively. Dataview queries give you SQL-like power over markdown. The user can edit fields by hand in Obsidian and the next pipeline run respects those edits (e.g. tightening a CompanyNote.tier). No separate UI to build. Version-controllable. Diff-able. + +**Trade-off**: schema enforcement happens at write time via Pydantic, but a user editing the file in Obsidian can introduce invalid values. Phase 1.A bug #d40f36e was a literal example — `tier: applynow` (typo) crashed the next run. Fixed by tolerating invalid Literal values on read. + +### Why MCP for HITL (and not a web UI)? + +**Decision**: the human approves paused jobs via MCP tools called from Claude Code, not via a separate web app. + +**Why**: Akash already lives in Claude Code daily. Building a Flask UI to surface 5 pending approvals would be over-engineered. MCP tools (`pending_approvals()`, `approve(tid, True, "LGTM")`) are zero-friction from his existing workflow. + +**Trade-off**: the dashboard doesn't surface pending approvals (Dataview can't read SQLite). Phase 2.B may add a thin web view if this becomes a daily-flow friction point. + +### Why three vaults (pipeline / compass-vault / learning-vault)? + +**Decision**: agent-written content goes to `~/Documents/compass-vault/`. Personal notes stay in `~/Documents/learning-vault/`. The pipeline code only writes to compass-vault. + +**Why**: separation of concerns. The agent can rewrite anything in compass-vault freely — it's the system's output. The learning-vault is Akash's brain — the agent never touches it; it only resolves `learning-vault://` URIs to read evidence. + +**Trade-off**: cross-vault evidence requires the URI protocol (`compass/vault/learning_bridge.py`). Slight indirection. But preserves the cognitive model: "the agent doesn't read my journal." + +### Why LangGraph (and not custom state management)? + +**Decision**: use LangGraph for the pipeline state machine. + +**Why**: `interrupt()` + `AsyncSqliteSaver` checkpointing is what makes real HITL work. Building this from scratch — durable pause-resume across process restarts with a SQLite checkpointer — would be a multi-week side project. LangGraph 1.2 gives it for free. + +**Trade-off**: LangGraph has its own conventions (state schema as TypedDict, edges defined declaratively). Required Phase 1.B.1 to be careful about the "compile graph inside async with checkpointer" invariant. The framework's once-only warning sets and msgpack allowlist API have required 5+ adversarial review iterations to use correctly. + +### Why subagent-driven development? + +**Decision**: most implementation work goes through fresh implementer subagents with combined spec+quality reviewers between tasks. + +**Why**: it caught 8+ real bugs across Phase 1.B.1, several of which were genuine silent-data-correctness defects the human reviewer wouldn't have spotted. Fresh subagent per task means no context pollution. The reviewer cycle creates a forcing function — implementer can't ship questionable code because a different agent will read it. + +**Trade-off**: more LLM invocations per task (implementer + 2 reviewers minimum). Higher token cost. But far cheaper than debugging silent bugs in production. + +### Why so much adversarial review? + +**Decision**: every plan goes through 3-6 adversarial review iterations before execution. Every implementation gets a spec+quality review. Live smoke verification is mandatory before tagging. + +**Why**: Phase 0 caught 16 silent bugs. Phase 1.A caught 20 more. Phase 1.B.1 caught 8 more. The pattern is consistent: tests pass + smoke tests pass + code looks correct → but real data is wrong. The only thing that catches this is *adversarial inspection of real outputs*. + +The portfolio cost of shipping a silently-broken Compass to recruiters is enormous. The cost of one more review pass is ~30 minutes. Math is obvious. + +**Trade-off**: process overhead. But it's now the project's defining methodology and (frankly) its strongest interview talking point — "I built a system that caught its own data-correctness bugs through structured adversarial review, where most agentic systems ship with the bugs." + +--- + +## 9. The State of the System Right Now + +### What works end-to-end + +- **Daily pipeline**: `uv run python -m compass.pipeline.graph` against the configured ATS boards → scrapes, filters, scores, writes. Real cost per 20 jobs: ~$0.05. +- **HITL flow**: above-threshold jobs pause; `pending_approvals()` MCP tool lists them; `approve(tid, True, "LGTM")` resumes into tailor + vault_write. Audit trail intact. +- **Timeout cancel**: `uv run python -m compass.hitl.timeout_checker` resumes stale pending approvals as `timed_out`, writes to agent-log per spec. +- **RAG**: `uv run python -m compass.rag.indexer --force` builds the Chroma index. `score_node` uses retrieved chunks instead of full inventory. +- **Gap plan**: regenerates after every pipeline run AND after every resume. Out-of-scope JobNotes excluded. +- **Skill assessor**: `assess_skills(scope=[...])` MCP tool grades skills against evidence URIs. Adversarial grader catches under-documented claims. + +### What's stubbed but not yet exercised in production + +- **Modal cron**: `timeout_checker.py` is callable but no schedule wires it. → Phase 1.B.3. +- **Langfuse traces**: callback API mismatch (`host` kwarg) — every run logs the error, pipeline continues. → Phase 1.B.3. +- **Eval harness**: 30 labeled JDs + nightly score-MAE regression. Not built. → Phase 2.A. +- **Application lifecycle**: tools work but `~/Documents/compass-vault/applications/` is empty — nobody has applied via the tool yet. + +### What's deferred and why + +[`docs/KNOWN_DATA_QUALITY_ISSUES.md`](KNOWN_DATA_QUALITY_ISSUES.md) lists 8 known bug classes (B1-B8) with severity, evidence, and Phase 2.A fix surface for each. The headline ones: + +- **B1**: extract under-extracts skills_required on best-fit JDs +- **B7**: gap_aggregator includes auto_rejected/timed_out/null at full weight +- **B8**: tailor_node has no programmatic constraint against hallucination + +Each needs a regression test against labeled JDs (Phase 2.A) before tuning, otherwise we whack-a-mole. + +--- + +## 10. What's Next + +### Phase 1.B.3 — automation + observability (next sub-phase) + +- **Modal cron** for daily scrape (9 AM Central) + weekly skill_assessor (Sunday 2 AM Central). The `timeout_checker` becomes a Modal-scheduled job. +- **Langfuse callback fix**. Then a public trace URL goes in the README. Portfolio claim. +- **Modal Secrets**: secrets out of `.env`, into Modal's secret manager. Compass repo can be public. +- **URL dedup for filtered-jobs**. Currently the same sales JDs hit `intake_filter` every run. Trivial cache. +- **I4 race fix**: `claim_pending` atomic transition prevents double-resume races once Modal cron and human MCP can fire simultaneously. + +### Phase 2.A — eval harness + +- 30 hand-labeled JDs in `compass/evals/dataset.json` +- Nightly: extract recall, score MAE vs ground truth, cost-per-run, p50 latency +- LangFuse-logged with regression alert if MAE > baseline + 2σ +- **Then** tune extract/score prompts to fix B1-B3. Measure each change. + +### Phase 2.B — portfolio polish + +- README: architecture diagram (mermaid), screenshots, public Langfuse trace URL, "what I learned" section, eval results table +- Public Langfuse trace URL reachable without auth +- Repo link in resume Projects section + +### Phase 2.C — blog post + +"Building an agent that grades my skills against the live job market." The skill_assessor loop is the differentiated story. + +### Phase 3+ — coverage expansion + +Workday, Apple, Google, AWS, Microsoft scrapers. YC WAAS, HN Who's Hiring. Long-tail. + +--- + +## 11. Glossary + +| Term | Meaning | +|---|---| +| **Agentic AI** | software that orchestrates LLM calls + tools to accomplish multi-step tasks (e.g. agent that searches the web, summarizes findings, drafts a response). Compass itself is agentic. | +| **ATS** | Applicant Tracking System (Greenhouse, Lever, Ashby). Standard interface job postings flow through. Compass scrapes each ATS's public unauthenticated API. | +| **CompassState** | the TypedDict shared between all LangGraph nodes. One JD's-worth of pipeline state. | +| **Canonical taxonomy** | the 95-skill registry at `_meta/skill-taxonomy.md`. Maps synonyms ("pydantic-ai", "Pydantic AI") to one canonical name. | +| **Chroma** | the persistent vector DB used for RAG. Stores `skill-inventory.md` chunks + their embeddings. | +| **`interrupt()`** | LangGraph function that pauses graph execution mid-node. The graph state is checkpointed; control returns to the caller. The same node re-runs on resume with the human's value substituted. | +| **JobNote** | one markdown file per scored JD at `compass-vault/jobs/`. Frontmatter has `match_score`, `hitl_decision`, `skills_matched`, etc. | +| **MCP** | Model Context Protocol. Anthropic's standard for exposing tools to LLM clients (Claude Code, Cursor). Compass's MCP server exposes 16 tools — `pending_approvals`, `approve`, `add_application`, etc. | +| **Modal** | the serverless platform where Phase 1.B.3 will host the daily cron. Free tier covers Compass's expected usage. | +| **Pydantic AI** | the library that gives LLMs typed structured output. Define a Pydantic model; the LLM is constrained to return data that validates against it. Used for extract + score nodes. | +| **RAG** | Retrieval-Augmented Generation. Instead of dumping all candidate-profile data into every LLM prompt, retrieve the relevant chunks via vector similarity. | +| **Role family** | the classification an in-scope JD falls into: `agent-engineer`, `applied-ai`, `swe-backend`, `fde-eng`, `research-eng`, or `out-of-scope`. Stored on JobNote frontmatter. | +| **score_threshold** | currently 3.5. Jobs scoring ≥ this pause for human approval. Below auto-reject. Captured into pipeline state at score time so pause-resume across env changes stays consistent. | +| **`learning-vault://` URI** | the protocol the skill_assessor uses to read evidence files from `~/Documents/learning-vault/`. Read-only; the agent never writes there. | + +--- + +## 12. Files You'll Touch Most + +If you want to understand any one piece deeply, these are the entry points: + +- **The whole pipeline**: [compass/pipeline/graph.py](../compass/pipeline/graph.py) — `run_pipeline()` is the top-level function. Every node hangs off the graph defined here. +- **A specific node**: `compass/pipeline/nodes/{intake,intake_filter,extract,score,reflect,hitl,tailor,vault_write}.py` +- **HITL infrastructure**: `compass/hitl/{state_store,resume,timeout_checker}.py` +- **RAG**: `compass/rag/{indexer,retriever}.py` +- **MCP tools**: [compass/mcp_server/server.py](../compass/mcp_server/server.py) +- **Skill assessor**: [compass/analysis/skill_assessor.py](../compass/analysis/skill_assessor.py) +- **Gap aggregator**: [compass/analysis/gap_aggregator.py](../compass/analysis/gap_aggregator.py) +- **Vault schemas**: [compass/vault/schemas.py](../compass/vault/schemas.py) — what every JobNote/SkillNote/CompanyNote/ApplicationNote frontmatter must look like +- **Config**: [compass/config.py](../compass/config.py) — every env var and default path lives here + +And the corresponding test files in `tests/` mirror the same structure. 259 tests at current tag. + +--- + +## 13. The Big Picture (one paragraph) + +You're building an agentic career coach that scrapes job boards, drops noise, scores in-scope JDs against your profile via Gemini Flash, pauses high-scoring jobs for your approval through real LangGraph `interrupt()`, generates Sonnet-tailored paragraphs for approved ones, derives a master gap plan from every scored JD via weighted demand math, and houses an adversarial skill grader that reads evidence from your personal learning vault and proposes grade changes when you've shipped new work. All output is markdown in an Obsidian vault. Cost ~$0.05 per 20-job batch. 259 tests passing. Six phases left to complete: automation cron, observability, eval harness, polish, blog post, coverage expansion. + +You're roughly halfway through the planned 8-10 sessions. The hardest architectural calls are made. What remains is operational discipline + measurement. diff --git a/docs/TWO_WEEK_SPRINT.md b/docs/TWO_WEEK_SPRINT.md new file mode 100644 index 0000000..c2ee9c3 --- /dev/null +++ b/docs/TWO_WEEK_SPRINT.md @@ -0,0 +1,391 @@ +# Compass 2-Week Portfolio Sprint — Handoff Doc + +> **You're picking this up in a fresh chat. Read this top-to-bottom first. It tells you what state the project is in, what we're shipping over the next 14 days, what to do today, and how to work with the user.** + +--- + +## TL;DR for a fresh agent + +- **Goal:** ship a complete portfolio-ready Compass in 14 days (ending ~2026-06-02). +- **State today (2026-05-19):** `phase-1b2-rag` tag landed; 7 commits past it on `phase-1b2-rag` branch; 259 tests passing; ruff clean; vault state coherent. +- **Daily commitment:** 2-4 focused hours, every day, including weekends. +- **What's done:** Phases 0.A, 0.B, 1.A, 1.B.1, 1.B.2. Skill assessor wired with real evidence URIs (4 skills). B6 role_family migration shipped. +- **What remains:** Obsidian Part 1 → Phase 1.B.3 → Phase 2.A → Phase 2.B (+ folded Part-2 ideas) → Phase 2.C blog post. +- **Today's first action:** start Day 1 of the sprint (Obsidian Part 1 — JobNote/SkillNote body wikilinks + dashboard panels + auto-tags). + +--- + +## The 14-day plan + +| Day | Phase | Work | Hours | +|---|---|---|---| +| 1 | Obsidian P1 | Writer change: add `## Skills` section with `[[Python]]·[[LangGraph]]` wikilinks to JobNote body. Backfill migration script for 23 existing JobNotes. | 3 | +| 2 | Obsidian P1+P2+P3 | SkillNote body gets `## Jobs requiring this skill` list (regenerated by `gap_aggregator._sync_skill_counters`). Dashboard panels: "Skill rarity" + "Rare/specialized." Auto-tags on JobNote frontmatter. Backfill SkillNote bodies + JobNote tags. Tests. **Ship the Obsidian foundation.** | 3 | +| 3 | Phase 1.B.3 | Plan-write the phase. Langfuse callback fix (`compass/pipeline/graph.py:_langfuse_config` — `host=`/`secret_key=` kwargs cause `TypeError`). URL dedup for `_meta/filtered-jobs.md` (intake_filter writes log but never reads — same OUT titles hit LLM every run). | 3 | +| 4 | Phase 1.B.3 | `claim_pending` atomic transition in `compass/hitl/state_store.py` for the I4 double-resume race. Regression tests. | 2 | +| 5 | Phase 1.B.3 | `modal_app.py` at repo root + Modal Secrets for `OPENROUTER_API_KEY` / `LANGFUSE_*`. Cron schedules: daily scrape 9 AM CT, weekly skill_assessor Sunday 2 AM CT. First Modal deploy. | 3 | +| 6 | **USE IT** | Run pipeline manually 2-3 times. Apply to 1-2 real jobs via `add_application` MCP tool — first time exercising that codepath on real data. | 1-2 | +| 7 | Phase 2.A | Eval harness scaffolding. `compass/evals/dataset.json` schema. Runner code (`compass/evals/runner.py`) computing score MAE + skill recall. Label 10 of the existing JobNotes. | 4 | +| 8 | Phase 2.A | Label 10 more JDs (target: 20 labeled). Run eval harness → first MAE/recall baseline. Log results to Langfuse if working. | 3 | +| 9 | Phase 2.A | Prompt-tune `extract_node` (fix B1+B2) using eval feedback. Iterate 2-3 times; stop when MAE drops or stops improving. Document baseline in `docs/EVAL_BASELINE.md`. | 3 | +| 10 | Phase 2.B | Rewrite top-level `README.md`. Architecture diagram (Mermaid). Capture screenshots: dashboard, graph view, master gap plan, JobNote with audit trail. | 3 | +| 11 | Phase 2.B + folded Part-2 | Promote 3-5 Compass architectural decisions to `learning-vault/projects/compass/decisions/*.md` (the msgpack saga, C1 audit-trail divergence, cosine-pinning silent bug, adversarial review methodology). Wire them as `evidence:` URIs to relevant skills. Run `assess_skills` → screenshot the grade changes. | 3 | +| 12 | Phase 2.B | Daily-note → skill evidence auto-sync script. Assessor results dashboard panel. Public Langfuse trace URL embedded in README. Eval baseline numbers in README. | 3 | +| 13 | Phase 2.C | Blog post draft (1500-2000 words) — *"Building an agent that grades my skills against the live job market."* Outline + first draft. | 4 | +| 14 | Phase 2.C | Edit blog post. Final README polish. Make repo public on GitHub. Push. **Done.** | 3 | + +**Total: ~40 hours over 14 days.** Realistic if you commit to daily cadence. + +--- + +## Where the project actually is (the honest state) + +### Code +- Tag: `phase-1b2-rag` +- HEAD: 7 commits past the tag on `phase-1b2-rag` branch: + - `3828d8f` intake_filter OUT keyword widening (5 leak titles caught) + - `7ebd4a1` docs: expanded `KNOWN_DATA_QUALITY_ISSUES.md` + - `db3b132` B6 fix: role_family migration script + `gap_aggregator` skips out-of-scope + - `5b39d58` docs: PROJECT_OVERVIEW.md (510 lines) + - `daa4bb5` style: TYPE_CHECKING guard on test_gap_aggregator_role_filter.py + - `ad297ef` docs: OBSIDIAN_LEVERAGE.md (P1-P8) + - `19bcf16` docs: OBSIDIAN_LEVERAGE_PART_2.md (use-case oriented) +- Tests: **259 passing** (`uv run pytest -q`) +- Ruff: clean (`uv run ruff check && uv run ruff format --check`) +- pytest-randomly: NOT installed; consider `uv add --dev pytest-randomly` early to catch isolation leaks + +### Vault state +- `~/Documents/compass-vault/jobs/` — 23 JobNotes; 5 migrated to `role_family: out-of-scope` (correctly drop from gap plan) +- `~/Documents/compass-vault/skills/` — 95 SkillNotes; 4 have populated `evidence:` URIs (MCP, LangGraph, Eval_harness, HiTL) +- `~/Documents/compass-vault/applications/` — empty; `add_application` codepath never exercised on real data +- `~/Documents/compass-vault/study-plans/master-gap-plan.md` — fresh; top gaps: TypeScript (9 jobs), Go (5), RAG (4) +- `~/.compass/checkpoints.db` — 20 KB (purged from 13.8 MB on 2026-05-19; 0 stale threads) +- `~/.compass/hitl.db` — 0 pending approvals + +### What's known broken (deferred, documented) +- **Langfuse callback** — every run logs `LangchainCallbackHandler.__init__() got an unexpected keyword argument 'host'`, runs without traces. Fix in Day 3 (Phase 1.B.3). +- **`state_store.claim_pending` missing** — Modal cron + MCP approve will race. Fix in Day 4 (Phase 1.B.3). +- **B1-B3 LLM behavior bugs** — `extract` under-extracts on best-fit JDs; mis-reads OR-lists; `score` sometimes marks candidate-strong skills as missing. See `docs/KNOWN_DATA_QUALITY_ISSUES.md`. Fix in Day 7-9 (Phase 2.A via eval harness). + +--- + +## Required reading (in order, before Day 1) + +1. **`/Users/akmini/Documents/compass/CLAUDE.md`** — repo conventions, what you can/can't do +2. **`/Users/akmini/Documents/compass/docs/PROJECT_OVERVIEW.md`** — the 510-line plain-language walkthrough of the whole project. Get the architecture in your head. +3. **`/Users/akmini/Documents/compass/docs/KNOWN_DATA_QUALITY_ISSUES.md`** — what's documented as deferred, with severity + fix surface +4. **`/Users/akmini/Documents/compass/docs/OBSIDIAN_LEVERAGE.md`** — P1-P8 foundation proposals; **Day 1-2 work uses P1+P2+P3 from here** +5. **`/Users/akmini/Documents/compass/docs/OBSIDIAN_LEVERAGE_PART_2.md`** — deeper use-case ideas; **Day 11-12 folds in Ideas 9.1 + 1.1 + 8.1 from here** +6. **`/Users/akmini/Documents/compass/docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md`** — the authoritative master spec; the Definition of Done lives there + +You can skim PHASE_0_COMPLETE.md / PHASE_1A_COMPLETE.md / phase-1b1 plan / phase-1b2-rag plan for context but the PROJECT_OVERVIEW.md is the consolidated version. + +--- + +## Critical lessons carried forward (do not relearn) + +These were earned the hard way across Phases 0 → 1.B.2. Every one of them cost a real bug or a real review iteration: + +1. **Tests check shape, smoke tests check counts, neither catches data-correctness bugs on real inputs.** After ANY claim a phase is "ready," run adversarial probing on real vault output before believing the claim. The pattern repeats every phase: rubber-stamp review finds 0 bugs; adversarial review with grep-before-assert finds 3-7 real defects. + +2. **Module-level `from compass.config import X` freezes the value at import time** and silently breaks the `temp_*` test fixtures. Always late-bind via `import compass.config as cfg; cfg.X` inside function bodies. The `temp_vault` fixture monkeypatches `compass.config.VAULT_PATH`; module-level capture defeats it. + +3. **A LangGraph graph compiled without a checkpointer silently breaks `interrupt()`.** Never compile `build_graph()` at module level. Call it INSIDE `async with AsyncSqliteSaver.from_conn_string(...)` only. + +4. **`or`-fallback against falsy values is a recurring trap.** `state.get("score_threshold") or SCORE_THRESHOLD` would mis-fallback on `0.0`. Use `is None` checks always. + +5. **JSON-serializability of MCP return shapes** matters. FastMCP can't transmit `datetime` or Pydantic instances. Use `model_dump(mode="json")` or hand-build plain dicts. + +6. **Production code must be terse, WHY-only docstrings.** Match `compass/hitl/state_store.py` calibration. No plan-meta narrative ("verified at planning time"). No defensive paranoia (`try: ... except Exception: pass`). This is a portfolio repo — recruiters will read the source. + +7. **`with_msgpack_allowlist` is a no-op when default `allowed_msgpack_modules=True`.** Pass the allowlist via the CONSTRUCTOR: `JsonPlusSerializer(allowed_msgpack_modules=[(module, classname), ...])`. The `_warned_unregistered_types` set is per-session; clear it before measuring warning suppression in tests. + +8. **Chroma's `get_or_create_collection` does NOT update an existing collection's metadata.** Inspect existing metadata; drop+recreate if metric mismatches. + +9. **Derived-field staleness**: counters (`appears_in_jobs`, `roles_seen`) are recomputed at gap-aggregator time. But `role_family` is stored once at intake and never reconciled. If you add a new "set-once classification" field in future work, you need a migration script. + +10. **Adversarial plan review catches defects pre-execution.** Cost: 30-60 min per pass. Benefit: 1-7 real bugs per pass. Cap at 3 iterations before escalating. + +--- + +## Communication style with the user + +Read the user's CLAUDE.md preferences. Specifically: + +- **Be direct and concise.** Don't restate what they said. +- **Lead with the answer or action.** Not the reasoning. +- **Don't add comments / docstrings / type annotations to code you didn't change.** +- **Don't create files unless necessary. Prefer editing existing ones.** +- **When you save to the vault: tell them path + why.** + +What they DON'T want: +- Long menus of options when one recommendation is obviously right +- Excessive caveats +- AI-slop code (defensive paranoia, speculative helpers, chatty comments, plan-meta narrative in production) +- Apologetic preambles ("I apologize for...") +- Mention of the date/time unless asked + +What they DO want: +- Honest harsh assessments when asked +- Real adversarial reviews that find real bugs +- Direct recommendations +- Code that reads like a senior wrote it + +When unsure between two approaches, briefly state both and recommend one. Don't paralyze them with options. + +--- + +## Workflow for each day's work + +The user has been using subagent-driven development heavily. Follow the same pattern: + +1. **Read the day's task** in this doc. Note the files affected. +2. **For each phase, write a per-phase implementation plan** in `docs/superpowers/plans/YYYY-MM-DD-<phase-name>.md` using the `superpowers:writing-plans` skill. +3. **Dispatch a plan-document-reviewer** (general-purpose agent) with the actual codebase grepping required. 2 passes minimum. +4. **Apply fixes from review.** +5. **Execute via `superpowers:subagent-driven-development`**: + - Fresh implementer subagent per task + - Combined spec+quality reviewer subagent between tasks + - Pause before any task that hits production (real LLM, real vault writes) +6. **Live smoke** at the end of each phase. Snapshot vault + DBs first. Verify on real data. +7. **Tag** the phase: `phase-1b2-obsidian-p1`, `phase-1b3-automation`, `phase-2a-eval`, `phase-2b-polish`, `phase-2c-public`. + +--- + +## Day 1 — START HERE + +### Goal + +Add `## Skills` section with `[[Python]] · [[LangGraph]] · [[MCP]]` style wikilinks to every JobNote body. Backfill the 23 existing JobNotes via a one-shot migration script. This is the foundation that makes the Obsidian graph view meaningful. + +### Files to modify + +- `compass/vault/writer.py` — `write_job_note` function (currently around line 60-105). Add a body section between `jd_summary` and `## Full JD`. +- `scripts/migrate_jobnote_skills_section.py` — new one-shot migration (model after `scripts/migrate_role_family.py`). +- `tests/vault/test_writer.py` — add 1-2 tests verifying the new section. + +### Expected output + +A JobNote written by `write_job_note` should look like: + +```markdown +--- +... frontmatter ... +--- +# Sierra — Software Engineer, Agent Architecture + +...jd_summary text... + +## Skills + +**Required:** [[Python]] · [[LangGraph]] · [[MCP]] · [[Sub-agents]] +**Nice to have:** [[FastAPI]] · [[Pydantic AI]] +**Matched:** [[Python]] · [[MCP]] +**Missing:** [[LangGraph]] · [[Sub-agents]] + +## Full JD + +...full JD text... +``` + +Sections render `[[Python]]` as a clickable wikilink in Obsidian. Empty categories (e.g. no nice-to-have skills) should be omitted from the rendered output, not shown as "Nice to have: (none)". + +### Migration script behavior + +`scripts/migrate_jobnote_skills_section.py`: +- Walks `~/Documents/compass-vault/jobs/*.md` +- For each, reads frontmatter `skills_required`, `skills_nice_to_have`, `skills_matched`, `skills_missing` +- Replaces existing `## Skills` section if present, OR inserts before `## Full JD` if not +- Dry-run by default (lists changes); `--apply` to commit +- Same pattern as `scripts/migrate_role_family.py` + +### Test cases + +```python +def test_jobnote_body_has_skills_section(temp_vault): + note = JobNote( + company="Test", title="Engineer", url="x://1", source="manual", + date_found=date.today(), match_score=4.0, + skills_required=["Python", "LangGraph"], + skills_nice_to_have=["FastAPI"], + skills_matched=["Python"], + skills_missing=["LangGraph"], + ) + path = write_job_note(note) + body = path.read_text() + assert "## Skills" in body + assert "[[Python]]" in body + assert "[[LangGraph]]" in body + assert "[[FastAPI]]" in body + + +def test_jobnote_body_omits_empty_skill_categories(temp_vault): + note = JobNote( + company="Test", title="Engineer", url="x://2", source="manual", + date_found=date.today(), match_score=2.0, + skills_required=["Python"], + skills_nice_to_have=[], # empty + skills_matched=[], # empty + skills_missing=["Python"], + ) + path = write_job_note(note) + body = path.read_text() + assert "**Nice to have:**" not in body # empty category omitted + assert "**Matched:**" not in body + assert "**Missing:** [[Python]]" in body +``` + +### Execution + +1. Write the implementation plan via `superpowers:writing-plans` (this is small enough that a tight plan with 4-5 task steps is fine; doesn't need full multi-task plan structure) +2. One adversarial review pass (general-purpose agent, grep against the writer code) +3. Implement (subagent-driven if helpful; or inline since this is small) +4. Run dry-run migration: `uv run python -m scripts.migrate_jobnote_skills_section` +5. Inspect output; verify 23 JobNotes would be updated cleanly +6. Apply: `uv run python -m scripts.migrate_jobnote_skills_section --apply` +7. Open one JobNote in Obsidian, confirm graph view shows new edges +8. Commit + +### Definition of done for Day 1 + +- `compass/vault/writer.py` modified to render `## Skills` section with wikilinks +- 2 unit tests passing +- 23 existing JobNotes backfilled via migration +- Open Obsidian → click `[[Python]]` on any JobNote → lands on `skills/Python.md` and sees Linked Mentions populated +- Full test suite: 259+ passing +- Ruff clean +- Committed + +**Estimated time: 3 hours including the plan + review cycle.** + +--- + +## Day 2 — Obsidian P1 (continued) + P2 + P3 + +Once Day 1 is committed: + +1. **SkillNote body update**: extend `compass/analysis/gap_aggregator._sync_skill_counters` to also write a `## Jobs requiring this skill` section to each SkillNote body. **Preserve existing body content** (the seeded notes have descriptions like `_Category: mcp · Tier-2 demand: highest_`). Insert or replace just the `## Jobs requiring this skill` block. + + Format: + ```markdown + ## Jobs requiring this skill + + - [[2026-03-27-sierra-Software_Engineer_Agent_Architecture-04a9b65f|Sierra — Software Engineer, Agent Architecture]] · score 3.0 · apply-now + - [[2026-05-06-cognition-Software_Engineer-7c26fdaf|Cognition — Software Engineer]] · score 3.0 · apply-now + ``` + +2. **Dashboard panels** (`~/Documents/compass-vault/dashboard.md`): add two Dataview blocks. See `docs/OBSIDIAN_LEVERAGE.md` P2 for the exact queries. + +3. **Auto-tags on JobNote frontmatter** (`compass/pipeline/nodes/vault_write.py`): generate `#tier/apply-now`, `#fit/strong`, `#role/agent-engineer`, `#decision/approved` style tags. Backfill the 23 existing JobNotes via the existing migration pattern. + +4. Tests for each. + +5. Tag `phase-1b2-obsidian-p1` when all P1+P2+P3 work shipped. + +--- + +## Day 3+ planning + +When you reach each phase, write the per-phase implementation plan using `superpowers:writing-plans`: + +- Day 3-5: `docs/superpowers/plans/2026-05-22-compass-phase-1b3-automation.md` +- Day 7-9: `docs/superpowers/plans/2026-05-26-compass-phase-2a-eval-harness.md` +- Day 10-12: `docs/superpowers/plans/2026-05-29-compass-phase-2b-portfolio-polish.md` +- Day 13-14: `docs/superpowers/plans/2026-06-01-compass-phase-2c-blog-post.md` + +Each plan needs 2 adversarial review passes minimum. Compress from the previous 3-6 pattern; accept slightly higher residual risk for the 2-week timeline. + +--- + +## What you SHOULD NOT do + +- **Don't add scope.** When tempted to add "just one more Obsidian feature" or "one more JD to label" or "one more scraper," stop. Stick to the plan. +- **Don't touch the deferred B1-B3 LLM bugs ad-hoc.** Wait for Phase 2.A's eval harness. Ad-hoc prompt tuning without measurement is whack-a-mole. +- **Don't refactor "while you're in there."** If a piece of code is ugly but works, leave it. Phase 2.B's polish phase handles cosmetic improvements. +- **Don't create more brainstorm/design docs.** Two Obsidian leverage docs exist. The implementation plans are per-phase, not master docs. +- **Don't relitigate decisions.** The 2-week plan is the plan. Within each phase, the per-phase plan is the plan. Execute. +- **Don't bypass the adversarial review step.** It's compressed from 3-6 passes to 2, but not zero. The pattern of "rubber-stamp review finds 0 bugs; adversarial review finds 5" repeats every phase. + +--- + +## What to do when something goes wrong + +### Tests fail unexpectedly +- Check `uv run pytest --tb=short` for the failure mode +- Test isolation leak? Use `monkeypatch.setattr` not raw `module._var = ...` (see commit `c4c7fe8`) +- Schema validation failure? Some new field needed in `compass/vault/schemas.py`? + +### Pipeline run fails +- Check `~/Documents/compass-vault/_meta/agent-log.md` for the error +- Check `~/Documents/compass-vault/_meta/pipeline-runs.md` for the run summary row +- Langfuse callback errors are expected pre-Day-3; ignore them + +### A subagent reports DONE but you don't trust it +- Run the actual verification commands yourself +- `git diff` to see the real changes +- `uv run pytest -q` to confirm tests pass +- Inspect real vault output if the change affects vault state + +### You hit a langgraph / chromadb / pydantic-ai library quirk +- The lesson from Phases 1.B.1 + 1.B.2: probe empirically before assuming. Five plan-review iterations on the msgpack API surfaced the real working form. +- If a documented API doesn't work, run a one-liner probe with `uv run python -c "..."` to find the actual contract. + +### You realize you missed a bug class during review +- Document it in `docs/KNOWN_DATA_QUALITY_ISSUES.md` with severity + fix surface +- If P0 (blocks current phase): stop and fix +- If lower: continue; fix in Phase 2.A or post-launch + +--- + +## Resources + +- **Phase 1.B.2 plan** (for plan format calibration): `docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md` +- **Phase 1.B.1 plan**: `docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md` +- **Master spec**: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- **CLAUDE.md**: `/Users/akmini/Documents/compass/CLAUDE.md` +- **The user's vault**: `~/Documents/compass-vault/` +- **The user's learning vault**: `~/Documents/learning-vault/` (read-only via `learning-vault://` URIs) + +--- + +## Pre-flight check before Day 1 + +Run this: + +```bash +cd /Users/akmini/Documents/compass +git status # expect: clean +git branch --show-current # expect: phase-1b2-rag +git log --oneline -5 # confirm: 19bcf16 is HEAD +uv run pytest -q # expect: 259 passed +uv run ruff check # expect: All checks passed +ls ~/Documents/compass-vault/jobs/ | wc -l # expect: 23 +``` + +If all green: start Day 1. + +If any red: STOP. Diagnose before proceeding. + +--- + +## Communication first interaction with the user + +When the user opens a fresh chat and you have this doc, your first message should be something like: + +> Read the handoff. State: `phase-1b2-rag` HEAD `19bcf16`, 259 tests pass, vault coherent. Day 1 starts with Obsidian Part 1 — JobNote body wikilinks + migration. Estimated 3 hours. +> +> Want me to write the Day 1 implementation plan and start, or do you want to review the handoff first? + +Don't ask permission for things the doc already authorized. Don't restate the plan. Get to work. + +--- + +## Honest closing note + +The codebase is in good shape entering this sprint. 7+ adversarial review cycles. Known issues documented. Tests green. No surprises waiting. + +The biggest risk over the next 14 days is **scope creep**, not technical debt. Hold the line on this plan. Ship daily. Let imperfections in adjacent features stay until Phase 2.B polish or post-launch iteration. + +Two weeks from now: a working agentic career coach in your daily workflow + a public repo + a published blog post + measurable eval baselines. That's the bar. + +Go. diff --git a/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md new file mode 100644 index 0000000..b95ed27 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md @@ -0,0 +1,2236 @@ +# Compass Phase 1.B.1 — Real HiTL via `interrupt()` + `AsyncSqliteSaver` (Implementation Plan) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the Phase 0.B/1.A auto-approve `hitl_node` with a real LangGraph `interrupt()`-based human approval flow, backed by `AsyncSqliteSaver` checkpointing and a SQLite-backed pending-approval queue. Add MCP tools (`pending_approvals`, `approve`) so the human approves/rejects scored-but-pre-tailor jobs from Claude Code or Cursor. Add a `timeout_checker` module that any cron (Modal cron lands in 1.B.3) can call to auto-cancel approvals older than `HITL_TIMEOUT_HOURS`. End state: above-threshold jobs pause at `hitl`, sit in `pending_approvals()` until a human acts, then resume into `tailor` + `vault_write` (or skip tailor on reject). Below-threshold jobs continue to auto-reject the same way they do today — no interrupt, no human prompt — so the cost of running the pipeline is unchanged for the long tail. + +**Architecture:** Three new modules under `compass/hitl/`: `state_store.py` (aiosqlite-backed pending queue, one row per paused thread), `resume.py` (re-opens the checkpointer + recompiles the graph + calls `graph.ainvoke(Command(resume=...), config={"configurable": {"thread_id": ...}})`), `timeout_checker.py` (polls the queue, resumes timed-out threads with `{"approved": False}`). `hitl_node` becomes a one-line `interrupt(...)` call wrapped with a below-threshold short-circuit. `run_pipeline()` is restructured so the `AsyncSqliteSaver` is opened once per batch (`async with` block), the graph is compiled inside it, and the per-job invocation generates a deterministic `thread_id` from the job URL + scrape timestamp. After every batch, the orchestrator detects which jobs paused (via `__interrupt__` in returned state, registering them in the state store) versus which ran to completion. New MCP tools (`pending_approvals`, `approve`) wrap the resume entrypoint. No changes to `extract` / `score` / `reflect` / `tailor` / `vault_write` node bodies. + +**Module-level discipline (carried forward from Phase 1.A):** any module that touches `VAULT_PATH`, `HITL_STATE_DB`, or `HITL_TIMEOUT_HOURS` must reference it via `compass.config.<NAME>` *inside the function body*, never as a module-level captured constant. The `temp_vault` and new `temp_hitl_db` pytest fixtures monkeypatch `compass.config`, which only affects code that re-reads the attribute each call. + +**Tech Stack:** Python 3.12 · langgraph 1.2 (`langgraph.types.interrupt`, `langgraph.types.Command`) · langgraph-checkpoint-sqlite 3.1 (`langgraph.checkpoint.sqlite.aio.AsyncSqliteSaver`) · aiosqlite 0.20 (for the pending-approval queue, kept separate from the checkpoint DB) · pytest + pytest-asyncio (`asyncio_mode = "auto"` already set). + +**Authoritative spec:** `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` § Phase 1.B +**Previous-phase handoff:** `docs/PHASE_1A_COMPLETE.md` + +**Closes these deferred items from Phase 1.A:** +1. Real `interrupt()` + `AsyncSqliteSaver` checkpointing (P1.A handoff "What's deferred", row 1) +2. The `_route_after_hitl` three-way comment in `graph.py:48–58` that already anticipates this flow +3. The `HITL_STATE_DB` + `HITL_TIMEOUT_HOURS` env vars in `compass/config.py:46–47` and `.env.example:30–31` (defined for 1.B; used here for the first time) + +**Does NOT touch in this phase:** +- RAG / Chroma — deferred to **Phase 1.B.2** +- Modal cron decorators (`modal_app.py`, `@app.function(schedule=Cron(...))`) — deferred to **Phase 1.B.3**. This phase ships `timeout_checker.py` as a callable module + plain CLI entrypoint (`python -m compass.hitl.timeout_checker`) so the 1.B.3 Modal cron just imports it. +- Langfuse callback API mismatch (bug #23) — deferred to **Phase 1.B.3** observability work +- `extract` / `score` / `reflect` / `tailor` / `vault_write` node bodies +- `compass/llm.py`, taxonomy, schemas, applications lifecycle + +--- + +## File Structure + +### New +- `compass/hitl/state_store.py` — async aiosqlite store of pending approvals; pure data access +- `compass/hitl/resume.py` — `resume_pending(thread_id, decision, feedback)` reopens the checkpointer, recompiles the graph, drives the resume +- `compass/hitl/timeout_checker.py` — `check_and_resume_timeouts()` plus `__main__` CLI entrypoint +- `compass/hitl/__init__.py` — re-export the public surface + +### Test scaffolding (new) +- `tests/hitl/__init__.py` +- `tests/hitl/conftest.py` — `temp_hitl_db` fixture (monkeypatch `compass.config.HITL_STATE_DB` to a tmp path) and `frozen_now` fixture +- `tests/hitl/test_state_store.py` +- `tests/hitl/test_resume.py` +- `tests/hitl/test_timeout_checker.py` +- `tests/pipeline/test_hitl_node_interrupt.py` — replaces (does not extend) the auto-approve tests in `tests/pipeline/test_hitl_node.py` +- `tests/pipeline/test_graph_checkpointing.py` — end-to-end: invoke graph, confirm pause; resume via `resume_pending`; confirm tailor + vault_write fire +- `tests/mcp_server/test_pending_approvals.py` + +### Modify +- `compass/pipeline/nodes/hitl.py` — replace auto-approve body with `interrupt(...)` + below-threshold short-circuit +- `compass/vault/schemas.py` — tighten `JobNote.hitl_decision` from `str | None` to `Literal["approved","rejected","auto_rejected","timed_out"] | None` (Task 2.5) +- `compass/pipeline/nodes/vault_write.py` — populate `hitl_decision` + `hitl_at` on the JobNote from `state["human_approved"]` so the vault carries the audit trail +- `compass/pipeline/graph.py` — open `AsyncSqliteSaver` once per `run_pipeline` invocation; compile graph inside the `async with`; generate deterministic `thread_id` per job; detect paused vs completed jobs; register paused jobs in the state store +- `compass/pipeline/state.py` — add `thread_id: str | None` to `CompassState` (so `hitl_node` and `vault_write_node` can read it from `RunnableConfig` and surface it in logs/state) +- `compass/mcp_server/server.py` — add `pending_approvals()` and `approve()` tools +- `compass/config.py` — `HITL_CHECKPOINT_DB` env var (defaults to `~/.compass/checkpoints.db`); keep `HITL_STATE_DB` (pending queue) and `HITL_TIMEOUT_HOURS` as-is +- `.env.example` — document `HITL_CHECKPOINT_DB` +- `tests/pipeline/test_routing.py` — remove the three existing `hitl_node` auto-approve tests (lines 46-66); the new behaviour is covered by `test_hitl_node_interrupt.py` +- `tests/pipeline/test_graph_integration.py` — add an `auto_approve_hitl` fixture (monkeypatches `compass.pipeline.nodes.hitl.interrupt` to a stub returning `{"approved": True}`); opt the three above-threshold integration tests into it so they still exercise the full pipeline without needing real checkpoint+resume + +### Untouched +- `compass/pipeline/nodes/{intake,intake_filter,extract,score,reflect,tailor,vault_write}.py` +- `compass/pipeline/role_family.py` +- `compass/vault/*`, `compass/scrapers/*`, `compass/applications/*`, `compass/analysis/*`, `compass/llm.py` +- All Phase 1.A tests (must still pass) + +### Decomposition rationale +The pending-approval queue (`state_store.py`) is intentionally separate from the LangGraph checkpoint DB. The checkpoint DB is LangGraph-owned (schema controlled by `AsyncSqliteSaver`) and stores graph node state; the pending queue is *our* schema, optimized for `pending_approvals()` (we don't need to deserialize a full checkpoint just to list pending jobs by company + title + score). Two SQLite files in `~/.compass/`. The `resume.py` module is the only place that mounts the checkpointer for resumes — keeping it focused makes it the single audit point for "did we recompile the graph correctly with the checkpointer". `timeout_checker.py` calls `resume_pending` rather than reimplementing resume, so the timeout path and the human path go through identical code. + +--- + +## Task 0: Pre-flight + +**Files:** none + +- [ ] **Step 1: Verify clean tree on `phase-1a-application-tracking` tag** + +```bash +cd ~/Documents/compass +git status # expected: clean +git describe --tags --abbrev=0 # expected: phase-1a-application-tracking +uv run pytest -q # expected: 204 passed +uv run ruff check # expected: All checks passed +``` + +If working tree is dirty, STOP and ask the user before proceeding. + +- [ ] **Step 2: Confirm LangGraph + checkpointer libs are at expected versions** + +```bash +uv run python -c " +import importlib.metadata as m +print('langgraph', m.version('langgraph')) +print('langgraph-checkpoint-sqlite', m.version('langgraph-checkpoint-sqlite')) +from langgraph.types import interrupt, Command # smoke +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver # smoke +" +``` + +Expected: `langgraph >= 1.2`, `langgraph-checkpoint-sqlite >= 3.1`, both imports succeed. If lower, ask the user before bumping — version skew bit bug #4 in Phase 0. + +- [ ] **Step 3: Create branch** + +```bash +git checkout -b phase-1b1-hitl +``` + +--- + +## Task 1: Pending-approval state store (`compass/hitl/state_store.py`) + +**Why first:** Pure data layer with no LangGraph dependency. We can TDD it without touching the graph, then plug it in from the orchestrator in Task 3. + +**Schema:** + +```sql +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, -- JSON array + missing_skills TEXT NOT NULL, -- JSON array + created_at TEXT NOT NULL, -- ISO8601 UTC + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +``` + +**Files:** +- Create: `compass/hitl/__init__.py` (empty for now) +- Create: `compass/hitl/state_store.py` +- Create: `tests/hitl/__init__.py` +- Create: `tests/hitl/conftest.py` +- Create: `tests/hitl/test_state_store.py` + +- [ ] **Step 1: Write the `temp_hitl_db` fixture** + +Create `tests/hitl/conftest.py`: + +```python +"""Shared fixtures for HiTL tests.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + + +@pytest.fixture +def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point HITL_STATE_DB at a fresh per-test SQLite file. + + The store reads `compass.config.HITL_STATE_DB` inside function bodies (per + the module-level discipline rule), so monkeypatching the attribute is + sufficient — no module reimport needed. + """ + db = tmp_path / "pending.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_STATE_DB", db) + return db + + +@pytest.fixture +def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: + """Freeze the wall clock the state store uses.""" + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.timezone.utc) + + class _FrozenDT(_dt.datetime): + @classmethod + def now(cls, tz=None): + return fixed if tz is None else fixed.astimezone(tz) + + import compass.hitl.state_store as ss + monkeypatch.setattr(ss, "_now", lambda: fixed) + return fixed +``` + +- [ ] **Step 2: Write the failing tests** + +Create `tests/hitl/test_state_store.py`: + +```python +"""state_store CRUD: add/get/list/resolve + status transitions + serialisation.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.hitl import state_store + + +pytestmark = pytest.mark.usefixtures("temp_hitl_db") + + +async def _add_one(thread_id: str = "tid-1", **overrides) -> None: + defaults = dict( + thread_id=thread_id, + job_url="https://jobs.example.com/abc", + company="Sierra", + title="Software Engineer, Agent", + score=4.2, + score_reasoning="Strong match on MCP + LangGraph.", + matched_skills=["MCP", "Python"], + missing_skills=["LangGraph"], + ) + defaults.update(overrides) + await state_store.add_pending(**defaults) + + +async def test_add_and_get_round_trips(frozen_now): + await _add_one() + row = await state_store.get_pending("tid-1") + assert row is not None + assert row["thread_id"] == "tid-1" + assert row["company"] == "Sierra" + assert row["score"] == pytest.approx(4.2) + assert row["matched_skills"] == ["MCP", "Python"] + assert row["missing_skills"] == ["LangGraph"] + assert row["status"] == "pending" + assert row["created_at"] == frozen_now.isoformat() + assert row["resolved_at"] is None + assert row["feedback"] is None + + +async def test_add_pending_is_idempotent_on_thread_id(): + """Re-running a pipeline that re-pauses the same thread_id is a no-op, not a crash.""" + await _add_one() + # Second call with same thread_id but different score should be a no-op + # (we don't overwrite — the resume must use the original checkpoint). + await _add_one(score=2.0) + row = await state_store.get_pending("tid-1") + assert row["score"] == pytest.approx(4.2) + + +async def test_list_pending_only_returns_pending_status(): + await _add_one(thread_id="tid-pending") + await _add_one(thread_id="tid-approved") + await state_store.mark_resolved("tid-approved", status="approved") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-pending"] + + +async def test_list_pending_orders_oldest_first(): + """The MCP UI shows the queue oldest-first so things don't get lost.""" + import compass.hitl.state_store as ss + # First insertion at frozen_now + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.timezone.utc) + ss._now = lambda: fixed + await _add_one(thread_id="tid-old") + ss._now = lambda: fixed + _dt.timedelta(minutes=5) + await _add_one(thread_id="tid-new") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-old", "tid-new"] + + +async def test_mark_resolved_records_status_and_timestamp(frozen_now): + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="LGTM") + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + assert row["resolved_at"] == frozen_now.isoformat() + + +async def test_mark_resolved_rejects_unknown_status(): + await _add_one() + with pytest.raises(ValueError, match="status"): + await state_store.mark_resolved("tid-1", status="bogus") + + +async def test_mark_resolved_unknown_thread_raises(): + with pytest.raises(LookupError): + await state_store.mark_resolved("tid-missing", status="approved") + + +async def test_list_timed_out_returns_only_old_pending(frozen_now): + import compass.hitl.state_store as ss + # Insert one row "5 hours ago", one "1 hour ago" + old = frozen_now - _dt.timedelta(hours=5) + young = frozen_now - _dt.timedelta(hours=1) + ss._now = lambda: old + await _add_one(thread_id="tid-old") + ss._now = lambda: young + await _add_one(thread_id="tid-young") + ss._now = lambda: frozen_now + rows = await state_store.list_timed_out(timeout_hours=4) + assert [r["thread_id"] for r in rows] == ["tid-old"] + + +async def test_get_pending_unknown_returns_none(): + assert await state_store.get_pending("nope") is None +``` + +Run: + +```bash +uv run pytest tests/hitl/test_state_store.py -v +``` + +Expected: every test fails with `ModuleNotFoundError: compass.hitl.state_store` or `AttributeError`. + +- [ ] **Step 3: Implement `state_store.py`** + +Create `compass/hitl/state_store.py`: + +```python +"""Pending-approval queue for the HiTL flow — aiosqlite, separate from the +LangGraph checkpoint DB. + +Public surface (all coroutines): + add_pending(thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills) + get_pending(thread_id) -> row | None + list_pending() -> list[row] # oldest first + list_timed_out(timeout_hours) -> list[row] # pending AND older than cutoff + mark_resolved(thread_id, status, feedback=None) + +A "row" is a plain dict with the schema documented in the implementation plan. +matched_skills / missing_skills are JSON-encoded in the DB and decoded on read. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +from typing import Any + +import aiosqlite + +logger = logging.getLogger(__name__) + +_VALID_STATUSES = {"pending", "approved", "rejected", "timed_out", "error"} + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS pending_approvals ( + thread_id TEXT PRIMARY KEY, + job_url TEXT NOT NULL, + company TEXT NOT NULL, + title TEXT NOT NULL, + score REAL NOT NULL, + score_reasoning TEXT NOT NULL, + matched_skills TEXT NOT NULL, + missing_skills TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','approved','rejected','timed_out','error')), + resolved_at TEXT, + feedback TEXT +); +CREATE INDEX IF NOT EXISTS idx_pending_status_created + ON pending_approvals(status, created_at); +""" + + +def _now() -> _dt.datetime: + """Wall clock as UTC-aware. Indirected so tests can freeze it.""" + return _dt.datetime.now(_dt.timezone.utc) + + +def _db_path(): + """Late-bound HITL_STATE_DB lookup — see module-level discipline rule.""" + import compass.config as cfg + cfg.HITL_STATE_DB.parent.mkdir(parents=True, exist_ok=True) + return cfg.HITL_STATE_DB + + +async def _connect() -> aiosqlite.Connection: + conn = await aiosqlite.connect(_db_path()) + conn.row_factory = aiosqlite.Row + # WAL mode lets concurrent readers + a single writer coexist without + # exclusive-lock contention. Phase 1.B.3 Modal cron + a human pressing + # `approve` in MCP will race on this file; cheaper to set the pragma now + # than debug a flaky lock-timeout in production. PRAGMA is per-DB and + # persists across opens. + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA busy_timeout=5000") # 5s retry on transient lock + await conn.executescript(_SCHEMA) + await conn.commit() + return conn + + +def _row_to_dict(row: aiosqlite.Row) -> dict[str, Any]: + d = dict(row) + d["matched_skills"] = json.loads(d["matched_skills"]) + d["missing_skills"] = json.loads(d["missing_skills"]) + return d + + +async def add_pending( + *, + thread_id: str, + job_url: str, + company: str, + title: str, + score: float, + score_reasoning: str, + matched_skills: list[str], + missing_skills: list[str], +) -> None: + """Insert a new pending row. INSERT OR IGNORE — re-pausing the same thread_id is a no-op.""" + async with await _connect() as conn: + await conn.execute( + """ + INSERT OR IGNORE INTO pending_approvals + (thread_id, job_url, company, title, score, score_reasoning, + matched_skills, missing_skills, created_at, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending') + """, + ( + thread_id, + job_url, + company, + title, + float(score), + score_reasoning, + json.dumps(matched_skills), + json.dumps(missing_skills), + _now().isoformat(), + ), + ) + await conn.commit() + + +async def get_pending(thread_id: str) -> dict[str, Any] | None: + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + row = await cur.fetchone() + return _row_to_dict(row) if row else None + + +async def list_pending() -> list[dict[str, Any]]: + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals WHERE status = 'pending' " + "ORDER BY created_at ASC" + ) as cur: + rows = await cur.fetchall() + return [_row_to_dict(r) for r in rows] + + +async def list_timed_out(*, timeout_hours: int) -> list[dict[str, Any]]: + cutoff = (_now() - _dt.timedelta(hours=timeout_hours)).isoformat() + async with await _connect() as conn: + async with conn.execute( + "SELECT * FROM pending_approvals " + "WHERE status = 'pending' AND created_at < ? " + "ORDER BY created_at ASC", + (cutoff,), + ) as cur: + rows = await cur.fetchall() + return [_row_to_dict(r) for r in rows] + + +async def mark_resolved( + thread_id: str, + *, + status: str, + feedback: str | None = None, +) -> None: + if status not in _VALID_STATUSES or status == "pending": + raise ValueError(f"invalid resolve status: {status!r}") + async with await _connect() as conn: + async with conn.execute( + "SELECT 1 FROM pending_approvals WHERE thread_id = ?", (thread_id,) + ) as cur: + if not await cur.fetchone(): + raise LookupError(f"no pending row for thread_id {thread_id!r}") + await conn.execute( + "UPDATE pending_approvals " + "SET status = ?, feedback = ?, resolved_at = ? " + "WHERE thread_id = ?", + (status, feedback, _now().isoformat(), thread_id), + ) + await conn.commit() +``` + +- [ ] **Step 4: Run the tests, confirm pass** + +```bash +uv run pytest tests/hitl/test_state_store.py -v +``` + +Expected: 9 passed. + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/__init__.py compass/hitl/state_store.py tests/hitl/__init__.py \ + tests/hitl/conftest.py tests/hitl/test_state_store.py +git commit -m "feat(hitl): aiosqlite-backed pending-approval state store" +``` + +--- + +## Task 2: Rewire `hitl_node` to call `interrupt()` + +**Files:** +- Modify: `compass/pipeline/nodes/hitl.py` +- Replace: `tests/pipeline/test_hitl_node.py` → `tests/pipeline/test_hitl_node_interrupt.py` + +**Design:** + +``` +hitl_node(state): + score = state.get("score_result") + if score is None or score.score < SCORE_THRESHOLD: + # auto-reject for missing / low-score — exactly the 1.A behaviour. + # No interrupt fires; the orchestrator never sees this thread. + return {"human_approved": False} + + decision = interrupt({ + "kind": "approval_request", + "job_url": state["current_job"].url, + "company": state["current_job"].company, + "title": state["current_job"].title, + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": score.matched_skills, + "missing_skills": score.missing_skills, + }) + + # On resume, `decision` is whatever was passed to Command(resume=...) + if not isinstance(decision, dict): + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } +``` + +The `interrupt(...)` payload above is what the orchestrator captures (via the `__interrupt__` marker on the returned graph state) to populate `state_store.add_pending(...)`. + +- [ ] **Step 1: Remove the three old auto-approve tests from `test_routing.py`** + +The auto-approve tests live in `tests/pipeline/test_routing.py:46-66` (NOT in a standalone `test_hitl_node.py`). Delete the three test functions: + +- `test_hitl_node_approves_when_score_meets_threshold` +- `test_hitl_node_rejects_when_score_below_threshold` +- `test_hitl_node_handles_missing_score` + +…along with their shared `_state(score)` helper if it's no longer referenced elsewhere in the file. The `reflect_node` tests in the same file stay. Confirm with: + +```bash +grep -n "hitl_node\|SCORE_THRESHOLD" tests/pipeline/test_routing.py +``` + +Expected: no matches after the deletion. If `_state` is still used by reflect tests, keep it; otherwise delete it too. + +- [ ] **Step 2: Write the new interrupt-based tests** + +Create `tests/pipeline/test_hitl_node_interrupt.py`: + +```python +"""hitl_node calls interrupt() above threshold; auto-rejects below threshold.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.pipeline.nodes.hitl import hitl_node +from compass.pipeline.state import CompassState, JobScore, RawJob + + +def _state(score: float | None) -> CompassState: + job = RawJob( + company="Sierra", + title="SWE, Agent", + url="https://jobs.example.com/sierra-1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + sr = ( + None + if score is None + else JobScore( + score=score, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="", + ) + ) + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": sr, + "in_scope": True, + "role_family": "agent-engineer", + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": "tid-test", + } + + +async def test_below_threshold_auto_rejects_without_interrupt(monkeypatch): + """Below SCORE_THRESHOLD short-circuits — interrupt MUST NOT fire (no human prompt cost).""" + called = {"interrupt": 0} + + def boom(_payload): + called["interrupt"] += 1 + raise AssertionError("interrupt should not have been called") + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", boom) + result = await hitl_node(_state(score=2.0)) + assert result == {"human_approved": False} + assert called["interrupt"] == 0 + + +async def test_missing_score_auto_rejects(): + result = await hitl_node(_state(score=None)) + assert result == {"human_approved": False} + + +async def test_above_threshold_calls_interrupt_with_payload(monkeypatch): + captured = {} + + def fake_interrupt(payload): + captured.update(payload) + # Simulate resume value (this is what Command(resume=...) sends back) + return {"approved": True, "feedback": "Strong fit"} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + result = await hitl_node(_state(score=4.2)) + assert captured["kind"] == "approval_request" + assert captured["company"] == "Sierra" + assert captured["score"] == pytest.approx(4.2) + assert captured["matched_skills"] == ["MCP"] + assert result == {"human_approved": True, "human_feedback": "Strong fit"} + + +async def test_resume_rejection_propagates(monkeypatch): + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda _p: {"approved": False, "feedback": "Not a fit"}, + ) + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False, "human_feedback": "Not a fit"} + + +async def test_malformed_resume_value_defaults_to_rejected(monkeypatch): + """If Command(resume=...) somehow sends a non-dict, treat as reject not crash.""" + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", lambda _p: "bogus") + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False} +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_hitl_node_interrupt.py -v +``` + +Expected: all fail (interrupt symbol not imported in hitl.py yet). + +- [ ] **Step 3: Implement the new `hitl_node`** + +Replace `compass/pipeline/nodes/hitl.py` entirely: + +```python +"""hitl_node — Phase 1.B.1 real human-in-the-loop via LangGraph interrupt(). + +Behaviour: + • score < SCORE_THRESHOLD or score missing -> auto-reject, no interrupt fires + • score >= SCORE_THRESHOLD -> interrupt() with the approval payload; the + orchestrator catches the interrupt and registers the thread in + compass.hitl.state_store. A human resumes via Command(resume={...}). +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from langgraph.types import interrupt + +from compass.config import SCORE_THRESHOLD + +if TYPE_CHECKING: + from compass.pipeline.state import CompassState + +logger = logging.getLogger(__name__) + + +async def hitl_node(state: "CompassState") -> dict: + score = state.get("score_result") + job = state.get("current_job") + + if score is None or score.score < SCORE_THRESHOLD: + logger.info( + "hitl: auto-reject %s (score=%s, threshold=%.2f)", + job.url if job else "(unknown)", + getattr(score, "score", None), + SCORE_THRESHOLD, + ) + return {"human_approved": False} + + payload = { + "kind": "approval_request", + "job_url": job.url if job else "", + "company": job.company if job else "", + "title": job.title if job else "", + "score": score.score, + "score_reasoning": score.reasoning, + "matched_skills": list(score.matched_skills), + "missing_skills": list(score.missing_skills), + } + logger.info("hitl: interrupting for approval — %s (score=%.2f)", payload["job_url"], score.score) + decision = interrupt(payload) + + if not isinstance(decision, dict): + logger.warning("hitl: malformed resume value %r — defaulting to reject", decision) + return {"human_approved": False} + return { + "human_approved": bool(decision.get("approved", False)), + "human_feedback": decision.get("feedback"), + } +``` + +- [ ] **Step 4: Add `thread_id` to `CompassState`** + +Edit `compass/pipeline/state.py`. Add inside the `CompassState` TypedDict, after `errors: list[str]`: + +```python + thread_id: str | None +``` + +- [ ] **Step 5: Update the three state-dict literals to include `thread_id`** + +Three places: `compass/pipeline/graph.py:~120` (`_initial_state`), `compass/pipeline/graph.py:~190` (aggregate), `compass/mcp_server/server.py:~82` (the `score_jd` ad-hoc state). Confirm with `grep -n '"raw_jobs"' compass/` — there should be exactly three module-level matches. + +For each: add `"thread_id": None,` to the literal. (We'll set a real value in Task 3.) + +- [ ] **Step 6: Run the new hitl tests** + +```bash +uv run pytest tests/pipeline/test_hitl_node_interrupt.py -v +``` + +Expected: 5 passed. + +- [ ] **Step 7: Add the `auto_approve_hitl` fixture for existing integration tests** + +`tests/pipeline/test_graph_integration.py` has three tests that stub `score=4.2` (above threshold) and expect the pipeline to run to completion: `test_run_pipeline_end_to_end`, `test_run_pipeline_regenerates_gap_plan`, `test_run_pipeline_appends_to_run_log`. With real `interrupt()` they will now pause and the assertions on `jobs_written`, `tailored_paragraph`, etc. will fail. + +Add this fixture at the top of `tests/pipeline/test_graph_integration.py` (next to `mocked_llms`): + +```python +@pytest.fixture +def auto_approve_hitl(monkeypatch): + """Stub the `interrupt()` call in hitl_node so the integration tests can + exercise the full extract -> score -> hitl -> tailor -> vault_write path + without needing a real human resume.""" + def fake_interrupt(_payload): + return {"approved": True, "feedback": None} + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) +``` + +Then add `auto_approve_hitl` to the parameter list of each above-threshold test: + +```python +async def test_run_pipeline_end_to_end(temp_vault, mocked_llms, auto_approve_hitl): +async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms, auto_approve_hitl): +async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms, auto_approve_hitl): +``` + +Leave `test_run_pipeline_skips_tailor_when_below_threshold_but_still_writes` alone — it uses `score=2.0`, never reaches `interrupt()`. + +Leave `test_run_pipeline_skips_dedup_urls` alone — it dedup-drops the job before the graph runs. + +- [ ] **Step 8: Run the full suite — Phase 1.A tests must still pass** + +```bash +uv run pytest -q +``` + +Expected: 204 (pre-existing) − 3 (deleted auto-approve tests in `test_routing.py`) + 5 (new `test_hitl_node_interrupt.py`) = **206 passed**. + +If anything else fails: a state-literal somewhere is missing `thread_id` — find it and add `"thread_id": None,`. Or an above-threshold integration test was missed in Step 7 — opt it into `auto_approve_hitl`. + +- [ ] **Step 9: Commit** + +```bash +git add compass/pipeline/nodes/hitl.py compass/pipeline/state.py \ + compass/pipeline/graph.py compass/mcp_server/server.py \ + tests/pipeline/test_hitl_node_interrupt.py \ + tests/pipeline/test_routing.py \ + tests/pipeline/test_graph_integration.py +git commit -m "feat(hitl): replace auto-approve with LangGraph interrupt()" +``` + +--- + +## Task 2.5: Populate `hitl_decision` + `hitl_at` on the JobNote (audit trail) + +**Why:** `JobNote` has `hitl_decision: str | None` and `hitl_at: datetime | None` fields ([compass/vault/schemas.py:62-63](compass/vault/schemas.py:62)) that have been unpopulated since Phase 0. The whole point of HiTL is that the vault becomes the audit trail for human decisions — without these fields populated, an Obsidian reader can't tell whether a JobNote was approved, rejected, or auto-rejected for low score. + +**Mapping:** +- `state["human_approved"] is True` → `hitl_decision = "approved"` +- `state["human_approved"] is False` and score < threshold → `hitl_decision = "auto_rejected"` +- `state["human_approved"] is False` and score ≥ threshold → `hitl_decision = "rejected"` (human said no on resume) +- `state["human_approved"] is False` and feedback startswith `"auto-cancelled after"` → `hitl_decision = "timed_out"` (timeout_checker path) +- `hitl_at = datetime.now()` whenever any decision was made (i.e. always, except the never-reached-hitl branch which doesn't go through vault_write anyway) + +**Files:** +- Modify: `compass/pipeline/nodes/vault_write.py` +- Modify: `tests/pipeline/test_vault_write.py` + +- [ ] **Step 1: Tighten the `JobNote.hitl_decision` type** + +In `compass/vault/schemas.py`, change: + +```python +hitl_decision: str | None = None +``` + +to: + +```python +HitlDecision = Literal["approved", "rejected", "auto_rejected", "timed_out"] +# ... and on JobNote: +hitl_decision: HitlDecision | None = None +``` + +`Literal` is already imported. All existing vault JobNotes have `hitl_decision: null` (never populated since Phase 0), so the tightening doesn't break round-tripping existing notes. + +- [ ] **Step 2: Rename the test helper if it exists; otherwise create it** + +`tests/pipeline/test_vault_write.py` currently has a `_state(skills_required, skills_matched, score=4.2)` helper at the top of the file used by multiple tests. **Rename it to `_build_state_for_score`** and extend the signature: + +```python +def _build_state_for_score( + *, + score: float = 4.2, + skills_required: list[str] | None = None, + skills_matched: list[str] | None = None, + skills_missing: list[str] | None = None, + human_approved: bool = True, + human_feedback: str | None = None, +) -> "CompassState": + ... +``` + +Update the existing test call sites in the file to use the new name + kwargs (mechanical find/replace; tests that called `_state(4.2, [...], [...])` become `_build_state_for_score(score=4.2, skills_required=[...], skills_matched=[...])`). + +- [ ] **Step 3: Write the failing tests** + +Append to `tests/pipeline/test_vault_write.py`: + +```python +async def test_vault_write_records_approved_decision(temp_vault): + """state['human_approved'] = True -> hitl_decision='approved', hitl_at set.""" + import datetime as _dt + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + from compass.pipeline.state import JobScore, JobRequirements, RawJob + + state = _build_state_for_score( # existing helper in this file + score=4.5, + human_approved=True, + human_feedback="LGTM", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "approved" + assert "hitl_at" in md and md["hitl_at"] is not None + + +async def test_vault_write_records_auto_rejected_for_low_score(temp_vault): + """Below-threshold path: hitl never interrupts; decision is 'auto_rejected'.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=2.0, human_approved=False, human_feedback=None) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "auto_rejected" + + +async def test_vault_write_records_rejected_when_human_said_no(temp_vault): + """Above-threshold path with human_approved=False = explicit reject.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=4.2, human_approved=False, human_feedback="not a fit") + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "rejected" + + +async def test_vault_write_records_timed_out(temp_vault): + """Timeout-checker resume sets feedback='auto-cancelled after Xh timeout'.""" + import frontmatter + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.2, + human_approved=False, + human_feedback="auto-cancelled after 4h timeout", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "timed_out" +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_vault_write.py -v -k "hitl_decision or auto_rejected or rejected_when_human or timed_out" +``` + +Expected: 4 failures (the fields aren't populated yet). + +- [ ] **Step 4: Implement the mapping in `vault_write_node`** + +In `compass/pipeline/nodes/vault_write.py`, before building the `JobNote(...)` kwargs, add: + +```python +def _derive_hitl_decision(state: "CompassState") -> tuple[str | None, "datetime | None"]: + """Map state -> (hitl_decision, hitl_at). Returns (None, None) if hitl never ran.""" + from datetime import datetime as _dt + + from compass.config import SCORE_THRESHOLD + + approved = state.get("human_approved") + if approved is None: + # hitl never reached (e.g. extract/score errored). Leave fields null. + return (None, None) + + feedback = (state.get("human_feedback") or "").lower() + score = state.get("score_result") + score_value = score.score if score is not None else 0.0 + + if approved is True: + decision = "approved" + elif feedback.startswith("auto-cancelled after"): + decision = "timed_out" + elif score_value < SCORE_THRESHOLD: + decision = "auto_rejected" + else: + decision = "rejected" + return (decision, _dt.now()) +``` + +Wire it into the JobNote construction (replace the existing `JobNote(...)` call's tailored_paragraph line region): + +```python + hitl_decision, hitl_at = _derive_hitl_decision(state) + note = JobNote( + # … all existing fields … + tailored_paragraph=state.get("tailored_paragraph"), + hitl_decision=hitl_decision, + hitl_at=hitl_at, + ) +``` + +- [ ] **Step 5: Run the new tests** + +```bash +uv run pytest tests/pipeline/test_vault_write.py -v +``` + +Expected: all previous + 4 new pass. + +- [ ] **Step 6: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 210 passed (206 from Task 2 + 4 new). + +- [ ] **Step 7: Commit** + +```bash +git add compass/pipeline/nodes/vault_write.py tests/pipeline/test_vault_write.py \ + compass/vault/schemas.py +git commit -m "feat(hitl): record hitl_decision + hitl_at on JobNote" +``` + +--- + +## Task 3: Wire `AsyncSqliteSaver` + register paused threads in `run_pipeline` + +**The risky one.** This is where the new graph topology meets the new state store. Key invariants we must preserve: + +1. **The checkpointer is opened ONCE per `run_pipeline` invocation**, inside an `async with` block. Reusing a closed checkpointer across batches is a known LangGraph foot-gun. +2. **`build_graph()` is called inside the `async with`** so the compiled graph holds the open checkpointer. Compiling at module level (or before the `async with`) silently breaks `interrupt()`. Do not refactor this for "neatness" later — the Phase 1.A handoff calls this out by name. +3. **The `thread_id` is deterministic from `(job.url, scrape_timestamp)`** so a rerun of the same batch reuses the same paused checkpoint. We hash `f"{job.url}|{start_wall.isoformat()}"` with SHA-1 truncated to 16 chars. +4. **After `graph.ainvoke(...)` returns**, inspect the result for `__interrupt__` markers. If present, the job is paused — call `state_store.add_pending(...)` and treat it as "not written, not errored, pending". Otherwise the job ran to completion as before. + +**Files:** +- Modify: `compass/pipeline/graph.py` +- Modify: `compass/config.py` (add `HITL_CHECKPOINT_DB`) +- Modify: `.env.example` +- Create: `tests/pipeline/test_graph_checkpointing.py` + +- [ ] **Step 1: Add `HITL_CHECKPOINT_DB` to config** + +Edit `compass/config.py`, in the `── HiTL ──` section: + +```python +HITL_STATE_DB: Path = Path(os.getenv("HITL_STATE_DB", "~/.compass/hitl.db")).expanduser() +HITL_CHECKPOINT_DB: Path = Path( + os.getenv("HITL_CHECKPOINT_DB", "~/.compass/checkpoints.db") +).expanduser() +HITL_TIMEOUT_HOURS: int = int(os.getenv("HITL_TIMEOUT_HOURS", "4")) +``` + +Edit `.env.example`, in the `── HiTL ──` section: + +``` +HITL_STATE_DB=~/.compass/hitl.db +HITL_CHECKPOINT_DB=~/.compass/checkpoints.db +HITL_TIMEOUT_HOURS=4 +``` + +- [ ] **Step 2: Write the integration test for graph checkpointing** + +Create `tests/pipeline/test_graph_checkpointing.py`: + +```python +"""End-to-end: graph pauses at hitl, state_store gets the row, + resume_pending finishes the run. + + These tests STUB the LLM-touching nodes (extract, score, tailor) — we only + exercise the graph machinery + interrupt + checkpointer + state_store + interaction. Real LLM calls live in the live-smoke check in Task 7.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + """Replace extract / score / tailor / vault_write with deterministic stubs.""" + async def fake_extract(state: CompassState) -> dict: + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP", "Python"], + nice_to_have_skills=["LangGraph"], + seniority="mid", + remote_policy="remote", + summary="An agent role.", + ) + } + + async def fake_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=4.2, + reasoning="Strong match.", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="Lead with MCP.", + ) + } + + async def fake_tailor(state: CompassState) -> dict: + return {"tailored_paragraph": "Tailored: lead with MCP work."} + + async def fake_vault_write(state: CompassState) -> dict: + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(state: CompassState) -> dict: + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +def _job(url: str = "https://jobs.example.com/sierra-1") -> RawJob: + return RawJob( + company="Sierra", + title="SWE, Agent", + url=url, + source="ashby", + description="An agent engineering role at Sierra.", + date_posted=_dt.date(2026, 5, 18), + ) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_above_threshold_job_pauses_and_registers_in_state_store(monkeypatch): + """run_pipeline with a single above-threshold job: paused=1, written=0, + state_store has a row.""" + from compass.pipeline.graph import run_pipeline + + # No real scraping — pass the job in directly. + result = await run_pipeline(raw_jobs=[_job()]) + + assert result["jobs_processed"] == 1 + assert result["jobs_written"] == 0 # paused before vault_write + assert result["jobs_paused"] == 1 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["company"] == "Sierra" + assert rows[0]["score"] == pytest.approx(4.2) + assert rows[0]["job_url"] == "https://jobs.example.com/sierra-1" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_below_threshold_job_runs_to_completion_no_state_store_row(monkeypatch): + """Below-threshold path is unchanged from 1.A — no interrupt, vault_write fires.""" + async def low_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=2.0, + reasoning="Mismatched.", + matched_skills=[], + missing_skills=["A", "B"], + tailoring_notes="", + ) + } + monkeypatch.setattr("compass.pipeline.graph.score_node", low_score) + from compass.pipeline.graph import run_pipeline + + result = await run_pipeline(raw_jobs=[_job()]) + assert result["jobs_written"] == 1 + assert result["jobs_paused"] == 0 + assert await state_store.list_pending() == [] + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_thread_id_is_deterministic_for_same_url_and_batch(): + """Re-running the SAME batch (same start_wall) reuses the same thread_id — + second run hits the idempotent INSERT OR IGNORE path.""" + from compass.pipeline.graph import _thread_id_for + tid_a = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_b = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_c = _thread_id_for("https://jobs.example.com/y", _dt.datetime(2026, 5, 19, 9, 0, 0)) + assert tid_a == tid_b + assert tid_a != tid_c + assert len(tid_a) == 16 +``` + +Run: + +```bash +uv run pytest tests/pipeline/test_graph_checkpointing.py -v +``` + +Expected: all three fail (run_pipeline doesn't take `raw_jobs` paths through this flow yet, `_thread_id_for` doesn't exist, `jobs_paused` not in aggregate). + +- [ ] **Step 3: Rewrite the relevant chunks of `graph.py`** + +Apply the following surgical edits to `compass/pipeline/graph.py`: + +**3a. Add new imports near the top:** + +```python +import hashlib + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + +from compass.hitl import state_store +``` + +**3b. Add `_thread_id_for` as a module-level helper, just above `_initial_state`:** + +```python +def _thread_id_for(job_url: str, batch_started_at: datetime) -> str: + """Deterministic 16-char SHA-1 of (url, batch start) — same batch + same job = same thread.""" + raw = f"{job_url}|{batch_started_at.isoformat()}" + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] +``` + +**3c. Extend `_initial_state` to accept and store `thread_id`:** + +```python +def _initial_state(job: RawJob, thread_id: str | None = None) -> CompassState: + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": None, + "in_scope": None, + "role_family": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": thread_id, + } +``` + +**3d. Replace `_process_one` with a checkpointer-aware version that detects paused state:** + +```python +async def _process_one( + graph, + job: RawJob, + sem: asyncio.Semaphore, + thread_id: str, +) -> tuple[CompassState, bool]: + """Invoke the graph for one job. Returns (final_state, was_paused).""" + config = { + "configurable": {"thread_id": thread_id}, + **_langfuse_config(), + } + async with sem: + try: + result = await graph.ainvoke(_initial_state(job, thread_id=thread_id), config=config) + except Exception as e: + logger.exception("pipeline: graph crashed on %s", job.url) + return ( + {**_initial_state(job, thread_id=thread_id), "errors": [f"graph: {type(e).__name__}: {e}"]}, + False, + ) + + # When interrupt() fires, ainvoke returns with state containing the + # __interrupt__ marker AND without progressing past the hitl node. + # vault_written stays False, jobs_written stays 0 — that's our signal. + if "__interrupt__" in result and result["__interrupt__"]: + interrupts = result["__interrupt__"] + payload = interrupts[0].value if hasattr(interrupts[0], "value") else interrupts[0] + if isinstance(payload, dict) and payload.get("kind") == "approval_request": + await state_store.add_pending( + thread_id=thread_id, + job_url=payload["job_url"], + company=payload["company"], + title=payload["title"], + score=float(payload["score"]), + score_reasoning=payload["score_reasoning"], + matched_skills=list(payload["matched_skills"]), + missing_skills=list(payload["missing_skills"]), + ) + logger.info( + "pipeline: paused %s for approval (thread_id=%s, score=%.2f)", + job.url, thread_id, payload["score"], + ) + return (result, True) + # Unknown interrupt shape — DO NOT silently swallow. Phase 0 bug pattern: + # a future interrupt() added elsewhere in the graph would otherwise + # disappear into a "succeeded with jobs_paused=0" black hole. Log loudly + # and still count as paused so the caller's bookkeeping reflects reality. + logger.error( + "pipeline: graph paused at UNKNOWN interrupt kind for %s — payload=%r", + job.url, payload, + ) + return (result, True) + return (result, False) +``` + +**3e. Restructure `run_pipeline` to mount the checkpointer:** + +```python +async def run_pipeline(raw_jobs: list[RawJob] | None = None) -> CompassState: + """Scrape (or accept) jobs, dedup, run per-job graph under a single + AsyncSqliteSaver, regenerate gap plan.""" + from compass.config import HITL_CHECKPOINT_DB + + start_monotonic = time.monotonic() + start_wall = datetime.now() + if raw_jobs is None: + raw_jobs = await _scrape_all() + + seen_urls = _vault_url_set() + fresh = [j for j in raw_jobs if j.url not in seen_urls] + dropped = len(raw_jobs) - len(fresh) + if dropped: + logger.info("pipeline: dropping %d/%d jobs already in vault", dropped, len(raw_jobs)) + + HITL_CHECKPOINT_DB.parent.mkdir(parents=True, exist_ok=True) + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + # Enable WAL on the checkpoint DB once per process. Cheap if already set. + # See state_store._connect for the rationale. + try: + await checkpointer.conn.execute("PRAGMA journal_mode=WAL") + await checkpointer.conn.execute("PRAGMA busy_timeout=5000") + except Exception: + logger.debug("checkpoint: WAL pragma set already or unsupported; continuing") + graph = build_graph(checkpointer=checkpointer) + sem = asyncio.Semaphore(MAX_CONCURRENT_JOBS) + coros = [ + _process_one(graph, j, sem, thread_id=_thread_id_for(j.url, start_wall)) + for j in fresh + ] + results = await asyncio.gather(*coros) + + pairs = results # list of (state, was_paused) + paused_count = sum(int(p) for _, p in pairs) + final_states = [s for s, _ in pairs] + + aggregate: CompassState = { + "raw_jobs": raw_jobs, + "current_job": None, + "extracted_requirements": None, + "score_result": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": any(r.get("vault_written") for r in final_states), + "jobs_processed": len(fresh), + "jobs_written": sum(int(bool(r.get("vault_written"))) for r in final_states), + "errors": [e for r in final_states for e in r.get("errors", [])], + "in_scope": None, + "role_family": None, + "thread_id": None, + } + # `jobs_paused` is informational only — not in CompassState TypedDict. + # Returned via the aggregate dict; orchestrators (CLI, MCP) read it. + aggregate["jobs_paused"] = paused_count # type: ignore[typeddict-unknown-key] + + if aggregate["jobs_written"] > 0: + gap_aggregator.regenerate(write=True) + + duration_s = time.monotonic() - start_monotonic + _append_run_log(aggregate, duration_s) + unknown_count = _count_unknown_skills_seen_this_run(start_wall) + logger.info( + "pipeline: processed=%d written=%d paused=%d errors=%d unknown_skills_seen=%d duration=%.1fs", + aggregate["jobs_processed"], aggregate["jobs_written"], paused_count, + len(aggregate["errors"]), unknown_count, duration_s, + ) + return aggregate +``` + +**3f. Make `build_graph` accept a checkpointer:** + +```python +def build_graph(checkpointer=None): + builder = StateGraph(CompassState) + # ... unchanged node + edge wiring ... + return builder.compile(checkpointer=checkpointer) +``` + +**3g. Update `_append_run_log` to record the paused column:** + +```python +def _append_run_log(state: CompassState, duration_s: float) -> None: + log_path = VAULT_PATH / "_meta" / "pipeline-runs.md" + log_path.parent.mkdir(parents=True, exist_ok=True) + if not log_path.exists(): + log_path.write_text( + "# Pipeline Run Log\n\n" + "| Timestamp | Processed | Written | Paused | Errors | Duration |\n" + "|---|---|---|---|---|---|\n", + encoding="utf-8", + ) + ts = datetime.now().isoformat(timespec="seconds") + paused = state.get("jobs_paused", 0) # type: ignore[typeddict-item] + row = ( + f"| {ts} | {state['jobs_processed']} | {state['jobs_written']} | " + f"{paused} | {len(state['errors'])} | {duration_s:.1f}s |\n" + ) + with log_path.open("a", encoding="utf-8") as f: + f.write(row) +``` + +**3h. Update the `__main__` print:** + +```python +if __name__ == "__main__": + result = asyncio.run(run_pipeline()) + print( + f"Processed: {result['jobs_processed']} | " + f"Written: {result['jobs_written']} | " + f"Paused: {result.get('jobs_paused', 0)} | " + f"Errors: {len(result['errors'])}" + ) +``` + +- [ ] **Step 4: Run the new checkpointing tests** + +```bash +uv run pytest tests/pipeline/test_graph_checkpointing.py -v +``` + +Expected: 3 passed. If a test fails because `__interrupt__` isn't in the returned state, drop a `print(result)` after the ainvoke to inspect the exact shape — LangGraph 1.x has been seen to also expose interrupts via `graph.get_state(config).next` instead. If so, switch the detection to `state = await graph.aget_state(config); was_paused = bool(state.next)`. + +- [ ] **Step 5: Run the full suite — Phase 1.A tests must still pass** + +```bash +uv run pytest -q +``` + +Expected: 213 passed (210 from Task 2.5 + 3 new). + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/graph.py compass/config.py .env.example \ + tests/pipeline/test_graph_checkpointing.py +git commit -m "feat(hitl): mount AsyncSqliteSaver in run_pipeline; register paused threads" +``` + +--- + +## Task 4: Resume entrypoint (`compass/hitl/resume.py`) + +**Files:** +- Create: `compass/hitl/resume.py` +- Create: `tests/hitl/test_resume.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/hitl/test_resume.py`: + +```python +"""resume_pending re-opens the checkpointer, recompiles the graph, and drives + the resume via Command(resume=...). Verifies vault_write fires on approve + and tailor is skipped on reject.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="agent role")} + + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes="")} + + async def fake_tailor(_): return {"tailored_paragraph": "lead with MCP."} + + written = {"calls": 0, "with_tailor": 0} + async def fake_vault_write(state): + written["calls"] += 1 + if state.get("tailored_paragraph"): + written["with_tailor"] += 1 + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + return written + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_approve_runs_tailor_then_vault_write(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + + pending = await state_store.list_pending() + tid = pending[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": True, "feedback": "LGTM"}) + assert final["vault_written"] is True + assert final["human_approved"] is True + assert stub_llm_nodes["with_tailor"] == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_reject_skips_tailor_writes_to_vault(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/2", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": False}) + assert final["human_approved"] is False + assert stub_llm_nodes["with_tailor"] == 0 + assert stub_llm_nodes["calls"] == 1 # vault_write still fired (rejected branch) + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_unknown_thread_raises(): + from compass.hitl.resume import resume_pending + with pytest.raises(LookupError, match="thread"): + await resume_pending("nope-1234", decision={"approved": True}) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_resume_already_resolved_is_noop(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/3", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + await resume_pending(tid, decision={"approved": True}) + + # Second resume call should not crash and should not re-fire vault_write + before = stub_llm_nodes["calls"] + with pytest.raises(ValueError, match="already resolved"): + await resume_pending(tid, decision={"approved": True}) + assert stub_llm_nodes["calls"] == before +``` + +Run — expect all 4 failing with `ModuleNotFoundError`. + +- [ ] **Step 2: Implement `resume.py`** + +Create `compass/hitl/resume.py`: + +```python +"""Resume a paused LangGraph thread by re-opening the checkpointer and + recompiling the graph. Single source of truth for resumes — the timeout + checker and MCP `approve` tool both go through here.""" + +from __future__ import annotations + +import logging +from typing import Any + +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver +from langgraph.types import Command + +from compass.hitl import state_store + +logger = logging.getLogger(__name__) + + +async def resume_pending( + thread_id: str, + *, + decision: dict[str, Any], + status_override: str | None = None, +) -> dict: + """Drive a paused thread to completion. + + decision: passed as `Command(resume=decision)`. Expected shape + {"approved": bool, "feedback": str | None}. + status_override: optional explicit status to write to the state store + (timeout_checker uses "timed_out"). If None, derived from + decision["approved"]: True -> "approved", False -> "rejected". + Raises LookupError if the thread isn't in the state store. + Raises ValueError if the thread is already resolved. + """ + from compass.config import HITL_CHECKPOINT_DB + + row = await state_store.get_pending(thread_id) + if row is None: + raise LookupError(f"no pending thread {thread_id!r}") + if row["status"] != "pending": + raise ValueError(f"thread {thread_id!r} already resolved ({row['status']})") + + # Late import to avoid circular: graph.py imports state_store + from compass.pipeline.graph import build_graph + + config = {"configurable": {"thread_id": thread_id}} + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + resolved_status = "approved" if decision.get("approved") else "rejected" + await state_store.mark_resolved( + thread_id, + status=resolved_status, + feedback=decision.get("feedback"), + ) + logger.info("hitl: resumed thread %s -> %s", thread_id, resolved_status) + return final +``` + +- [ ] **Step 3: Run resume tests** + +```bash +uv run pytest tests/hitl/test_resume.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 217 passed (213 + 4). + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/resume.py tests/hitl/test_resume.py +git commit -m "feat(hitl): resume_pending entrypoint via Command(resume=...)" +``` + +--- + +## Task 5: Timeout checker + +**Files:** +- Create: `compass/hitl/timeout_checker.py` +- Create: `tests/hitl/test_timeout_checker.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/hitl/test_timeout_checker.py`: + +```python +"""timeout_checker resumes pending rows older than HITL_TIMEOUT_HOURS with + {"approved": False}, marks them as 'timed_out' in the state store, and + leaves young pending rows untouched.""" + +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="x")} + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes="")} + async def fake_vault_write(_): return {"vault_written": True, "jobs_written": 1} + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + async def fake_tailor(_): return {"tailored_paragraph": "..."} + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_resumes_old_pending_as_rejected(monkeypatch): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the row's created_at to look 5h old + import aiosqlite + from compass.config import HITL_STATE_DB + five_h_ago = (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute( + "UPDATE pending_approvals SET created_at = ? WHERE thread_id = ?", + (five_h_ago, tid), + ) + await conn.commit() + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "timed_out" + # Spec DoD line 230 — timeout must be logged to _meta/agent-log.md + from compass.config import AGENT_LOG_PATH + log_text = AGENT_LOG_PATH.read_text(encoding="utf-8") + assert "hitl-timeout" in log_text + assert tid in log_text + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_leaves_young_pending_alone(): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="https://x/2", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 18)) + await run_pipeline(raw_jobs=[job]) + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 0 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["status"] == "pending" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes") +async def test_timeout_checker_continues_after_one_resume_failure(monkeypatch): + """A single broken thread (e.g. checkpoint corrupted) must not block the + rest of the queue from timing out.""" + from compass.hitl import timeout_checker + + # Two rows, both stale + await state_store.add_pending( + thread_id="tid-broken", job_url="x://1", company="C", title="T", + score=4.0, score_reasoning="r", matched_skills=[], missing_skills=[], + ) + await state_store.add_pending( + thread_id="tid-ok", job_url="x://2", company="C", title="T", + score=4.0, score_reasoning="r", matched_skills=[], missing_skills=[], + ) + # Backdate both + import aiosqlite + from compass.config import HITL_STATE_DB + five_h_ago = (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute("UPDATE pending_approvals SET created_at = ?", (five_h_ago,)) + await conn.commit() + + # Make resume_pending raise for tid-broken + async def flaky(thread_id, **kw): + if thread_id == "tid-broken": + raise RuntimeError("checkpoint missing") + from compass.hitl.state_store import mark_resolved + await mark_resolved(thread_id, status="timed_out") + monkeypatch.setattr("compass.hitl.timeout_checker.resume_pending", flaky) + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 # one succeeded, one failed + row_broken = await state_store.get_pending("tid-broken") + assert row_broken["status"] == "error" +``` + +- [ ] **Step 2: Implement `timeout_checker.py`** + +Create `compass/hitl/timeout_checker.py`: + +```python +"""Auto-cancel pending approvals older than HITL_TIMEOUT_HOURS. + +Module entrypoint (CLI): python -m compass.hitl.timeout_checker +Async API: check_and_resume_timeouts(timeout_hours: int | None = None) + +In Phase 1.B.1 this is human-run. In Phase 1.B.3 a Modal cron will import and +schedule it — see modal_app.py.""" + +from __future__ import annotations + +import asyncio +import logging + +from compass.hitl import state_store +from compass.hitl.resume import resume_pending + +logger = logging.getLogger(__name__) + + +async def check_and_resume_timeouts(*, timeout_hours: int | None = None) -> int: + """Resume every stale pending row with {"approved": False}. Returns the + count successfully timed-out (i.e. exclude rows that errored).""" + from compass.config import HITL_TIMEOUT_HOURS + + hrs = timeout_hours if timeout_hours is not None else HITL_TIMEOUT_HOURS + stale = await state_store.list_timed_out(timeout_hours=hrs) + if not stale: + return 0 + + logger.info("hitl: %d pending approval(s) past %dh timeout", len(stale), hrs) + timed_out = 0 + for row in stale: + tid = row["thread_id"] + try: + await resume_pending( + tid, + decision={"approved": False, "feedback": f"auto-cancelled after {hrs}h timeout"}, + status_override="timed_out", + ) + # Spec § DoD line 230: HiTL timeout must log to _meta/agent-log.md + _log_to_agent_log( + f"hitl-timeout: auto-cancelled {tid} ({row['company']} / {row['title']}, " + f"score={row['score']:.2f}) after {hrs}h" + ) + timed_out += 1 + except Exception as e: + logger.exception("hitl: timeout resume failed for %s — marking as error", tid) + try: + await state_store.mark_resolved(tid, status="error", feedback=f"resume error: {e}") + _log_to_agent_log(f"hitl-timeout-error: {tid} resume failed: {e}") + except Exception: + logger.exception("hitl: also failed to mark %s as error", tid) + return timed_out + + +def _log_to_agent_log(line: str) -> None: + """Append a timestamped row to compass-vault/_meta/agent-log.md. + + Late-binds via compass.vault.writer.append_agent_log so the temp_vault + fixture works in tests. + """ + from compass.vault.writer import append_agent_log + try: + append_agent_log(line) + except Exception: + logger.exception("hitl: failed to write to agent-log.md") + + +if __name__ == "__main__": + n = asyncio.run(check_and_resume_timeouts()) + print(f"Timed out: {n}") +``` + +- [ ] **Step 3: Run tests** + +```bash +uv run pytest tests/hitl/test_timeout_checker.py -v +``` + +Expected: 3 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 220 passed (217 + 3). + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/timeout_checker.py tests/hitl/test_timeout_checker.py +git commit -m "feat(hitl): timeout_checker auto-cancels stale approvals" +``` + +--- + +## Task 6: MCP tools (`pending_approvals` + `approve`) + +**Files:** +- Modify: `compass/mcp_server/server.py` +- Create: `tests/mcp_server/__init__.py` if missing +- Create: `tests/mcp_server/test_pending_approvals.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/mcp_server/test_pending_approvals.py`: + +```python +"""MCP wrappers around state_store + resume_pending — JSON-serializable + return shapes (no datetime objects, no Pydantic instances).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from compass.hitl import state_store + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_pending_approvals_tool_returns_jsonable_rows(): + from compass.mcp_server.server import pending_approvals + + await state_store.add_pending( + thread_id="tid-1", job_url="https://x/1", company="Sierra", + title="SWE Agent", score=4.2, score_reasoning="solid", + matched_skills=["MCP"], missing_skills=["LangGraph"], + ) + rows = await pending_approvals() + assert len(rows) == 1 + assert rows[0]["thread_id"] == "tid-1" + # Must be plain JSON-serialisable + json.dumps(rows) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_invokes_resume(monkeypatch): + from compass.mcp_server import server as mcp_server + + captured = {} + async def fake_resume(thread_id, *, decision, status_override=None): + captured["thread_id"] = thread_id + captured["decision"] = decision + captured["status_override"] = status_override + return {"vault_written": True, "human_approved": decision["approved"]} + + monkeypatch.setattr(mcp_server, "_resume_pending", fake_resume) + + result = await mcp_server.approve(thread_id="tid-x", approved=True, feedback="LGTM") + assert result == {"vault_written": True, "human_approved": True} + assert captured == { + "thread_id": "tid-x", + "decision": {"approved": True, "feedback": "LGTM"}, + "status_override": None, + } + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_handles_unknown_thread_as_error_dict(): + from compass.mcp_server.server import approve + result = await approve(thread_id="missing", approved=False) + assert "error" in result and "missing" in result["error"] +``` + +- [ ] **Step 2: Add the MCP tools** + +Append to `compass/mcp_server/server.py` (after the existing `tailor_resume` block): + +```python +# ── HiTL approvals ─────────────────────────────────────────────────────────── + +from compass.hitl import state_store as _state_store +from compass.hitl.resume import resume_pending as _resume_pending + + +@mcp.tool() +async def pending_approvals() -> list[dict]: + """List jobs paused at hitl awaiting human approval. Oldest first.""" + rows = await _state_store.list_pending() + # rows are already plain dicts of JSON-safe primitives + list[str] + return rows + + +@mcp.tool() +async def approve(thread_id: str, approved: bool, feedback: str | None = None) -> dict: + """Resume a paused thread. approved=True runs tailor + vault_write; + approved=False skips tailor and writes the rejected JobNote.""" + try: + final = await _resume_pending( + thread_id, + decision={"approved": approved, "feedback": feedback}, + ) + except (LookupError, ValueError) as e: + return {"error": str(e)} + # Strip non-JSON-safe state from the return (current_job is a Pydantic model) + return { + "vault_written": bool(final.get("vault_written")), + "human_approved": bool(final.get("human_approved")), + "human_feedback": final.get("human_feedback"), + } +``` + +Also update the module docstring's tool listing to include `pending_approvals()` and `approve(thread_id, approved, feedback)`. + +- [ ] **Step 3: Run MCP tests** + +```bash +mkdir -p tests/mcp_server && touch tests/mcp_server/__init__.py +uv run pytest tests/mcp_server/test_pending_approvals.py -v +``` + +Expected: 3 passed. + +- [ ] **Step 4: Full suite** + +```bash +uv run pytest -q +``` + +Expected: 223 passed (220 + 3). + +- [ ] **Step 5: Lint** + +```bash +uv run ruff check +uv run ruff format --check +``` + +Expected: clean. If `format --check` complains, run `uv run ruff format` and re-stage. + +- [ ] **Step 6: Commit** + +```bash +git add compass/mcp_server/server.py tests/mcp_server/__init__.py \ + tests/mcp_server/test_pending_approvals.py +git commit -m "feat(mcp): pending_approvals + approve tools" +``` + +--- + +## Task 7: Live smoke + checkpoint-survives-restart verification + +> **PAUSE HERE before running this task.** Step 2 hits real OpenRouter + writes to the user's real vault + writes to `~/.compass/checkpoints.db`. Confirm with the user before running. This is the only LLM-touching step in the whole plan. + +**Goal:** Prove the three things tests cannot prove: +1. A real graph invocation with a real LLM score actually pauses at hitl +2. The pause SURVIVES a Python process restart (closing + re-opening the AsyncSqliteSaver from a fresh process) +3. Resume via `approve()` MCP tool produces a real JobNote with a real tailored paragraph + +**Files:** none + +- [ ] **Step 1: Snapshot vault + DBs** + +```bash +cp -r ~/Documents/compass-vault/jobs ~/Documents/compass-vault/jobs.preB1.bak 2>/dev/null || true +cp ~/.compass/hitl.db ~/.compass/hitl.db.preB1.bak 2>/dev/null || true +cp ~/.compass/checkpoints.db ~/.compass/checkpoints.db.preB1.bak 2>/dev/null || true +``` + +- [ ] **Step 2: Run pipeline against one apply-now board, expect paused jobs** + +```bash +cd ~/Documents/compass +MAX_JOBS_PER_RUN=10 \ + GREENHOUSE_BOARDS= \ + LEVER_COMPANIES= \ + ASHBY_BOARDS=sierra \ + uv run python -m compass.pipeline.graph +``` + +Expected output: +``` +Processed: N | Written: M | Paused: K | Errors: 0 +``` +where `K >= 1` (at least one above-threshold Sierra job should pause). + +- [ ] **Step 3: Confirm pending row exists from a SEPARATE Python process** + +This is the critical "checkpoint survives restart" check. Run a fresh process: + +```bash +uv run python -c " +import asyncio +from compass.hitl import state_store +async def main(): + rows = await state_store.list_pending() + print(f'Pending: {len(rows)}') + for r in rows: + print(f' {r[\"thread_id\"]} {r[\"company\"]} / {r[\"title\"]} score={r[\"score\"]}') +asyncio.run(main()) +" +``` + +Expected: at least one row printed with a real Sierra job. + +- [ ] **Step 4: Resume one thread via the MCP path, confirm JobNote written** + +Pick a `thread_id` from Step 3's output, then run (replace `<TID>`): + +```bash +uv run python -c " +import asyncio +from compass.hitl.resume import resume_pending +async def main(): + final = await resume_pending('<TID>', decision={'approved': True, 'feedback': 'smoke test'}) + print('vault_written =', final.get('vault_written')) + print('human_approved =', final.get('human_approved')) +asyncio.run(main()) +" +``` + +Expected: `vault_written = True`, `human_approved = True`. Then verify the JobNote: + +```bash +ls -ltr ~/Documents/compass-vault/jobs/ | tail -3 +``` + +The most recent file should be the Sierra job. Open it and confirm: +- `tailored_paragraph:` is populated with real Sonnet output (not empty) +- `match_score:` is the same float seen in Step 3 +- `hitl_decision: approved` in the frontmatter (NOT `human_approved` — that key lives in pipeline state, not the JobNote) +- `hitl_at:` is a recent ISO timestamp + +- [ ] **Step 5: Run `timeout_checker` against an empty queue, confirm 0 timed out** + +```bash +uv run python -m compass.hitl.timeout_checker +``` + +Expected: `Timed out: 0` (the one row we approved is no longer pending). + +- [ ] **Step 6: Restore backups if anything went wrong; otherwise delete them** + +```bash +# If anything looks wrong (no tailored paragraph, mis-sorted, etc.), restore: +# rm -rf ~/Documents/compass-vault/jobs && mv ~/Documents/compass-vault/jobs.preB1.bak ~/Documents/compass-vault/jobs +# mv ~/.compass/hitl.db.preB1.bak ~/.compass/hitl.db +# mv ~/.compass/checkpoints.db.preB1.bak ~/.compass/checkpoints.db +# If everything looks good: +rm -rf ~/Documents/compass-vault/jobs.preB1.bak ~/.compass/hitl.db.preB1.bak ~/.compass/checkpoints.db.preB1.bak +``` + +- [ ] **Step 7: Tag the phase** + +```bash +git tag phase-1b1-hitl +``` + +--- + +## Definition of Done + +All of the following must hold before declaring Phase 1.B.1 complete: + +**Code & tests** +- 223 tests passing (`uv run pytest -q`) +- Ruff clean (`uv run ruff check && uv run ruff format --check`) +- No module-level `from compass.config import HITL_*` — every reference is inside a function body +- `build_graph()` accepts an optional `checkpointer` parameter; called only from inside the `async with AsyncSqliteSaver(...)` block in `run_pipeline` AND from `resume_pending` + +**Behaviour (verified empirically in Task 7)** +- A real above-threshold job pauses at `hitl`, registers in `~/.compass/hitl.db`, does NOT write a JobNote +- The pending row is visible from a fresh Python process (proves the checkpoint isn't process-local) +- `resume_pending(tid, decision={"approved": True})` writes a JobNote with a non-empty `tailored_paragraph` and `hitl_decision: approved` +- `resume_pending(tid, decision={"approved": False})` writes a JobNote with `hitl_decision: rejected` and no `tailored_paragraph` +- `timeout_checker.check_and_resume_timeouts()` resumes only rows older than `HITL_TIMEOUT_HOURS`, marks them `timed_out`, AND appends a row to `_meta/agent-log.md` (spec § DoD line 230) +- Below-threshold jobs are unchanged from Phase 1.A — no interrupt fires, no state-store row +- An unknown `interrupt()` payload (anything not `kind == "approval_request"`) is logged at ERROR and counted as `jobs_paused`, never silently dropped + +**Docs** +- `.env.example` documents `HITL_CHECKPOINT_DB` +- This plan's "What's deferred" cross-references are accurate (RAG → 1.B.2, Modal cron → 1.B.3) +- Final commit message of Task 7: `git tag phase-1b1-hitl` + +--- + +## What's deferred (and to which sub-phase) + +| Concern | Severity | Phase | Why deferred | +|---|---|---|---| +| Chroma RAG for profile retrieval | Portfolio claim | **1.B.2** | Independent of HiTL; separate plan | +| Modal cron schedule (`@app.function(schedule=Cron(...))`) for `timeout_checker` + daily scrape + weekly assessor | Required for "no babysitting" | **1.B.3** | This phase ships the callable; 1.B.3 wires the schedule | +| Modal Secrets for `OPENROUTER_API_KEY`, `LANGFUSE_*` | Cloud deploy | **1.B.3** | Same — couples with cron deploy | +| Langfuse callback API mismatch (`host=` kwarg) | Observability | **1.B.3** | Bug #23 from Phase 0; dedicated 1.B.3 work | +| URL dedup for `intake_filter`-rejected JDs | Cost + log growth | **1.B.3** | Natural fit with caching layer | +| Hebbia Greenhouse 404 cleanup in `.env.example` | Coverage | **1.B.3** | Bundle with config restructure for Modal Secrets | +| `approve(thread_id, approved=None)` "request changes" branch | UX | post-1.B | Today binary; if needed, ship as third decision dict key | +| Per-thread auth (anyone with MCP access can approve) | Security | post-1.B | Solo-user system; revisit if hosted | +| Surface `pending_approvals` in the Dataview dashboard | UX polish | **2.B** | Dataview can't read SQLite; needs a tiny exporter | +| Atomic `claim_pending()` to prevent double-resume race (I4: MCP approve vs. `timeout_checker` cron, or MCP double-click) | Race / dup writes | **1.B.3** | Real-world trigger is the Modal cron; ship the claim alongside cron in 1.B.3. Today's MCP-only flow is single-user, no concurrent caller. | + +--- + +## Critical lessons to carry forward (from Phases 0 + 1.A) + +These bit us before; they will bite us again if we forget them: + +1. **Tests check shape; only adversarial real-data inspection catches data-correctness bugs.** Task 7 is non-optional — do not declare 1.B.1 done from green CI alone. +2. **Module-level `from compass.config import X` freezes the value at import time** and silently breaks the `temp_*_db` fixtures. Always late-bind inside function bodies (e.g. `from compass.config import HITL_CHECKPOINT_DB` *inside* `run_pipeline`). +3. **A graph compiled without a checkpointer silently breaks `interrupt()`.** Never compile `build_graph()` at module level. The only call sites are inside the `async with AsyncSqliteSaver(...)` block in `run_pipeline` and `resume_pending`. +4. **Two-stage review (spec compliance → code quality) catches ~1 issue per task.** Don't skip even on "mechanical" tasks — the CompanyNote tier race (Phase 1.A bug #1) and the Greenhouse `?content=true` flag (Phase 0 bug #1) both looked mechanical and shipped silent data bugs. +5. **JSON-serializability of MCP return shapes.** Phase 1.A bug #11: `date` objects don't serialise through FastMCP. Use `model_dump(mode="json")` or hand-build plain-dict rows (which `state_store._row_to_dict` already does). +6. **Don't trust the tag until you've resumed a real paused thread across a process restart.** Task 7 Step 3 is specifically that check. + +--- + +## Plan Review Loop + +Before executing this plan, dispatch a single plan-document-reviewer subagent with: +- Path to this plan: `docs/superpowers/plans/2026-05-18-compass-phase-1b1-hitl.md` +- Path to spec: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- Phase scope reference: `docs/PHASE_1A_COMPLETE.md` § "How to start Phase 1.B" + +If the reviewer flags issues, fix them in place and re-dispatch the reviewer on the whole plan. Cap at 3 iterations before escalating. + +--- + +## Execution handoff + +Once the plan is approved, two execution options: + +1. **Subagent-Driven (recommended)** — Use `superpowers:subagent-driven-development`. Fresh implementer subagent per Task 1–6, two-stage reviewer (spec compliance → code quality) between tasks. Task 7 PAUSES for human confirmation before live-LLM run. + +2. **Inline Execution** — Use `superpowers:executing-plans`. Single session, batch through Tasks 1–6, hard checkpoint before Task 7. + +Phase 1.A used Subagent-Driven and it worked — keep the pattern. Task 7 always pauses for the human regardless of approach because it costs real money and writes to the real vault. diff --git a/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md b/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md new file mode 100644 index 0000000..5af25f0 --- /dev/null +++ b/docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md @@ -0,0 +1,1324 @@ +# Compass Phase 1.B.2 — RAG via Chroma + Phase 1.B.1 carryover cleanup (Implementation Plan) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace `score_node`'s wholesale-inject of `_profile/skill-inventory.md` with semantic retrieval against a Chroma vector index (closes the spec's RAG portfolio claim). Also close two Phase 1.B.1 carryover defects flagged in the post-smoke adversarial reviews: LangGraph msgpack deprecation warnings on `RawJob` / `JobRequirements` / `JobScore` (will break paused threads on the next LangGraph major) and `checkpoints.db` bloat from never-deleted resolved-thread checkpoint blobs. + +**Architecture:** New `compass/rag/` package with two modules. `indexer.py` parses `_profile/skill-inventory.md` into one Chroma document per `## SkillName` section, embeds via `sentence-transformers` (`all-MiniLM-L6-v2`), persists to `CHROMA_PATH`. Idempotent on re-run (upsert by stable id = skill heading). `retriever.py` exposes `retrieve(query, k=8) -> list[Chunk]` with lazy index init (build-on-miss). `score_node._profile_text()` is rewritten to retrieve top-k chunks given the JD's `required_skills + nice_to_have + summary` joined as a query, instead of injecting the whole inventory. Resume markdown still ships in full (it's small and load-bearing for context). Token-cost savings: ~2,500 tokens → ~750 tokens per scored JD. Two small infrastructure cleanups land alongside: a serde registration for the three Pydantic state types via `JsonPlusSerializer.register_pydantic_class()` (closes msgpack warnings) and a `_purge_thread_checkpoints(thread_id)` step at the end of `resume_pending` (closes the bloat). + +**Module-level discipline (carried forward from 1.A + 1.B.1):** every module that touches `compass.config.CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH`, `HITL_CHECKPOINT_DB`, or `VAULT_PATH` must reference it via `import compass.config as cfg; cfg.<NAME>` *inside function bodies*, never as module-level captured constants. The `temp_*` test fixtures monkeypatch these attributes. + +**Tech Stack:** Python 3.12 · chromadb 1.5 (`PersistentClient`) · sentence-transformers 5.5 (`all-MiniLM-L6-v2`, ~90MB local model cache) · langgraph 1.2 (`from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer`) · pytest + pytest-asyncio (`auto` mode, no `asyncio.run` in tests). + +**Authoritative spec:** `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` § Phase 1.B +**Previous-phase handoff:** Phase 1.B.1 retrospective lives in this branch's commits between `phase-1a-application-tracking` and `phase-1b1-hitl`. See `docs/PHASE_1A_COMPLETE.md` for the broader handoff and adversarial-review pattern. + +**Closes these deferred items from Phase 1.B.1:** +1. **I-2 (Phase 1.B.1 post-smoke pass 6)** — `~/.compass/checkpoints.db` bloat from never-purged resolved-thread blobs. Fix lands in Task 2. +2. **I1 (Phase 1.B.1 post-smoke pass 4)** — LangGraph msgpack `Deserializing unregistered type compass.pipeline.state.RawJob` warnings. Will become hard errors when `LANGGRAPH_STRICT_MSGPACK=true` defaults on (currently False, verified by `langgraph.checkpoint.serde.jsonplus`). Fix lands in Task 1. + +**Does NOT touch in this phase:** +- HiTL flow (Phase 1.B.1 — done; only carryover cleanup is in scope) +- I4 race claim_pending (deferred to Phase 1.B.3; documented in 1.B.1 plan's deferred table) +- Modal cron + Langfuse callback fix + URL-dedup for filtered-jobs → **Phase 1.B.3** +- 30-JD eval harness → **Phase 2.A** +- Do not modify Phase 1.B.1 HiTL code outside the explicit carryover fixes in Tasks 1–2. + +--- + +## File Structure + +### New +- `compass/rag/__init__.py` — empty +- `compass/rag/indexer.py` — parse skill-inventory.md → embed → persist to Chroma; `build_index(force_rebuild=False)`; `__main__` CLI entrypoint +- `compass/rag/retriever.py` — `retrieve(query, k=8) -> list[RetrievedChunk]`; lazy index init (build if missing) +- `tests/rag/__init__.py` +- `tests/rag/conftest.py` — `temp_chroma_path` fixture (monkeypatches `compass.config.CHROMA_PATH`); `tiny_inventory` fixture (writes a small 3-section inventory to the temp vault); model-cache reuse fixture so the 90MB sentence-transformers model is only fetched once per test session +- `tests/rag/test_indexer.py` +- `tests/rag/test_retriever.py` +- `tests/pipeline/test_score_node_rag.py` — verifies score_node uses retrieved chunks instead of full inventory + +### Modify +- `compass/pipeline/nodes/score.py` — `_profile_text()` rewritten to call retriever; keep resume inline; signature compatibility preserved +- `compass/pipeline/graph.py` — pass `serde=JsonPlusSerializer(...)` to `AsyncSqliteSaver.from_conn_string(...)` AND register the three Pydantic state types +- `compass/hitl/resume.py` — call `_purge_thread_checkpoints(thread_id)` after `mark_resolved` succeeds (only on `vault_written=True`, mirrors the regen pattern) + +### Untouched +- All Phase 1.B.1 HiTL infrastructure (`compass/hitl/{state_store,timeout_checker,__init__}.py`) +- All other pipeline nodes (`intake`, `intake_filter`, `extract`, `reflect`, `hitl`, `tailor`, `vault_write`) +- Vault schemas, vault writer, scrapers, applications lifecycle +- MCP server tools + +### Decomposition rationale +The two HiTL cleanups (msgpack + checkpoint purge) are intentionally Tasks 1 and 2 — small, isolated, and lower-risk than the RAG work. They land FIRST so the new tests pass against a stable baseline before any RAG changes. RAG splits across three files: `indexer.py` (write path), `retriever.py` (read path), and the `score_node` integration. The indexer/retriever split lets tests exercise each side in isolation (build an index; query an index) without needing the full pipeline. The retriever is the only one `score_node` depends on, keeping the dependency arrow one-way. `temp_chroma_path` is per-test to avoid sentence-transformer-cache leakage across tests, but the model itself (≈90MB) is loaded once per session via a module-cached factory. + +--- + +## Task 0: Pre-flight + +**Files:** none + +- [ ] **Step 1: Verify clean tree on `phase-1b1-hitl` tag** + +```bash +cd /Users/akmini/Documents/compass +git status # expected: clean +git describe --tags --abbrev=0 # expected: phase-1b1-hitl +uv run pytest -q # expected: 239 passed +uv run ruff check # expected: All checks passed +``` + +If working tree is dirty, STOP and ask the user before proceeding. + +- [ ] **Step 2: Confirm Chroma + sentence-transformers + langgraph serde availability AND verify the construction form actually suppresses warnings** + +```bash +uv run python -c " +import chromadb, sentence_transformers, logging, datetime +print('chromadb', chromadb.__version__) # expected: 1.x +print('sentence_transformers', sentence_transformers.__version__) # expected: 5.x +from chromadb import PersistentClient +import langgraph.checkpoint.serde.jsonplus as _jp +_jp._warned_unregistered_types.clear() # reset module-level once-only set +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer +from compass.pipeline.state import RawJob, JobRequirements, JobScore + +# Pass the allowlist at construction — `with_msgpack_allowlist` is a no-op +# when `allowed_msgpack_modules=True` (langgraph's non-STRICT default). +captured = [] +class H(logging.Handler): + def emit(self, r): + msg = r.getMessage() + if 'unregistered' in msg or 'fingerprint' in msg.lower(): + captured.append(msg) +logging.getLogger('langgraph').addHandler(H()) + +serde = JsonPlusSerializer(allowed_msgpack_modules=[ + ('compass.pipeline.state', 'RawJob'), + ('compass.pipeline.state', 'JobRequirements'), + ('compass.pipeline.state', 'JobScore'), +]) +rj = RawJob(company='C', title='T', url='u://x', source='ashby', description='d', date_posted=datetime.date(2026,5,19)) +ser = serde.dumps_typed(rj) +restored = serde.loads_typed(ser) +assert isinstance(restored, RawJob), f'expected RawJob, got {type(restored).__name__}' +assert not captured, f'allowlist did not suppress warnings: {captured}' +print('serde tuple-list allowlist verified — no warnings, types preserved') +" +``` + +Expected: prints `serde tuple-list allowlist verified — no warnings, types preserved`. If it fails with warnings still captured OR `restored` is not a RawJob, the allowlist API has shifted; STOP and inspect `inspect.signature(JsonPlusSerializer)` for the current constructor signature before continuing. + +**Lesson:** the deprecation warning text is misleading — it says "add to allowed_msgpack_modules" suggesting you can call a method after construction. You cannot. The allowlist MUST be passed at construction time as `(module, classname)` tuples. + +- [ ] **Step 2.5: Confirm checkpoint DB table names match what Task 2 will DELETE from** + +```bash +sqlite3 ~/.compass/checkpoints.db "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;" +``` + +Expected: exactly `checkpoints` and `writes`. If the table names differ, Task 2's DELETE statements must be updated to match — there's no try/except wrapper to mask a mismatch. + +- [ ] **Step 3: Verify `_profile/skill-inventory.md` structure (drives chunking)** + +```bash +grep -c '^## ' ~/Documents/compass-vault/_profile/skill-inventory.md +``` + +Expected: ≥ 15 level-2 headings. If the structure has materially changed, the chunking strategy (`## SkillName` per chunk) needs revisiting first. + +- [ ] **Step 4: Create branch** + +```bash +git checkout -b phase-1b2-rag +``` + +--- + +## Task 1: Register Pydantic state types with LangGraph checkpointer serde (carryover I1) + +**Why first:** Smallest, lowest-risk fix. Closes the msgpack deprecation warnings logged on every resume in Phase 1.B.1's smoke. Lands at the top of the phase so all subsequent tests benefit from a quieter log. + +**Files:** +- Modify: `compass/pipeline/graph.py` +- Create: `tests/pipeline/test_checkpoint_serde.py` + +**Background:** LangGraph 1.2 uses msgpack for checkpoint serialization. When it encounters an unregistered class (our `RawJob`, `JobRequirements`, `JobScore` Pydantic models), it logs `Deserializing unregistered type ... This will be blocked in a future version`. The clean fix is to construct a `JsonPlusSerializer` with explicit type registration and pass it to `AsyncSqliteSaver(serde=...)`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/pipeline/test_checkpoint_serde.py`: + +```python +"""Verify LangGraph checkpoint serde explicitly allowlists the + compass.pipeline.state module so the 'Deserializing unregistered type' + warnings (logged in 1.B.1 smoke) don't recur, and so a future + LANGGRAPH_STRICT_MSGPACK=true default doesn't break paused-thread + deserialization.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +async def test_build_checkpoint_serde_allowlists_state_module(): + """_build_checkpoint_serde must return a JsonPlusSerializer whose + allowlist suppresses the 'unregistered type' warning AND preserves + the Pydantic class on round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer + from compass.pipeline.graph import _build_checkpoint_serde + + # langgraph caches "already warned for this type" in a module-level set; + # clear it so this test sees a clean slate regardless of prior imports. + _jp._warned_unregistered_types.clear() + + serde = _build_checkpoint_serde() + assert isinstance(serde, JsonPlusSerializer) + + from compass.pipeline.state import RawJob + import logging + captured = [] + class _H(logging.Handler): + def emit(self, r): + if "unregistered type" in r.getMessage() or "fingerprint" in r.getMessage().lower(): + captured.append(r.getMessage()) + h = _H() + logging.getLogger("langgraph").addHandler(h) + try: + import datetime + rj = RawJob(company="C", title="T", url="u://x", source="ashby", + description="d", date_posted=datetime.date(2026, 5, 19)) + ser = serde.dumps_typed(rj) + restored = serde.loads_typed(ser) + assert isinstance(restored, RawJob) + finally: + logging.getLogger("langgraph").removeHandler(h) + assert not captured, f"serde still emitted unregistered-type warnings: {captured}" + + +async def test_run_pipeline_emits_no_unregistered_warnings(checkpoint_db, monkeypatch, temp_vault): + """Smoke: run_pipeline mounts AsyncSqliteSaver with our serde, not the + default. No 'unregistered type' warnings should be emitted during + a checkpoint round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from compass.pipeline.state import JobRequirements, JobScore, RawJob + import datetime as _dt + import logging + + # See note in test_build_checkpoint_serde_allowlists_state_module — clear + # the once-only warned-types set so this test isn't fooled by warnings + # already emitted in earlier imports. + _jp._warned_unregistered_types.clear() + + captured = [] + class _Handler(logging.Handler): + def emit(self, record): + if "unregistered type" in record.getMessage(): + captured.append(record.getMessage()) + handler = _Handler() + logging.getLogger("langgraph").addHandler(handler) + + # Stub the LLM-touching nodes (existing pattern from test_graph_checkpointing.py) + async def fake_intake_filter(_): return {"in_scope": True, "role_family": "agent-engineer"} + async def fake_extract(_): return {"extracted_requirements": JobRequirements( + required_skills=["MCP"], nice_to_have_skills=[], seniority="mid", + remote_policy="remote", summary="x")} + async def fake_score(_): return {"score_result": JobScore( + score=4.5, reasoning="ok", matched_skills=["MCP"], missing_skills=[], + tailoring_notes=""), "score_threshold": 3.5} + async def fake_tailor(_): return {"tailored_paragraph": "..."} + async def fake_vault_write(_): return {"vault_written": True, "jobs_written": 1} + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + # Auto-approve interrupt so the graph completes + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", + lambda _: {"approved": True, "feedback": None}) + + from compass.pipeline.graph import run_pipeline + await run_pipeline(raw_jobs=[RawJob( + company="Sierra", title="SWE", url="u://x/1", source="ashby", + description="...", date_posted=_dt.date(2026, 5, 19), + )]) + + logging.getLogger("langgraph").removeHandler(handler) + assert not captured, f"Expected no 'unregistered type' warnings, got: {captured}" +``` + +Run: `uv run pytest tests/pipeline/test_checkpoint_serde.py -v` — expect both to fail (`_build_checkpoint_serde` doesn't exist). + +- [ ] **Step 2: Implement `_build_checkpoint_serde` in `compass/pipeline/graph.py`** + +Add near the existing checkpointer imports: + +```python +from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer +``` + +Add as a module-level helper just above `build_graph`: + +```python +def _build_checkpoint_serde() -> JsonPlusSerializer: + """Allow our Pydantic state classes on the msgpack allowlist. + + Must be set at construction — `with_msgpack_allowlist` is a no-op when the + default `allowed_msgpack_modules=True` (langgraph's non-STRICT default). + """ + return JsonPlusSerializer( + allowed_msgpack_modules=[ + ("compass.pipeline.state", "RawJob"), + ("compass.pipeline.state", "JobRequirements"), + ("compass.pipeline.state", "JobScore"), + ] + ) +``` + +- [ ] **Step 3: Wire the serde into `AsyncSqliteSaver.from_conn_string`** + +In `run_pipeline`, change: + +```python +async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: +``` + +to: + +```python +async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() +``` + +(`from_conn_string(conn_string: str)` in checkpoint-sqlite 3.1 doesn't accept `serde=`. Setting `checkpointer.serde` after entering the context manager is the supported pattern.) + +Apply the same change in `compass/hitl/resume.py`'s `async with AsyncSqliteSaver.from_conn_string(...)` block — same pattern, same one line. Otherwise resume reuses the default serde and the warnings recur. + +- [ ] **Step 4: Run the tests** + +```bash +uv run pytest tests/pipeline/test_checkpoint_serde.py -v +``` + +Expected: 2 passed. If the second test still captures warnings, the serde isn't being applied to the active checkpointer — re-inspect `run_pipeline` and `resume.py`. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 241 passed (239 + 2 new). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/graph.py compass/hitl/resume.py tests/pipeline/test_checkpoint_serde.py +git commit -m "feat(hitl): register Pydantic state types with checkpoint serde" +``` + +--- + +## Task 2: Purge resolved-thread checkpoint blobs (carryover I-2) + +**Background:** Phase 1.B.1 left ~7.6MB in `~/.compass/checkpoints.db` after the smoke. Every paused thread accumulates ~10 checkpoint blobs and is never deleted post-resolution. At 50 jobs/day ÷ 5 paused, the file grows ~1MB/day forever. + +**Files:** +- Modify: `compass/hitl/resume.py` — `_purge_thread_checkpoints(thread_id)` called inside the `async with checkpointer` block after `mark_resolved` succeeds +- Modify: `tests/hitl/test_resume.py` — add a regression test + +- [ ] **Step 1: Write the failing test** + +Append to `tests/hitl/test_resume.py`: + +```python +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_purges_thread_checkpoint_blobs(stub_llm_nodes): + """After a thread is resolved (approved OR rejected), its checkpoint rows + in HITL_CHECKPOINT_DB must be deleted — otherwise the DB grows + unboundedly. Phase 1.B.1 post-smoke I-2 finding.""" + import aiosqlite + from compass.config import HITL_CHECKPOINT_DB + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob(company="Sierra", title="SWE", url="u://purge-probe", + source="ashby", description="...", date_posted=_dt.date(2026, 5, 19)) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + tid = (await state_store.list_pending())[0]["thread_id"] + + # Verify checkpoint rows exist before resume + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (pre_count,) = await cur.fetchone() + assert pre_count > 0, f"setup error: no checkpoint rows for {tid}" + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + + # Verify checkpoint rows for this thread are gone (state_store row is + # what stays — that's the audit trail, not the LangGraph internals). + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (post_count,) = await cur.fetchone() + assert post_count == 0, f"checkpoint rows for resolved thread {tid} were not purged" +``` + +Run: `uv run pytest tests/hitl/test_resume.py::test_resume_purges_thread_checkpoint_blobs -v` — expect failure. + +- [ ] **Step 2: Implement `_purge_thread_checkpoints` in `compass/hitl/resume.py`** + +Add a private helper at the bottom of the file: + +```python +async def _purge_thread_checkpoints(thread_id: str) -> None: + """Drop LangGraph per-step history for a resolved thread; state_store is the audit trail.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + await conn.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,)) + await conn.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,)) + await conn.commit() +``` + +Wire the call inside `resume_pending` AFTER `mark_resolved` succeeds AND AFTER the `async with AsyncSqliteSaver(...)` block exits — `_purge_thread_checkpoints` opens its own connection, so it doesn't need the saver's context: + +```python + async with AsyncSqliteSaver.from_conn_string(str(HITL_CHECKPOINT_DB)) as checkpointer: + checkpointer.serde = _build_checkpoint_serde() # carried from Task 1 + graph = build_graph(checkpointer=checkpointer) + final = await graph.ainvoke(Command(resume=decision), config=config) + + if status_override is not None: + resolved_status = status_override + else: + resolved_status = "approved" if final.get("human_approved") is True else "rejected" + await state_store.mark_resolved( + thread_id, status=resolved_status, feedback=decision.get("feedback"), + ) + await _purge_thread_checkpoints(thread_id) + + if final.get("vault_written"): + from compass.analysis import gap_aggregator + try: + gap_aggregator.regenerate(write=True) + except Exception: + logger.exception("hitl: gap_aggregator.regenerate failed after resume") +``` + +- [ ] **Step 3: Run the test** + +```bash +uv run pytest tests/hitl/test_resume.py -v +``` + +Expected: all 6 (5 prior + 1 new) pass. + +- [ ] **Step 4: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 242 passed (241 + 1). Ruff clean. + +- [ ] **Step 5: Commit** + +```bash +git add compass/hitl/resume.py tests/hitl/test_resume.py +git commit -m "fix(hitl): purge thread checkpoint blobs on resume (bound checkpoints.db growth)" +``` + +--- + +## Task 3: RAG indexer (`compass/rag/indexer.py`) + +**Files:** +- Create: `compass/rag/__init__.py` (empty) +- Create: `compass/rag/indexer.py` +- Create: `tests/rag/__init__.py` +- Create: `tests/rag/conftest.py` +- Create: `tests/rag/test_indexer.py` + +**Chunking strategy:** one chunk per `## SkillName` heading in `_profile/skill-inventory.md`. The chunk's `id` is the kebab-cased skill name (deterministic across rebuilds). The chunk's `metadata` carries `skill` (raw heading text), `section` (parent grouping if applicable; for now just "skill"), `source` ("skill-inventory.md"). The chunk's `document` is the section body text (heading + everything until the next `## ` or EOF). + +**Embedding model:** `all-MiniLM-L6-v2` via sentence-transformers. The model downloads on first use (~90MB to `~/.cache/torch/sentence_transformers/`); for tests we use a session-scoped fixture so it loads once. + +- [ ] **Step 1: Write the conftest** + +Create `tests/rag/conftest.py`: + +```python +"""Shared fixtures for RAG tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def temp_chroma_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Per-test Chroma index path — avoids cross-test pollution.""" + p = tmp_path / "chroma" + import compass.config as cfg + monkeypatch.setattr(cfg, "CHROMA_PATH", p) + return p + + +@pytest.fixture +def tiny_inventory(temp_vault, monkeypatch: pytest.MonkeyPatch) -> Path: + """Write a small skill-inventory.md to the temp vault — 3 distinct sections + so retrieval tests can verify which section gets surfaced.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "# Skill Inventory\n\n" + "## Python\n\n" + "Level 4. 5 years building backends and agents in Python. " + "Cisco MCP server, LangGraph pipelines, FastAPI services.\n\n" + "## LangGraph\n\n" + "Level 3. Built Compass pipeline with stateful graph, conditional " + "edges, interrupt()+AsyncSqliteSaver checkpointing for HITL.\n\n" + "## React\n\n" + "Level 1. Touched in school projects. Not a primary skill.\n", + encoding="utf-8", + ) + return inv + + +@pytest.fixture(scope="session") +def embedding_model_cached(): + """Pre-load the sentence-transformers model once per test session. + Subsequent fixtures + tests reuse the in-process cache.""" + from sentence_transformers import SentenceTransformer + # Touch it to force download/cache. Use the same name as compass.config. + SentenceTransformer("all-MiniLM-L6-v2") +``` + +- [ ] **Step 2: Write the failing indexer tests** + +Create `tests/rag/test_indexer.py`: + +```python +"""indexer parses skill-inventory.md by ## SkillName, embeds, persists. + Idempotent on rebuild (upsert by stable id = kebab-cased skill name). + + These tests use real sentence-transformers (the model is cached at session + scope; ~90MB on first run). They do NOT touch the network beyond the + initial model download.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_indexer_builds_one_chunk_per_section(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + count = await indexer.build_index() + assert count == 3, f"expected 3 chunks (Python/LangGraph/React), got {count}" + + +async def test_indexer_uses_kebab_case_ids(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + client = indexer._client() + col = client.get_collection("skill_inventory") + got_ids = set(col.get()["ids"]) + assert got_ids == {"python", "langgraph", "react"} + + +async def test_indexer_metadata_carries_skill_and_source(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + client = indexer._client() + col = client.get_collection("skill_inventory") + res = col.get(ids=["python"]) + assert res["metadatas"][0]["skill"] == "Python" + assert res["metadatas"][0]["source"] == "skill-inventory.md" + + +async def test_indexer_is_idempotent_on_rebuild(tiny_inventory, temp_chroma_path): + """A second build_index() must not duplicate rows. Upsert by id.""" + from compass.rag import indexer + + await indexer.build_index() + await indexer.build_index() # second call + client = indexer._client() + col = client.get_collection("skill_inventory") + assert col.count() == 3 + + +async def test_indexer_force_rebuild_clears_stale_chunks(tiny_inventory, temp_chroma_path): + """If the inventory shrinks (skill removed), force_rebuild=True must remove + the stale chunk; otherwise the old embedding lingers and pollutes retrieval.""" + from compass.rag import indexer + + await indexer.build_index() + tiny_inventory.write_text( + "# Skill Inventory\n\n## Python\n\nLevel 4.\n", + encoding="utf-8", + ) + await indexer.build_index(force_rebuild=True) + client = indexer._client() + col = client.get_collection("skill_inventory") + got_ids = set(col.get()["ids"]) + assert got_ids == {"python"} + + +async def test_indexer_repairs_stale_l2_collection(temp_vault, temp_chroma_path): + """An existing L2 collection from a pre-1.B.2 install must be dropped+recreated + with the cosine metric — otherwise the retriever score formula breaks silently.""" + from chromadb import PersistentClient + + from compass.rag import indexer + + # Simulate pre-1.B.2 state: collection exists with default (L2) metric. + client = PersistentClient(path=str(temp_chroma_path)) + client.create_collection(name="skill_inventory") # no cosine metadata + + col = indexer._collection(client) + assert (col.metadata or {}).get("hnsw:space") == "cosine" + + +async def test_indexer_excludes_assessor_grades_heading(temp_vault, temp_chroma_path): + """Auto-generated `## Assessor-current grades` is metadata, not a skill — + must NOT land in the index or it pollutes retrieval with every skill name.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "## Python\n\nLevel 4.\n\n" + "## Assessor-current grades\n\nPython: 4 / LangGraph: 3 / MCP: 5\n", + encoding="utf-8", + ) + from compass.rag import indexer + + n = await indexer.build_index() + assert n == 1 + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_handles_empty_inventory_gracefully(temp_vault, temp_chroma_path): + """No ## sections — count is 0, no crash, no collection bloat.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Skill Inventory\n\nNo skills yet.\n", encoding="utf-8") + from compass.rag import indexer + + count = await indexer.build_index() + assert count == 0 +``` + +Run: `uv run pytest tests/rag/test_indexer.py -v` — expect all to fail with `ModuleNotFoundError: compass.rag.indexer`. + +- [ ] **Step 3: Implement `compass/rag/indexer.py`** + +```python +"""Chroma index of _profile/skill-inventory.md. + +One chunk per `## SkillName` section. Collection metric is cosine — the +retriever's similarity score formula depends on this. +""" + +from __future__ import annotations + +import asyncio +import logging +import re + +logger = logging.getLogger(__name__) + +_COLLECTION_NAME = "skill_inventory" +_SECTION_RE = re.compile(r"^## +(.+)$", re.M) +# Auto-generated assessor output; not a skill — exclude from the index. +_EXCLUDED_HEADINGS = {"Assessor-current grades"} + + +def _client(): + import compass.config as cfg + from chromadb import PersistentClient + + cfg.CHROMA_PATH.mkdir(parents=True, exist_ok=True) + return PersistentClient(path=str(cfg.CHROMA_PATH)) + + +def _collection(client): + """Return the skill_inventory collection pinned to cosine distance. + + `get_or_create_collection` does NOT update an existing collection's + metadata — so a pre-existing L2 collection would silently keep the wrong + metric and break the retriever's score formula. Drop and recreate if so. + """ + existing = {c.name for c in client.list_collections()} + if _COLLECTION_NAME in existing: + col = client.get_collection(_COLLECTION_NAME) + if (col.metadata or {}).get("hnsw:space") == "cosine": + return col + client.delete_collection(_COLLECTION_NAME) + return client.create_collection( + name=_COLLECTION_NAME, + metadata={"hnsw:space": "cosine"}, + ) + + +def _kebab(name: str) -> str: + # Drop parenthetical asides and em-dash commentary — they're not identity. + # `Fine-Tuning (awareness only — ...)` -> `fine-tuning` + # `MCP — your strongest cluster` -> `mcp` + primary = re.sub(r"\s*(?:\(|[—–]\s).*$", "", name).strip() + return re.sub(r"[^a-z0-9]+", "-", primary.lower()).strip("-") or "untitled" + + +def _parse_inventory(text: str) -> list[tuple[str, str]]: + """Return [(skill_name, section_text), ...] — one entry per ## heading. + + Skips headings in _EXCLUDED_HEADINGS (auto-generated metadata that isn't + a real skill and would otherwise pollute retrieval). + """ + matches = list(_SECTION_RE.finditer(text)) + sections = [] + for i, m in enumerate(matches): + name = m.group(1).strip() + if any(name.startswith(prefix) for prefix in _EXCLUDED_HEADINGS): + continue + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + sections.append((name, text[m.start():end].strip())) + return sections + + +def _embed(documents: list[str]) -> list[list[float]]: + import compass.config as cfg + from sentence_transformers import SentenceTransformer + + model = SentenceTransformer(cfg.EMBEDDING_MODEL) + arr = model.encode(documents, convert_to_numpy=True, show_progress_bar=False) + return [vec.tolist() for vec in arr] + + +async def build_index(force_rebuild: bool = False) -> int: + """Embed each `## Section` of skill-inventory.md and upsert into Chroma. + + Idempotent — repeat calls upsert by stable id. Pass `force_rebuild=True` + to drop the collection first (use when the inventory shrinks; otherwise + deleted skills' stale chunks linger). + """ + import compass.config as cfg + + if not cfg.SKILL_INVENTORY_PATH.exists(): + logger.warning("rag: skill-inventory.md not found at %s", cfg.SKILL_INVENTORY_PATH) + return 0 + + sections = _parse_inventory(cfg.SKILL_INVENTORY_PATH.read_text(encoding="utf-8")) + if not sections: + logger.info("rag: 0 ## sections in skill-inventory.md; nothing to index") + return 0 + + client = _client() + if force_rebuild and _COLLECTION_NAME in {c.name for c in client.list_collections()}: + client.delete_collection(_COLLECTION_NAME) + collection = _collection(client) + + documents = [body for _, body in sections] + embeddings = await asyncio.to_thread(_embed, documents) + + collection.upsert( + ids=[_kebab(name) for name, _ in sections], + documents=documents, + embeddings=embeddings, + metadatas=[ + {"skill": name, "source": "skill-inventory.md"} for name, _ in sections + ], + ) + logger.info("rag: indexed %d sections at %s", len(sections), cfg.CHROMA_PATH) + return len(sections) + + +def _main() -> None: + import argparse + + import compass.config as cfg + + parser = argparse.ArgumentParser(description="Rebuild Chroma index of skill-inventory.md") + parser.add_argument("--force", action="store_true", help="Drop+rebuild the collection") + n = asyncio.run(build_index(force_rebuild=parser.parse_args().force)) + print(f"Indexed {n} sections from {cfg.SKILL_INVENTORY_PATH}") + + +if __name__ == "__main__": + _main() +``` + +- [ ] **Step 4: Run the tests** + +```bash +uv run pytest tests/rag/test_indexer.py -v +``` + +Expected: 8 passed. First run downloads the model (~90MB, 30-60s on first install). Subsequent runs are fast. + +If a test fails because Chroma's `get_or_create_collection` API differs in your installed version, check the actual `chromadb` 1.5 API and adjust the call. The public surface has been stable in 0.5+. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 250 passed (242 + 8). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/rag/__init__.py compass/rag/indexer.py tests/rag/__init__.py \ + tests/rag/conftest.py tests/rag/test_indexer.py +git commit -m "feat(rag): chroma index of skill-inventory.md (one chunk per ## section)" +``` + +--- + +## Task 4: RAG retriever (`compass/rag/retriever.py`) + +**Files:** +- Create: `compass/rag/retriever.py` +- Create: `tests/rag/test_retriever.py` + +- [ ] **Step 1: Write the failing retriever tests** + +Create `tests/rag/test_retriever.py`: + +```python +"""retriever.retrieve(query, k) returns top-k chunks from the Chroma index. + Lazy-init: builds the index on miss so first-run usage doesn't crash.""" + +from __future__ import annotations + +import pytest + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_retrieve_returns_top_k_relevant_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + + hits = await retriever.retrieve("Python backend agent work", k=2) + assert len(hits) == 2 + # Soft assertion — embedding-model top-1 can flap on a 3-doc corpus. + # The real contract is "Python ranks in the top-2", not "exactly first". + skills = [h.skill for h in hits] + assert "Python" in skills + assert any("Cisco MCP" in h.document for h in hits) # body content present + assert all(0.0 <= h.score <= 1.0 for h in hits) # normalized similarity scores + + +async def test_retrieve_lazy_inits_index_on_miss(tiny_inventory, temp_chroma_path): + """If no index exists yet, retrieve() builds one before querying.""" + from compass.rag import retriever + + # No build_index() call beforehand + hits = await retriever.retrieve("LangGraph stateful pipeline", k=1) + assert len(hits) == 1 + assert hits[0].skill == "LangGraph" + + +async def test_retrieve_returns_empty_when_inventory_empty(temp_vault, temp_chroma_path): + """No sections in inventory → retrieve returns [] gracefully (no crash).""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Empty\n", encoding="utf-8") + from compass.rag import retriever + + hits = await retriever.retrieve("anything", k=5) + assert hits == [] + + +async def test_retrieve_k_larger_than_corpus_returns_all(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + hits = await retriever.retrieve("python or react", k=99) + assert 1 <= len(hits) <= 3 # corpus has 3 chunks; can't exceed +``` + +Run: `uv run pytest tests/rag/test_retriever.py -v` — expect failures (`compass.rag.retriever` doesn't exist). + +- [ ] **Step 2: Implement `compass/rag/retriever.py`** + +```python +"""Top-k retrieval over the skill-inventory Chroma index. + +Lazy-init on first call so scoring works before any explicit indexer run. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from compass.rag import indexer + + +@dataclass +class RetrievedChunk: + skill: str + document: str + score: float # cosine similarity in [0, 1]; 1.0 = identical + + +async def retrieve(query: str, k: int = 8) -> list[RetrievedChunk]: + if not (query or "").strip(): + return [] + + collection = indexer._collection(indexer._client()) + if collection.count() == 0 and await indexer.build_index() == 0: + return [] + + [query_emb] = await asyncio.to_thread(indexer._embed, [query]) + res = collection.query( + query_embeddings=[query_emb], + n_results=min(k, collection.count()), + ) + + # Collection is pinned to cosine (see indexer._collection); distance ∈ [0, 2]. + return [ + RetrievedChunk(skill=m["skill"], document=d, score=max(0.0, 1.0 - dist / 2.0)) + for m, d, dist in zip( + res["metadatas"][0], res["documents"][0], res["distances"][0], strict=True + ) + ] +``` + +- [ ] **Step 3: Run the tests** + +```bash +uv run pytest tests/rag/test_retriever.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 254 passed (250 + 4). Ruff clean. + +- [ ] **Step 5: Commit** + +```bash +git add compass/rag/retriever.py tests/rag/test_retriever.py +git commit -m "feat(rag): semantic retriever with lazy index init" +``` + +--- + +## Task 5: Wire retriever into `score_node` + +**Files:** +- Modify: `compass/pipeline/nodes/score.py` +- Create: `tests/pipeline/test_score_node_rag.py` + +**Strategy:** the existing `_profile_text()` returns `## RESUME\n{full resume}\n\n## SKILL INVENTORY\n{full inventory}`. The new version returns `## RESUME\n{full resume}\n\n## RELEVANT SKILLS\n{top-k chunks joined}`. Resume stays inline (it's small and contextual); inventory is replaced by retrieval. + +**Query construction:** join the JD's `required_skills + nice_to_have_skills + summary` into a single string. This embeds the candidate-relevant query in one shot. K=8 covers most expected coverage. + +- [ ] **Step 1: Write the failing test** + +Create `tests/pipeline/test_score_node_rag.py`: + +```python +"""score_node uses RAG retrieval to build profile context, not full inventory.""" + +from __future__ import annotations + +import pytest + +from compass.pipeline.state import JobRequirements + + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +def _stub_req(**overrides) -> JobRequirements: + base = dict( + required_skills=["Python", "LangGraph"], + nice_to_have_skills=["MCP"], + seniority="senior", + remote_policy="remote", + summary="Build agentic systems.", + ) + base.update(overrides) + return JobRequirements(**base) + + +async def test_profile_text_includes_resume_and_retrieved_chunks(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME TEXT") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME TEXT" in text + assert "## Python\nLevel 4." in text + + +async def test_retrieval_query_carries_jd_skills_and_summary(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + captured: dict[str, object] = {} + + async def fake_retrieve(query, k=8): + captured["query"] = query + return [RetrievedChunk(skill="Python", document="## Python", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "") + + await score_mod._profile_text(_stub_req()) + q = captured["query"] + assert "Python" in q and "LangGraph" in q and "MCP" in q + assert "Build agentic systems" in q + + +async def test_score_node_handles_empty_jd_skills(monkeypatch): + """A JD with no required/nice-to-have skills falls back to summary only.""" + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query: str, k: int = 8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME") + + req = JobRequirements( + required_skills=[], nice_to_have_skills=[], + seniority="mid", remote_policy="remote", summary="Just a summary.", + ) + text = await score_mod._profile_text(req) + assert "RESUME" in text + assert "Python" in text # still got a retrieved chunk + + +async def test_score_node_handles_empty_retrieval(monkeypatch): + """If retriever returns [] (e.g. empty corpus), the profile context still + includes the resume — score_node doesn't crash.""" + from compass.pipeline.nodes import score as score_mod + + async def fake_retrieve(query: str, k: int = 8): + return [] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME ONLY") + + req = JobRequirements( + required_skills=["X"], nice_to_have_skills=[], + seniority="mid", remote_policy="remote", summary="x", + ) + text = await score_mod._profile_text(req) + assert "RESUME ONLY" in text + # No "## RELEVANT SKILLS" block when retrieval is empty +``` + +Run: `uv run pytest tests/pipeline/test_score_node_rag.py -v` — expect failures (the score module doesn't import rag_retrieve and _profile_text is currently sync). + +- [ ] **Step 2: Rewrite `_profile_text` in `compass/pipeline/nodes/score.py`** + +Two edits to the file: + +**2a. Add the import near the top (alongside `read_resume` / `read_skill_inventory`):** + +```python +from compass.rag.retriever import retrieve as rag_retrieve +``` + +You can REMOVE the `read_skill_inventory` import from `score.py` only — `_profile_text` no longer uses it. **Do NOT** delete `read_skill_inventory` from `compass/vault/reader.py` — `tests/vault/test_reader.py` still exercises it, and the function may be useful for future tooling outside the pipeline. + +**2b. Rewrite `_profile_text` from sync to async, accepting `req: JobRequirements`:** + +```python +async def _profile_text(req: JobRequirements) -> str: + """Build the candidate-profile context for the score prompt. + + RAG via Chroma replaces the prior wholesale-inject of skill-inventory.md. + Resume stays inline (small, load-bearing); inventory is now top-k chunks + retrieved against the JD's skill + summary query. + """ + query_parts = [*req.required_skills, *req.nice_to_have_skills] + if req.summary: + query_parts.append(req.summary) + query = " ".join(query_parts).strip() + + chunks = await rag_retrieve(query, k=8) if query else [] + + profile = f"## RESUME\n{read_resume()}" + if chunks: + ranked = "\n\n".join(c.document for c in chunks) + profile += f"\n\n## RELEVANT SKILLS (top-{len(chunks)} by similarity)\n{ranked}" + return profile +``` + +**2c. Update the caller in `score_node`:** + +```python + profile = await _profile_text(req) + result = await _score_with_retry(req, profile) +``` + +(Currently it's `await _score_with_retry(req, _profile_text())` — split into two lines and pass `req` to the new async helper. **Do not modify the `return {...}` block at the end of `score_node` — the `"score_threshold": SCORE_THRESHOLD` key was added in Phase 1.B.1 and must stay.**) + +- [ ] **Step 3: Run the new tests** + +```bash +uv run pytest tests/pipeline/test_score_node_rag.py -v +``` + +Expected: 4 passed. + +- [ ] **Step 4: Confirm existing score_node tests still pass** + +```bash +uv run pytest tests/pipeline/ -v +``` + +Expected: all green. If existing tests fail because they assumed `_profile_text()` is sync, the test file needs minor updates — usually wrapping a sync `_profile_text()` call site with `asyncio.run()` or making the test async. Inspect the test names that fail and patch them; do NOT change the production code to compensate. + +- [ ] **Step 5: Full suite + lint** + +```bash +uv run pytest -q +uv run ruff check && uv run ruff format --check +``` + +Expected: 258 passed (254 + 4). Ruff clean. + +- [ ] **Step 6: Commit** + +```bash +git add compass/pipeline/nodes/score.py tests/pipeline/test_score_node_rag.py +git commit -m "feat(rag): score_node uses Chroma retrieval instead of full skill-inventory inject" +``` + +--- + +## Task 6: Live smoke + retro + +> **PAUSE HERE before running this task.** Step 3 hits real OpenRouter on at least one JD with the new RAG-injected context. Confirm with the user before running. Cost: ~$0.01 for a single re-score, ~$0.05 for a full small-board pipeline run. + +**Goal:** Prove three things tests cannot prove: +1. The Chroma index actually builds against the real `~/Documents/compass-vault/_profile/skill-inventory.md` (21 sections, not the test fixture's 3) +2. Retrieval against a real JD's skills returns sensible chunks +3. score_node with RAG produces a score consistent with the pre-RAG score (no major drift) + +- [ ] **Step 1: Snapshot the vault and DBs** + +```bash +cp -r ~/Documents/compass-vault/jobs ~/Documents/compass-vault/jobs.preB2.bak 2>/dev/null || true +cp ~/.compass/hitl.db ~/.compass/hitl.db.preB2.bak 2>/dev/null || true +cp ~/.compass/checkpoints.db ~/.compass/checkpoints.db.preB2.bak 2>/dev/null || true +``` + +- [ ] **Step 2: Build the index against the real inventory** + +```bash +uv run python -m compass.rag.indexer --force +``` + +Expected output: `Indexed: N chunks` where N matches `grep -c '^## ' ~/Documents/compass-vault/_profile/skill-inventory.md` (currently 21). + +Verify the index exists: + +```bash +ls -la ~/.compass/chroma/ +``` + +Expected: at least one `.sqlite3` file present. First run may also populate `~/.cache/torch/sentence_transformers/` with the ~90MB model. + +- [ ] **Step 3: Probe retrieval against a real JD profile** + +```bash +uv run python -c " +import asyncio +from compass.rag.retriever import retrieve +async def main(): + hits = await retrieve('agentic AI engineer with LangGraph MCP Python production systems', k=5) + for h in hits: + print(f'{h.score:.3f} {h.skill}') +asyncio.run(main()) +" +``` + +Expected: top-5 results include skills like Python, MCP, LangGraph, Agent Frameworks, RAG (or similar — the inventory's strongest sections). Scores should range roughly 0.4–0.7. If top-5 has irrelevant skills (e.g. "Voice", "Fine-Tuning"), the embedding model isn't separating the inventory well — flag as a quality concern but don't block. + +- [ ] **Step 4: Re-score one existing JobNote with the new pipeline** + +Pick any previously-approved JobNote (e.g. `~/Documents/compass-vault/jobs/2024-10-05-cognition-Special_Projects_Engineer-5343fadf.md` from the Phase 1.B.1 smoke, if it still exists; otherwise pick any JobNote with a known pre-RAG match_score). Note the pre-RAG score; expect the post-RAG score to land within ±0.5. + +```bash +# Re-score via the MCP score_jd tool — bypasses vault writes +uv run python -c " +import asyncio +from compass.mcp_server.server import score_jd +async def main(): + jd_text = open('/Users/akmini/Documents/compass-vault/jobs/2024-10-05-cognition-Special_Projects_Engineer-5343fadf.md').read() + result = await score_jd(jd_text) + print(f'RAG score: {result[\"score\"][\"score\"]}') + print(f'matched: {result[\"score\"][\"matched_skills\"]}') + print(f'missing: {result[\"score\"][\"missing_skills\"]}') +asyncio.run(main()) +" +``` + +Expected: score within 2.5–3.5 (pre-RAG was 3.0). If the new score is wildly different (e.g. 0.5 or 5.0), the retrieval context is missing something critical — investigate the retrieved chunks vs the full inventory before proceeding. + +- [ ] **Step 5: Run a small live pipeline** + +```bash +MAX_JOBS_PER_RUN=5 \ + GREENHOUSE_BOARDS=anthropic \ + LEVER_COMPANIES= \ + ASHBY_BOARDS=decagon \ + uv run python -m compass.pipeline.graph +``` + +Expected output: pipeline completes with no `Deserializing unregistered type` warnings (Task 1 closes them) and no abnormal score drift. Inspect `_meta/agent-log.md` for any errors related to RAG. + +- [ ] **Step 6: Verify checkpoint DB stays bounded** + +After Step 5 + any resume action: + +```bash +sqlite3 ~/.compass/checkpoints.db "SELECT COUNT(DISTINCT thread_id) FROM checkpoints;" +``` + +Expected: count equals the number of still-pending threads (not the total number of threads ever paused). Task 2's purge keeps it bounded. + +- [ ] **Step 7: Tag the phase** + +```bash +git tag phase-1b2-rag +``` + +- [ ] **Step 8: Cleanup or keep backups** + +If everything looks healthy: + +```bash +rm -rf ~/Documents/compass-vault/jobs.preB2.bak ~/.compass/hitl.db.preB2.bak ~/.compass/checkpoints.db.preB2.bak +``` + +If anything looks wrong, restore: + +```bash +rm -rf ~/Documents/compass-vault/jobs && mv ~/Documents/compass-vault/jobs.preB2.bak ~/Documents/compass-vault/jobs +mv ~/.compass/hitl.db.preB2.bak ~/.compass/hitl.db +mv ~/.compass/checkpoints.db.preB2.bak ~/.compass/checkpoints.db +``` + +--- + +## Definition of Done + +**Code & tests** +- 258 tests passing (`uv run pytest -q`) +- Ruff clean (`uv run ruff check && uv run ruff format --check`) +- No module-level captures of `CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH` — every reference is inside a function body +- `_build_checkpoint_serde` is called inside the `async with AsyncSqliteSaver(...)` block in BOTH `run_pipeline` and `resume_pending` (carry forward the Phase 1.A discipline) +- `_purge_thread_checkpoints` runs only after `mark_resolved` succeeds, and inside the open checkpointer context + +**Behaviour (verified empirically in Task 6)** +- `python -m compass.rag.indexer --force` indexes all 21 sections of the real inventory +- Retrieval against a real JD-skills query surfaces sensible top-5 chunks +- A re-score of one existing JobNote produces a score within ±0.5 of the pre-RAG score +- A full small-board pipeline run completes with no `Deserializing unregistered type` warnings +- `checkpoints.db` size stays bounded — resolved threads' rows are deleted + +**Docs** +- Plan file's "What's deferred" table updated if any new items surface +- Phase 1.B.2 retrospective (optional but recommended) saved to `docs/PHASE_1B2_COMPLETE.md` + +--- + +## What's deferred (and to which sub-phase) + +| Concern | Severity | Phase | Why deferred | +|---|---|---|---| +| 30-JD eval harness comparing pre-RAG vs post-RAG score MAE | Portfolio claim | **2.A** | Needs labeled dataset; out of 1.B.2 scope | +| Modal cron + Langfuse callback fix + URL-dedup for filtered-jobs | Required for "no babysitting" | **1.B.3** | Spec phase ordering | +| I4 double-resume race claim_pending (Phase 1.B.1 deferred item) | Important | **1.B.3** | Cron is the real-world race trigger | +| Embedding-model upgrade (e.g. `nomic-embed-text` for higher quality) | Quality polish | **2.B** | `all-MiniLM-L6-v2` is good enough for v1; cost-free swap later | +| Skill-inventory re-indexing on Obsidian write (file-watch) | UX | **3+** | Manual `python -m compass.rag.indexer` is fine for solo use | +| Per-chunk metadata enrichment (level, category, calibration) | Quality | **2.B** | Current metadata = {skill, source} suffices for top-k retrieval | + +--- + +## Critical lessons to carry forward (from Phases 0 + 1.A + 1.B.1) + +1. **Tests check shape; only adversarial real-data inspection catches data-correctness bugs.** Task 6 is non-optional — do not declare 1.B.2 done from green CI alone. +2. **Module-level config captures freeze at import time** and silently break test fixtures. Always late-bind inside function bodies (`CHROMA_PATH`, `EMBEDDING_MODEL`, `SKILL_INVENTORY_PATH`, `HITL_CHECKPOINT_DB`). +3. **A graph compiled without a checkpointer silently breaks `interrupt()`** — apply the same discipline to the new serde: `_build_checkpoint_serde` must be called INSIDE the `async with` block, NEVER stashed in a module-level constant. +4. **JSON-serializability of MCP return shapes.** Phase 1.A bug #11: `date` objects don't serialize through FastMCP. Same applies to RAG retrieval — if any future MCP tool returns `RetrievedChunk`, convert to a plain dict first. +5. **`or`-fallback against falsy values is a recurring trap.** Phase 1.B.1 caught `state.get("score_threshold") or SCORE_THRESHOLD` would mis-fallback on 0.0. Same pattern applies anywhere we read state with a defensible-falsy default — use `is None` checks instead. +6. **The audit-trail/state-store/JobNote consistency triad is load-bearing.** Phase 1.B.1's C1 silent divergence (state_store said "approved", JobNote said "auto_rejected") was caught only by adversarial cross-checking. Apply the same rigor here: after any pipeline change, sample real JobNotes and confirm `match_score`, `matched_skills`, `missing_skills` are consistent with what RAG retrieved. + +--- + +## Plan Review Loop + +Before executing this plan, dispatch a single plan-document-reviewer subagent with: +- Path to this plan: `docs/superpowers/plans/2026-05-19-compass-phase-1b2-rag.md` +- Path to spec: `docs/superpowers/specs/2026-05-17-compass-mvp-to-portfolio-ship-design.md` +- Phase scope reference: this plan's "Closes these deferred items" section + +If the reviewer flags issues, fix in place and re-dispatch. Cap at 3 iterations. + +--- + +## Execution handoff + +Once approved, two options: + +1. **Subagent-Driven (recommended)** — Use `superpowers:subagent-driven-development`. Fresh implementer per Task 1–5, combined spec+quality reviewer between tasks, hard pause before Task 6 (live LLM + real vault). Same pattern that worked in Phase 1.B.1. + +2. **Inline Execution** — Use `superpowers:executing-plans`. Single session, batch through Tasks 1–5 with checkpoints, hard pause before Task 6. + +Phase 1.B.1 used Subagent-Driven and it caught 5+ real bugs across tasks. Keep the pattern. diff --git a/scripts/label_jd.py b/scripts/label_jd.py new file mode 100644 index 0000000..f56d288 --- /dev/null +++ b/scripts/label_jd.py @@ -0,0 +1,200 @@ +"""Interactive CLI for labeling JDs into the eval dataset. + +Two modes: + uv run python -m scripts.label_jd <JobNote filename> + → reads compass-vault/jobs/<JobNote>, shows the body + summary, runs + Compass's current extract on it, displays the agent's output, prompts + for your expected_score + expected_skills + notes. Appends to dataset. + + uv run python -m scripts.label_jd + → interactive mode without a JobNote: paste a JD, supply score/skills, + save. Useful for labeling JDs you found manually outside the vault. + +Mirrors the spec's `scripts/label_jd.py` requirement (Phase 2.A). Designed +to make hand-labeling 20 JDs over an hour feel fast — defaults, completion- +hints, and one-keystroke "use the agent's extract as ground truth" path. +""" + +from __future__ import annotations + +import argparse +import sys + +import frontmatter + +# IMPORTANT: read VAULT_PATH at call time via `cfg.VAULT_PATH`, not via +# `from compass.config import VAULT_PATH`. The latter freezes the import-time +# value and silently breaks the `temp_vault` test fixture (which monkeypatches +# cfg.VAULT_PATH). See CLAUDE.md lesson #2. +import compass.config as cfg +from compass.evals.dataset import add_example, load_dataset + + +def _prompt(label: str, default: str = "") -> str: + suffix = f" [{default}]" if default else "" + raw = input(f"{label}{suffix}: ").strip() + return raw or default + + +def _prompt_float(label: str, *, lo: float, hi: float, default: float | None = None) -> float: + while True: + raw = _prompt(label, default=("" if default is None else str(default))) + try: + val = float(raw) + except ValueError: + print(f" ! not a number — enter a value in [{lo}, {hi}]") + continue + if val < lo or val > hi: + print(f" ! out of range — enter a value in [{lo}, {hi}]") + continue + return val + + +def _prompt_skills(label: str, default: list[str] | None = None) -> list[str]: + default_str = ", ".join(default or []) + raw = _prompt(label, default=default_str) + if not raw: + return [] + return [s.strip() for s in raw.split(",") if s.strip()] + + +def _load_jobnote(filename: str) -> tuple[str, str, str, str]: + """Return (jd_text, company, title, source_label) from a JobNote on disk.""" + jobs_dir = (cfg.VAULT_PATH / "jobs").resolve() + path = (cfg.VAULT_PATH / "jobs" / filename).resolve() + try: + path.relative_to(jobs_dir) + except ValueError as e: + raise SystemExit(f"job_filename must be inside {jobs_dir}: {filename!r}") from e + if not path.exists(): + raise SystemExit(f"JobNote not found: {filename}") + post = frontmatter.load(path) + company = str(post.metadata.get("company") or "(unknown)") + title = str(post.metadata.get("title") or "(unknown)") + text = post.content + if "## Full JD" in text: + text = text.split("## Full JD", 1)[1].strip() + return text, company, title, f"{path.name} ({company} — {title})" + + +async def _show_agent_extract(jd_text: str) -> tuple[float | None, list[str]]: + """Run Compass's extract on the JD and print the agent's reading so the + human can quickly validate or override. Returns (None, agent_skills) — + score isn't shown because there's no profile-anchored fit until the + score node runs, and we don't want to anchor the labeler on a number.""" + from compass.pipeline.nodes.extract import ( + _extract, + _normalize_skill_list, + _seniority_with_title_fallback, + ) + + print("\n→ running Compass extract (this is one LLM call, ~$0.001)…") + try: + raw = await _extract(jd_text) + except Exception as e: + print(f" extract failed: {e}") + return None, [] + # Normalize the same way the production pipeline does so the labeler + # sees the canonical skill names. + unknown: list[str] = [] + required = _normalize_skill_list(raw.required_skills, jd_text, unknown) + nice = _normalize_skill_list(raw.nice_to_have_skills, jd_text, unknown) + print(f" required: {', '.join(required) or '(none)'}") + print(f" nice-to-have: {', '.join(nice) or '(none)'}") + print(f" seniority: {_seniority_with_title_fallback(raw.seniority, '')}") + if unknown: + print(f" ! unknown to taxonomy (dropped): {', '.join(unknown)}") + return None, list(dict.fromkeys(required + nice)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "jobnote", + nargs="?", + help="JobNote filename (e.g. 2026-05-19-sierra-Engineer-abc.md). " + "Omit for interactive paste mode.", + ) + parser.add_argument( + "--score", + type=float, + help="Skip the score prompt and use this value (0-5).", + ) + parser.add_argument( + "--skills", + help="Comma-separated expected skills (skips the skills prompt).", + ) + parser.add_argument( + "--no-agent-extract", + action="store_true", + help="Don't run Compass's extract — useful when you want a totally " + "blind label (no anchoring on the agent's output).", + ) + args = parser.parse_args() + + if args.jobnote: + jd_text, company, title, source = _load_jobnote(args.jobnote) + print(f"\nLoaded JobNote: {company} — {title}") + print(f"JD length: {len(jd_text)} chars\n") + print("--- JD body (first 800 chars) ---") + print(jd_text[:800] + ("…" if len(jd_text) > 800 else "")) + print("---\n") + else: + print("Paste the JD body, then end with Ctrl-D on a blank line:") + jd_text = sys.stdin.read().strip() + if not jd_text: + print("Empty input — exiting.") + return 1 + company = _prompt("company") + title = _prompt("title") + source = f"manual: {company} — {title}" + + agent_skills: list[str] = [] + if not args.no_agent_extract: + import asyncio + + _, agent_skills = asyncio.run(_show_agent_extract(jd_text)) + + print() + score = ( + args.score + if args.score is not None + else _prompt_float( + "expected_score (0-5)", + lo=0.0, + hi=5.0, + ) + ) + if args.skills is not None: + expected_skills = [s.strip() for s in args.skills.split(",") if s.strip()] + else: + expected_skills = _prompt_skills( + "expected_skills (comma-sep, [Enter] accepts agent's list)", + default=agent_skills, + ) + notes = _prompt("notes (optional)") + + print("\nLabeled record preview:") + print(f" source: {source}") + print(f" expected_score: {score}") + print(f" expected_skills: {expected_skills}") + print(f" notes: {notes!r}") + confirm = _prompt("save? [Y/n]", default="Y").lower() + if confirm not in ("y", "yes", ""): + print("Aborted.") + return 1 + + record = add_example( + jd_text=jd_text, + expected_score=score, + expected_skills=expected_skills, + source=source, + notes=notes, + ) + n_total = len(load_dataset()) + print(f"\n✓ Saved {record.id}. Dataset now has {n_total} labeled JD(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate_jobnote_skills_section.py b/scripts/migrate_jobnote_skills_section.py new file mode 100644 index 0000000..03961d1 --- /dev/null +++ b/scripts/migrate_jobnote_skills_section.py @@ -0,0 +1,118 @@ +"""One-time migration: backfill the `## Skills` wikilink section into existing JobNote bodies. + +Day-1 Obsidian P1 work added a `## Skills` block to `write_job_note` that +renders `[[Python]] · [[LangGraph]]` style wikilinks so the Obsidian graph view +edges JobNotes to SkillNotes. Existing JobNotes written before this change +have the right frontmatter but no body section; this script rewrites them +in-place by re-rendering with the current `write_job_note` logic. + +Idempotent: re-running is safe — replaces an existing `## Skills` block if +present, inserts before `## Full JD` (or appends) otherwise. + +Dry-run by default; --apply to commit. + +Usage: + uv run python -m scripts.migrate_jobnote_skills_section # dry-run + uv run python -m scripts.migrate_jobnote_skills_section --apply # commit +""" + +from __future__ import annotations + +import argparse +import re +import sys +from typing import TYPE_CHECKING + +import frontmatter + +from compass.config import VAULT_PATH +from compass.vault.schemas import JobNote +from compass.vault.writer import _render_skills_section + +if TYPE_CHECKING: + from pathlib import Path + + +# Matches `## Skills` heading through (but not including) the next `## ` heading or EOF. +_SKILLS_BLOCK = re.compile(r"(?ms)^## Skills\s*\n.*?(?=^## |\Z)") + + +def _splice_skills_section(body: str, rendered: str) -> str: + """Insert or replace the `## Skills` block in a JobNote body. + + - If a `## Skills` block already exists, replace it (re-runs stay clean). + - Else, insert before the first `## ` heading (typically `## Full JD`). + - Else, append at end. + - If `rendered` is empty (note has zero skills in all four lists), strip + any existing block and return. + """ + if _SKILLS_BLOCK.search(body): + # The regex consumes through the blank line separator before the next + # heading. Re-add a single blank line so subsequent re-runs are no-ops. + # If `rendered` is empty (no skills), substitute with empty — collapse + # leftover whitespace below. + replacement = (rendered + "\n") if rendered else "" + new_body = _SKILLS_BLOCK.sub(replacement, body, count=1) + return re.sub(r"\n{3,}", "\n\n", new_body) + + if not rendered: + return body + + next_heading = re.search(r"(?m)^## ", body) + block = rendered if rendered.endswith("\n") else rendered + "\n" + if next_heading: + idx = next_heading.start() + return body[:idx] + block + "\n" + body[idx:] + # No subheading at all — append after a blank line. + return body.rstrip() + "\n\n" + block + + +def _rewrite_body(path: Path) -> tuple[bool, str]: + """Return (changed, new_full_text). Does not write.""" + post = frontmatter.load(path) + try: + note = JobNote.model_validate(post.metadata) + except Exception as exc: + print(f" SKIP {path.name}: frontmatter does not validate as JobNote ({exc})") + return False, "" + rendered = _render_skills_section(note) + new_body = _splice_skills_section(post.content, rendered) + if new_body == post.content: + return False, "" + post.content = new_body + return True, frontmatter.dumps(post) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Actually rewrite the JobNotes") + args = parser.parse_args() + + jobs_dir = VAULT_PATH / "jobs" + paths = sorted(jobs_dir.glob("*.md")) + changes: list[tuple[Path, str]] = [] + for path in paths: + changed, new_text = _rewrite_body(path) + if changed: + changes.append((path, new_text)) + + if not changes: + print(f"All {len(paths)} JobNote(s) already have a current `## Skills` section.") + return 0 + + print(f"Will update {len(changes)} of {len(paths)} JobNote(s):\n") + for path, _ in changes: + print(f" {path.name}") + + if not args.apply: + print(f"\nDry-run. Pass --apply to rewrite {len(changes)} files.") + return 0 + + for path, new_text in changes: + path.write_text(new_text, encoding="utf-8") + print(f"\nMigrated {len(changes)} JobNote(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate_role_family.py b/scripts/migrate_role_family.py new file mode 100644 index 0000000..6840b1a --- /dev/null +++ b/scripts/migrate_role_family.py @@ -0,0 +1,89 @@ +"""One-time migration: re-classify existing JobNote role_family fields. + +`role_family` is stored once at intake and never reconciled. When the +intake_filter OUT keyword list expands (e.g. commit 3828d8f added +"engineering manager", "solutions engineer", "security engineer", +"operations specialist"), existing JobNotes keep their old role_family +and continue to feed the gap plan. This script walks every JobNote +and rewrites role_family to whatever current `keyword_classify` decides. + +Behavior: +- title resolves to out-of-scope → set role_family="out-of-scope" +- title resolves to a specific family → set role_family=that family +- title is ambiguous (None — LLM stage decides) → leave role_family alone + +Dry-run by default; --apply to commit. + +Usage: + uv run python -m scripts.migrate_role_family # dry-run + uv run python -m scripts.migrate_role_family --apply # commit changes +""" + +from __future__ import annotations + +import argparse +import sys +from typing import TYPE_CHECKING + +import frontmatter + +from compass.config import VAULT_PATH +from compass.pipeline.role_family import keyword_classify + +if TYPE_CHECKING: + from pathlib import Path + + +def find_changes() -> list[tuple[Path, str, str]]: + """Return [(file, old, new), ...] for JobNotes whose title now classifies as + out-of-scope. + + Only out-of-scope transitions are migrated. In-scope re-classifications + (e.g. keyword says swe-fullstack but body-signal upgrader originally set + agent-engineer) are left alone — the original write had access to the JD + body that this script doesn't re-evaluate. + """ + changes: list[tuple[Path, str, str]] = [] + for path in sorted((VAULT_PATH / "jobs").glob("*.md")): + post = frontmatter.load(path) + title = post.metadata.get("title", "") + old = post.metadata.get("role_family", "") + in_scope, new = keyword_classify(title) + if in_scope is False and old != "out-of-scope": + changes.append((path, old, new)) + return changes + + +def apply_changes(changes: list[tuple[Path, str, str]]) -> None: + for path, _, new in changes: + post = frontmatter.load(path) + post.metadata["role_family"] = new + path.write_text(frontmatter.dumps(post), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Actually rewrite the JobNotes") + args = parser.parse_args() + + changes = find_changes() + if not changes: + print("No JobNotes need role_family migration.") + return 0 + + print(f"Found {len(changes)} JobNote(s) whose role_family disagrees with current classifier:\n") + for path, old, new in changes: + print(f" {path.name}") + print(f" {old!r} -> {new!r}") + + if not args.apply: + print(f"\nDry-run. Pass --apply to rewrite {len(changes)} files.") + return 0 + + apply_changes(changes) + print(f"\nMigrated {len(changes)} JobNote(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/seed_companies_from_yaml.py b/scripts/seed_companies_from_yaml.py new file mode 100644 index 0000000..2d57b19 --- /dev/null +++ b/scripts/seed_companies_from_yaml.py @@ -0,0 +1,104 @@ +"""One-shot seed: create a CompanyNote in compass-vault/companies/ for every +company in _profile/target-companies.yaml, pre-populated with tier, geo, +why_interesting, interview_difficulty, cisco_adjacency from the YAML. + +Why this exists: write_company_note auto-creates default-shaped notes on +first sighting (tier=unknown, geo=[], etc.). Without this seed step, the +dashboard would be empty for 41 companies until the human filled them in by +hand. With it, the dashboard works the moment the pipeline produces its +first JobNote. + +Preserves existing CompanyNote human edits — same merge logic as +write_company_note (preserve incoming-default vs. existing-non-default). + +Dry-run by default; --apply to commit. + +Usage: + uv run python -m scripts.seed_companies_from_yaml # dry-run + uv run python -m scripts.seed_companies_from_yaml --apply # commit +""" + +from __future__ import annotations + +import argparse +import sys + +from compass.vault.schemas import CompanyNote +from compass.vault.target_companies import list_yaml_companies, refresh_yaml +from compass.vault.writer import write_company_note + + +def _to_company_note(entry: dict) -> CompanyNote: + """Project a YAML entry onto the CompanyNote schema. Invalid Literal values + collapse to safe defaults — write_company_note + Pydantic will validate.""" + tier = entry.get("tier") or "unknown" + if tier not in { + "apply-now", + "opportunistic", + "backend-prep", + "6-month", + "stretch", + "skip", + "unknown", + }: + tier = "unknown" + + difficulty = str(entry.get("interview_difficulty") or "unknown").strip().lower() + if difficulty not in { + "hackerrank", + "case", + "lc-easy", + "lc-medium", + "lc-medium-hard", + "lc-hard", + "takehome", + "unknown", + }: + difficulty = "unknown" + + return CompanyNote( + company=entry["company"], + tier=tier, # type: ignore[arg-type] + geo=list(entry.get("geos") or []), + why_interesting=str(entry.get("notes") or ""), + interview_difficulty=difficulty, # type: ignore[arg-type] + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Actually write the CompanyNotes") + args = parser.parse_args() + + refresh_yaml() + entries = list_yaml_companies() + if not entries: + print("No companies found in _profile/target-companies.yaml.") + return 1 + + # Defense against an empty-string `company:` field in YAML (would otherwise + # write `companies/.md` as a dotfile carrying no information). Drop with + # a warning rather than persist garbage. + valid_entries = [] + for e in entries: + if not (e.get("company") or "").strip(): + print(f" ! skipping YAML entry with empty company name: {e!r}") + continue + valid_entries.append(e) + notes = [_to_company_note(e) for e in valid_entries] + print(f"Will seed {len(notes)} CompanyNote(s):\n") + for n in notes: + print(f" {n.company:25s} tier={n.tier:14s} diff={n.interview_difficulty}") + + if not args.apply: + print(f"\nDry-run. Pass --apply to write {len(notes)} files.") + return 0 + + for n in notes: + write_company_note(n) + print(f"\nSeeded {len(notes)} CompanyNote(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/seed_vault.py b/scripts/seed_vault.py index cd37467..b0e9cfc 100644 --- a/scripts/seed_vault.py +++ b/scripts/seed_vault.py @@ -6,6 +6,7 @@ Creates all required folders and placeholder files. Does NOT overwrite existing files. """ + import sys from pathlib import Path diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py index 1068792..87c71a6 100644 --- a/scripts/test_scrape.py +++ b/scripts/test_scrape.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/test_scrape.py """ + from __future__ import annotations import asyncio diff --git a/scripts/test_vault_roundtrip.py b/scripts/test_vault_roundtrip.py index 977f378..165882d 100644 --- a/scripts/test_vault_roundtrip.py +++ b/scripts/test_vault_roundtrip.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/test_vault_roundtrip.py """ + from __future__ import annotations import sys @@ -48,7 +49,9 @@ def main() -> int: job_path = write_job_note(note) print(f" OK wrote job note: {job_path.name}") - company_path = write_company_note(CompanyNote(company=SENTINEL_COMPANY, tier="unknown", roles_seen=1)) + company_path = write_company_note( + CompanyNote(company=SENTINEL_COMPANY, tier="unknown", roles_seen=1) + ) print(f" OK wrote company note: {company_path.name}") skill_path = update_skill_note("Python", SENTINEL_URL) diff --git a/tests/analysis/__init__.py b/tests/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/analysis/test_gap_aggregator_role_filter.py b/tests/analysis/test_gap_aggregator_role_filter.py new file mode 100644 index 0000000..ac68642 --- /dev/null +++ b/tests/analysis/test_gap_aggregator_role_filter.py @@ -0,0 +1,44 @@ +"""Regression test for B6 fix: out-of-scope JobNotes must not contribute to gap plan.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import frontmatter + +if TYPE_CHECKING: + from pathlib import Path + + +def _write_job(vault: Path, name: str, role_family: str, required: list[str], score: float) -> None: + (vault / "jobs").mkdir(parents=True, exist_ok=True) + post = frontmatter.Post( + "# Test job\n", + company="Test", + title=f"Test Title {name}", + url=f"https://x/{name}", + source="manual", + date_found="2026-05-19", + match_score=score, + score_reasoning="t", + seniority="mid", + role_family=role_family, + tier="apply-now", + skills_required=required, + skills_nice_to_have=[], + skills_matched=[], + skills_missing=required, + jd_summary="t", + ) + (vault / "jobs" / f"{name}.md").write_text(frontmatter.dumps(post), encoding="utf-8") + + +def test_gap_aggregator_skips_out_of_scope_jobs(temp_vault): + from compass.analysis import gap_aggregator + + _write_job(temp_vault, "inscope", "agent-engineer", ["Python"], score=4.0) + _write_job(temp_vault, "outofscope", "out-of-scope", ["TypeScript"], score=4.0) + + jobs = gap_aggregator.load_jobs() + titles = sorted(j.title for j in jobs) + assert titles == ["Test Title inscope"] # out-of-scope job excluded diff --git a/tests/analysis/test_skill_backlinks.py b/tests/analysis/test_skill_backlinks.py new file mode 100644 index 0000000..afacb81 --- /dev/null +++ b/tests/analysis/test_skill_backlinks.py @@ -0,0 +1,146 @@ +"""P2 Obsidian leverage: gap_aggregator writes a `## Jobs requiring this skill` +block into each SkillNote body so Linked Mentions + Dataview can surface the +reverse mapping (every JD asking for Python, sorted by match_score). + +The block preserves existing body content above it (seed notes carry category +descriptions like `_Category: mcp · Tier-2 demand: highest_` that must not be +clobbered) and replaces itself on re-runs (idempotent). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import frontmatter + +if TYPE_CHECKING: + from pathlib import Path + + +def _seed_skill(vault: Path, canonical: str, category: str, body_extra: str = "") -> None: + (vault / "skills").mkdir(parents=True, exist_ok=True) + safe = canonical.replace(" ", "_").replace("/", "_") + text = ( + "---\n" + "type: skill\n" + f"skill: {canonical}\n" + f"category: {category}\n" + "appears_in_jobs: 0\n" + "my_level: 0\n" + "---\n" + f"# {canonical}\n" + f"{body_extra}" + ) + (vault / "skills" / f"{safe}.md").write_text(text, encoding="utf-8") + + +def _seed_job( + vault: Path, + name: str, + *, + company: str, + title: str, + required: list[str], + score: float, + tier: str = "apply-now", +) -> None: + (vault / "jobs").mkdir(parents=True, exist_ok=True) + post = frontmatter.Post( + "# job\n", + company=company, + title=title, + url=f"https://x/{name}", + source="manual", + date_found="2026-05-19", + match_score=score, + score_reasoning="t", + role_family="agent-engineer", + tier=tier, + skills_required=required, + skills_nice_to_have=[], + skills_matched=[], + skills_missing=required, + jd_summary="t", + ) + (vault / "jobs" / f"{name}.md").write_text(frontmatter.dumps(post), encoding="utf-8") + + +def test_skill_backlinks_block_added_to_skillnote(temp_vault): + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language", body_extra="_Category: language_\n") + _seed_job( + temp_vault, + "sierra", + company="Sierra", + title="Agent Eng", + required=["Python"], + score=4.0, + ) + _seed_job( + temp_vault, + "decagon", + company="Decagon", + title="MTS", + required=["Python"], + score=3.0, + ) + + gap_aggregator.regenerate(write=True) + + body = (temp_vault / "skills" / "Python.md").read_text() + assert "## Jobs requiring this skill" in body + # Existing description preserved + assert "_Category: language_" in body + # Both jobs linked, with company — title display + assert "[[sierra|Sierra — Agent Eng]]" in body + assert "[[decagon|Decagon — MTS]]" in body + # Sierra (score 4.0) appears before Decagon (score 3.0) + assert body.index("Sierra — Agent Eng") < body.index("Decagon — MTS") + # Score + tier rendered + assert "score 4.0" in body + assert "apply-now" in body + + +def test_skill_backlinks_idempotent_on_rerun(temp_vault): + """Two regenerate calls must not duplicate the block.""" + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language") + _seed_job( + temp_vault, + "sierra", + company="Sierra", + title="Agent Eng", + required=["Python"], + score=4.0, + ) + gap_aggregator.regenerate(write=True) + body1 = (temp_vault / "skills" / "Python.md").read_text() + gap_aggregator.regenerate(write=True) + body2 = (temp_vault / "skills" / "Python.md").read_text() + + assert body1 == body2 + assert body2.count("## Jobs requiring this skill") == 1 + + +def test_skill_backlinks_removed_when_no_jobs_reference_skill(temp_vault): + """If a skill's last referencing job is removed, the block goes away.""" + from compass.analysis import gap_aggregator + + _seed_skill(temp_vault, "Python", "language") + _seed_job( + temp_vault, + "sierra", + company="Sierra", + title="Agent Eng", + required=["Python"], + score=4.0, + ) + gap_aggregator.regenerate(write=True) + assert "## Jobs requiring this skill" in (temp_vault / "skills" / "Python.md").read_text() + + # Remove the only job + (temp_vault / "jobs" / "sierra.md").unlink() + gap_aggregator.regenerate(write=True) + assert "## Jobs requiring this skill" not in (temp_vault / "skills" / "Python.md").read_text() diff --git a/tests/conftest.py b/tests/conftest.py index 8f67a76..340727e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,29 @@ import pytest +@pytest.fixture(scope="session") +def embedding_model_cached(): + """Pre-load sentence-transformers once per test session. ~90MB on first run.""" + from sentence_transformers import SentenceTransformer + + SentenceTransformer("all-MiniLM-L6-v2") + + +@pytest.fixture +def temp_hitl_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point HITL_STATE_DB at a fresh per-test SQLite file. + + The store reads `compass.config.HITL_STATE_DB` inside function bodies (per + the module-level discipline rule), so monkeypatching the attribute is + sufficient — no module reimport needed. + """ + db = tmp_path / "pending.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_STATE_DB", db) + return db + + @pytest.fixture def temp_vault(tmp_path: Path, monkeypatch): """Create a minimal compass-vault structure in a tmp dir and point config at it.""" diff --git a/tests/evals/__init__.py b/tests/evals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/test_dataset.py b/tests/evals/test_dataset.py new file mode 100644 index 0000000..5ad2f6a --- /dev/null +++ b/tests/evals/test_dataset.py @@ -0,0 +1,57 @@ +"""Eval dataset round-trip + EvalRecord validation.""" + +from __future__ import annotations + +import pytest + +from compass.evals.dataset import EvalRecord, add_example, load_dataset, save_dataset + + +def test_evalrecord_validates_score_range(): + """Score must be 0.0-5.0 per the rubric. Out-of-range = ValidationError.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + EvalRecord(id="x", jd_text="t", expected_score=5.5) + with pytest.raises(ValidationError): + EvalRecord(id="x", jd_text="t", expected_score=-0.1) + + +def test_save_load_roundtrip(tmp_path): + p = tmp_path / "dataset.json" + records = [ + EvalRecord( + id="eval-001", + jd_text="Build agents.", + expected_score=4.0, + expected_skills=["Python", "MCP", "LangGraph"], + notes="Strong match", + ), + EvalRecord( + id="eval-002", + jd_text="Sr. engineer needed.", + expected_score=1.0, + expected_skills=["Python", "Kubernetes", "Go"], + ), + ] + save_dataset(records, p) + assert p.exists() + + loaded = load_dataset(p) + assert len(loaded) == 2 + assert loaded[0].id == "eval-001" + assert loaded[0].expected_skills == ["Python", "MCP", "LangGraph"] + assert loaded[1].expected_score == 1.0 + + +def test_load_missing_file_returns_empty(tmp_path): + """First-time setup: dataset doesn't exist yet → return [] not raise.""" + assert load_dataset(tmp_path / "nope.json") == [] + + +def test_add_example_auto_id_increments(tmp_path): + p = tmp_path / "dataset.json" + add_example("jd1", 3.0, ["Python"], path=p) + add_example("jd2", 4.0, ["MCP"], path=p) + recs = load_dataset(p) + assert [r.id for r in recs] == ["eval-001", "eval-002"] diff --git a/tests/evals/test_label_jd_cli.py b/tests/evals/test_label_jd_cli.py new file mode 100644 index 0000000..36d43b6 --- /dev/null +++ b/tests/evals/test_label_jd_cli.py @@ -0,0 +1,48 @@ +"""Smoke tests for scripts/label_jd.py — the spec-required interactive CLI. + +Heavy interactive coverage isn't worth it (prompts are I/O-bound and the +runtime is short). These tests pin the load + path-validation logic + the +non-interactive `--score`/`--skills` path. +""" + +from __future__ import annotations + +import pytest + + +def test_load_jobnote_path_traversal_rejected(temp_vault): + """`jobnote` argument is user input — escapes from vault/jobs/ must fail.""" + from scripts.label_jd import _load_jobnote + + with pytest.raises(SystemExit): + _load_jobnote("../_profile/resume.md") + + +def test_load_jobnote_missing_file(temp_vault): + from scripts.label_jd import _load_jobnote + + with pytest.raises(SystemExit): + _load_jobnote("nonexistent.md") + + +def test_load_jobnote_reads_full_jd_section(temp_vault): + """When the JobNote has a `## Full JD` section, that's what we label + against — not the summary or the wikilink table.""" + from scripts.label_jd import _load_jobnote + + note = temp_vault / "jobs" / "test.md" + note.write_text( + "---\ncompany: Sierra\ntitle: Engineer\nurl: https://x\nsource: ashby\n" + "date_found: 2026-05-19\nmatch_score: 4.0\nscore_reasoning: t\n" + "role_family: agent-engineer\ntier: apply-now\n" + "skills_required: []\nskills_nice_to_have: []\n" + "skills_matched: []\nskills_missing: []\njd_summary: short\n---\n" + "# header\n\nsummary text\n\n## Skills\n\n**Required:** stuff\n\n" + "## Full JD\n\nThis is the real JD body about LangGraph and MCP.\n" + ) + jd_text, company, title, source = _load_jobnote("test.md") + assert "LangGraph and MCP" in jd_text + assert "summary text" not in jd_text # confirms we sliced past summary + assert company == "Sierra" + assert title == "Engineer" + assert "Sierra" in source diff --git a/tests/evals/test_metrics.py b/tests/evals/test_metrics.py new file mode 100644 index 0000000..75af2f2 --- /dev/null +++ b/tests/evals/test_metrics.py @@ -0,0 +1,126 @@ +"""Pure-function metric primitives — no I/O, no network.""" + +from __future__ import annotations + +import pytest + +from compass.evals.metrics import ( + aggregate, + score_bias, + score_mae, + score_rmse, + skill_precision, + skill_recall, +) + + +class TestScoreMAE: + def test_perfect_match(self): + assert score_mae([4.0, 3.0, 2.0], [4.0, 3.0, 2.0]) == 0.0 + + def test_simple_diff(self): + # |4.0-3.0| + |3.0-3.0| + |2.0-3.0| = 2.0, divided by 3 → 0.667 + assert round(score_mae([4.0, 3.0, 2.0], [3.0, 3.0, 3.0]), 3) == 0.667 + + def test_empty_returns_zero(self): + assert score_mae([], []) == 0.0 + + +class TestScoreRMSE: + def test_penalizes_big_misses_more_than_mae(self): + """One large miss + several perfect matches: RMSE > MAE.""" + # MAE: |4.0-0.0| / 4 = 1.0; RMSE: sqrt(16/4) = 2.0 + mae = score_mae([4.0, 3.0, 3.0, 3.0], [0.0, 3.0, 3.0, 3.0]) + rmse = score_rmse([4.0, 3.0, 3.0, 3.0], [0.0, 3.0, 3.0, 3.0]) + assert mae == 1.0 + assert rmse == 2.0 + + +class TestScoreBias: + def test_positive_means_over_scoring(self): + # Compass scored 4.5 / 4.5; humans said 3.0 / 3.0 → Compass over by 1.5 + assert score_bias([4.5, 4.5], [3.0, 3.0]) == 1.5 + + def test_negative_means_under_scoring(self): + assert score_bias([2.0, 2.0], [3.5, 3.5]) == -1.5 + + def test_zero_when_unbiased(self): + # Symmetric over/under cancels out + assert score_bias([4.0, 2.0], [3.0, 3.0]) == 0.0 + + +class TestSkillRecall: + def test_all_found(self): + assert skill_recall(["Python", "MCP", "LangGraph"], ["Python", "MCP"]) == 1.0 + + def test_half_missed(self): + assert skill_recall(["Python"], ["Python", "MCP"]) == 0.5 + + def test_case_insensitive(self): + assert skill_recall(["python", "MCP"], ["Python", "mcp"]) == 1.0 + + def test_empty_expected_is_vacuous_truth(self): + """If the JD lists no skills, we trivially found them all.""" + assert skill_recall([], []) == 1.0 + assert skill_recall(["Python"], []) == 1.0 + + def test_empty_predicted_with_nonempty_expected(self): + assert skill_recall([], ["Python", "MCP"]) == 0.0 + + +class TestSkillPrecision: + def test_no_false_positives(self): + assert skill_precision(["Python", "MCP"], ["Python", "MCP", "LangGraph"]) == 1.0 + + def test_extra_skills_drop_precision(self): + # Predicted 4, only 2 are in expected → 50% precision + assert ( + skill_precision(["Python", "MCP", "Java", "Ruby"], ["Python", "MCP", "LangGraph"]) + == 0.5 + ) + + def test_empty_predicted_is_vacuous_truth(self): + """If we predicted no skills, we have zero false positives.""" + assert skill_precision([], ["Python"]) == 1.0 + + +class TestAggregate: + def test_full_aggregate(self): + metrics = aggregate( + predicted_scores=[4.0, 3.0], + expected_scores=[4.0, 4.0], + predicted_skill_lists=[["Python", "MCP"], ["Python"]], + expected_skill_lists=[["Python", "MCP"], ["Python", "LangGraph"]], + matched_skill_lists=[["Python", "MCP"], ["Python"]], + ) + assert metrics.n == 2 + assert metrics.score_mae == 0.5 # (0 + 1) / 2 + assert metrics.score_bias == -0.5 # both under by 1 / averaged + # Record 1: 2/2 recall, 2/2 precision. Record 2: 1/2 recall, 1/1 precision. + # Average recall: (1.0 + 0.5) / 2 = 0.75 + assert metrics.extract_skill_recall == 0.75 + assert metrics.extract_skill_precision == 1.0 + + def test_n_zero(self): + m = aggregate([], [], [], []) + assert m.n == 0 + assert m.score_mae == 0.0 + assert m.score_rmse == 0.0 + + def test_match_lists_default_to_predicted(self): + """When matched_skill_lists is None, match_skill_recall === extract_skill_recall.""" + m = aggregate( + predicted_scores=[4.0], + expected_scores=[4.0], + predicted_skill_lists=[["Python"]], + expected_skill_lists=[["Python"]], + matched_skill_lists=None, + ) + assert m.match_skill_recall == 1.0 + + +def test_zip_strict_catches_length_mismatch(): + """Defensive: passing mismatched lists should raise rather than silently + truncate — strict=True in the impl makes this an immediate ValueError.""" + with pytest.raises(ValueError): + score_mae([4.0, 3.0], [4.0]) diff --git a/tests/evals/test_runner.py b/tests/evals/test_runner.py new file mode 100644 index 0000000..5aa6416 --- /dev/null +++ b/tests/evals/test_runner.py @@ -0,0 +1,228 @@ +"""Runner integration test — mocked LLM calls, real metric aggregation.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from compass.evals.dataset import EvalRecord +from compass.evals.runner import run_against_judge, run_against_labels +from compass.pipeline.state import JobRequirements, JobScore + + +async def _fake_extract(jd_text: str) -> JobRequirements: + """Deterministic stub — pretends extract found Python + MCP from every JD.""" + return JobRequirements( + required_skills=["Python", "MCP"], + nice_to_have_skills=["LangGraph"], + years_experience=2, + seniority="mid", + remote_policy="hybrid", + summary="Build agents.", + ) + + +async def _fake_score(req, profile_text, job=None) -> JobScore: + """Deterministic stub — 4.0 with matched=Python+MCP, missed=LangGraph.""" + return JobScore( + score=4.0, + reasoning="Strong MCP + Python match. Score reasoning ends with period.", + matched_skills=["Python", "MCP"], + missing_skills=["LangGraph"], + tailoring_notes="lead with Cisco MCP work", + ) + + +@pytest.mark.asyncio +async def test_run_against_labels_computes_aggregate(temp_vault): + """JD bodies must mention the skills the stub returns — production + `_normalize_skill_list` drops skills the LLM emits but the JD doesn't + actually contain (anti-hallucination guard). Keep tests aligned with + production behavior.""" + records = [ + EvalRecord( + id="eval-001", + jd_text="Build LangGraph agents in Python with MCP servers.", + expected_score=4.0, + expected_skills=["Python", "MCP", "LangGraph"], + ), + EvalRecord( + id="eval-002", + jd_text="Senior systems engineer working with Python MCP and LangGraph.", + expected_score=1.0, + expected_skills=["Kubernetes", "Go"], + ), + ] + with ( + patch("compass.evals.runner._extract", new=_fake_extract), + patch("compass.evals.runner._score", new=_fake_score), + ): + metrics, per_record = await run_against_labels(records) + + assert metrics.n == 2 + # Score MAE: |4.0-4.0| + |4.0-1.0| = 3.0, /2 → 1.5 + assert metrics.score_mae == 1.5 + # Bias: ((4-4)+(4-1))/2 = 1.5 — Compass over-scores eval-002 (1.0 → 4.0) + assert metrics.score_bias == 1.5 + # Both JDs mention Python+MCP+LangGraph (post-normalization). + # Record 1: extracted=[Python, MCP, LangGraph], expected=[Python, MCP, LangGraph] → recall 1.0 + # Record 2: extracted=[Python, MCP, LangGraph], expected=[Kubernetes, Go] → recall 0.0 + assert metrics.extract_skill_recall == 0.5 + assert per_record[0]["missed_skills"] == [] + assert "kubernetes" in per_record[1]["missed_skills"] + assert "go" in per_record[1]["missed_skills"] + + +@pytest.mark.asyncio +async def test_run_against_judge_uses_judge_verdict(temp_vault): + """Judge mode replaces expected_score/expected_skills with LLM judge output. + EvalRecord's own expected_* fields are ignored.""" + from compass.evals.judge import JudgeVerdict + + async def fake_judge(jd_text, profile, predicted_skills, predicted_score): + return JudgeVerdict( + expected_skills=["Python", "MCP"], + expected_score=4.0, + reasoning="Agent did fine.", + ) + + records = [ + EvalRecord( + id="eval-001", + jd_text="Build agents.", + expected_score=2.5, # ignored in judge mode + expected_skills=["DontUseThis"], # ignored in judge mode + ), + ] + with ( + patch("compass.evals.runner._extract", new=_fake_extract), + patch("compass.evals.runner._score", new=_fake_score), + patch("compass.evals.judge.judge_jd", new=fake_judge), + ): + metrics, per_record = await run_against_judge(records) + + assert metrics.n == 1 + # Compass scored 4.0, judge said 4.0 → MAE 0.0 + assert metrics.score_mae == 0.0 + assert "judge_reasoning" in per_record[0] + assert per_record[0]["judge_reasoning"] == "Agent did fine." + + +@pytest.mark.asyncio +async def test_runner_applies_extract_normalization(temp_vault): + """Regression: runner used to call `_extract` raw, skipping + `_normalize_skill_list`. That made the recall metric measure something + DIFFERENT from what the production pipeline persists. Now the runner + runs the same normalization extract_node does — skills the LLM emits + that don't appear in the JD body are dropped (anti-hallucination).""" + + async def hallucinating_extract(jd_text): + # Stub emits skills the JD doesn't actually contain — production + # drops these as likely-hallucinated. + return JobRequirements( + required_skills=["Python", "Kubernetes", "Federated Learning"], + nice_to_have_skills=[], + years_experience=2, + seniority="mid", + remote_policy="hybrid", + summary="x", + ) + + records = [ + EvalRecord( + id="eval-001", + jd_text="Build a Python service that calls our API.", + expected_score=3.0, + expected_skills=["Python"], + ), + ] + with ( + patch("compass.evals.runner._extract", new=hallucinating_extract), + patch("compass.evals.runner._score", new=_fake_score), + ): + metrics, per_record = await run_against_labels(records) + + # The stub returned ["Python", "Kubernetes", "Federated Learning"] but only + # "Python" appears in the JD. Production drops the other two as hallucinations. + extracted_n = per_record[0]["extracted_skills_n"] + assert extracted_n == 1, f"expected 1 extracted skill after normalization, got {extracted_n}" + assert "kubernetes" not in per_record[0].get("extra_skills", []) + assert metrics.extract_skill_recall == 1.0 # "Python" found + + +@pytest.mark.asyncio +async def test_runner_applies_score_constraint(temp_vault): + """Regression: runner used to skip `_constrain_to_jd_skills`, letting + score_node hallucinations through to the metrics. Now matched/missing + are filtered to the JD's actual skill universe before comparison.""" + + async def hallucinating_score(req, profile_text, job=None): + return JobScore( + score=4.0, + reasoning="Real reasoning ending with terminal punctuation.", + # JD requested only Python — but the score LLM claims candidate + # also has matches in skills the JD didn't list. + matched_skills=["Python", "Rust", "Erlang"], + missing_skills=["Haskell"], + tailoring_notes="", + ) + + async def fake_extract_python_only(jd_text): + return JobRequirements( + required_skills=["Python"], + nice_to_have_skills=[], + years_experience=2, + seniority="mid", + remote_policy="hybrid", + summary="x", + ) + + records = [ + EvalRecord( + id="eval-001", + jd_text="Build a Python service.", + expected_score=4.0, + expected_skills=["Python"], + ), + ] + with ( + patch("compass.evals.runner._extract", new=fake_extract_python_only), + patch("compass.evals.runner._score", new=hallucinating_score), + ): + metrics, _per = await run_against_labels(records) + + # match_skill_recall uses score_result.matched_skills — production + # constrains to the JD universe. So "Rust" and "Erlang" are filtered + # out; only "Python" remains; recall against expected=["Python"] is 1.0. + assert metrics.match_skill_recall == 1.0 + + +@pytest.mark.asyncio +async def test_run_against_labels_handles_extract_failure(temp_vault): + """If extract fails for one JD, that record is logged but the others + still aggregate — pipeline never blocks on one bad JD.""" + + async def flaky_extract(jd_text): + if "broken" in jd_text: + raise RuntimeError("simulated extract failure") + return await _fake_extract(jd_text) + + records = [ + EvalRecord(id="eval-001", jd_text="broken jd", expected_score=4.0, expected_skills=["X"]), + EvalRecord( + id="eval-002", + jd_text="good jd", + expected_score=4.0, + expected_skills=["Python", "MCP"], + ), + ] + with ( + patch("compass.evals.runner._extract", new=flaky_extract), + patch("compass.evals.runner._score", new=_fake_score), + ): + metrics, per_record = await run_against_labels(records) + + assert metrics.n == 1 # only the good one aggregated + assert per_record[0].get("error") == "extract or score failed" + assert per_record[1]["predicted_score"] == 4.0 diff --git a/tests/hitl/__init__.py b/tests/hitl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py new file mode 100644 index 0000000..8870b79 --- /dev/null +++ b/tests/hitl/conftest.py @@ -0,0 +1,22 @@ +"""Shared fixtures for HiTL tests. + +`temp_hitl_db` lives in the top-level tests/conftest.py so pipeline tests can +reuse it without duplication. +""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + + +@pytest.fixture +def frozen_now(monkeypatch: pytest.MonkeyPatch) -> _dt.datetime: + """Freeze the wall clock the state store uses.""" + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) + + import compass.hitl.state_store as ss + + monkeypatch.setattr(ss, "_now", lambda: fixed) + return fixed diff --git a/tests/hitl/test_claim_pending.py b/tests/hitl/test_claim_pending.py new file mode 100644 index 0000000..8d2f891 --- /dev/null +++ b/tests/hitl/test_claim_pending.py @@ -0,0 +1,99 @@ +"""Atomic claim regression tests — confirms the Modal-cron + MCP-approve race +fix from the 2026-05-19 adversarial review (wave 2).""" + +from __future__ import annotations + +import pytest + + +@pytest.mark.asyncio +async def test_claim_pending_succeeds_once(temp_hitl_db): + from compass.hitl import state_store + + await state_store.add_pending( + thread_id="race-1", + job_url="https://x/1", + company="X", + title="Y", + score=4.0, + score_reasoning="t", + matched_skills=[], + missing_skills=[], + ) + assert await state_store.claim_pending("race-1") is True + + +@pytest.mark.asyncio +async def test_claim_pending_second_call_returns_false(temp_hitl_db): + """Modal cron + MCP approve race: only one consumer claims.""" + from compass.hitl import state_store + + await state_store.add_pending( + thread_id="race-2", + job_url="https://x/2", + company="X", + title="Y", + score=4.0, + score_reasoning="t", + matched_skills=[], + missing_skills=[], + ) + first = await state_store.claim_pending("race-2") + second = await state_store.claim_pending("race-2") + assert first is True + assert second is False, "second claim must lose the race" + + +@pytest.mark.asyncio +async def test_claim_pending_missing_thread_returns_false(temp_hitl_db): + from compass.hitl import state_store + + assert await state_store.claim_pending("nonexistent") is False + + +@pytest.mark.asyncio +async def test_mark_resolved_works_after_claim(temp_hitl_db): + """The claim → finalize flow: claim_pending transitions to 'resuming', + then mark_resolved completes the transition to the final status.""" + from compass.hitl import state_store + + await state_store.add_pending( + thread_id="finalize-1", + job_url="https://x/3", + company="X", + title="Y", + score=4.0, + score_reasoning="t", + matched_skills=[], + missing_skills=[], + ) + assert await state_store.claim_pending("finalize-1") is True + await state_store.mark_resolved("finalize-1", status="approved") + row = await state_store.get_pending("finalize-1") + # After resolution the row stays but with new status + assert row is None or row.get("status") == "approved" + + +@pytest.mark.asyncio +async def test_mark_resolved_blocked_after_claim_by_another_caller(temp_hitl_db): + """Once a row is in 'resuming' state, a SECOND mark_resolved still works + (the first claim → finalize completes). But a separate attempt that + didn't claim first should ALSO succeed if the row is still in 'resuming' — + this matches the legacy single-writer behavior.""" + from compass.hitl import state_store + + await state_store.add_pending( + thread_id="dual-1", + job_url="https://x/4", + company="X", + title="Y", + score=4.0, + score_reasoning="t", + matched_skills=[], + missing_skills=[], + ) + await state_store.claim_pending("dual-1") + await state_store.mark_resolved("dual-1", status="approved") + # Subsequent mark_resolved on an already-resolved row raises + with pytest.raises(ValueError): + await state_store.mark_resolved("dual-1", status="rejected") diff --git a/tests/hitl/test_resume.py b/tests/hitl/test_resume.py new file mode 100644 index 0000000..0f05e5c --- /dev/null +++ b/tests/hitl/test_resume.py @@ -0,0 +1,261 @@ +"""resume_pending re-opens the checkpointer, recompiles the graph, and drives +the resume via Command(resume=...). Verifies vault_write fires on approve +and tailor is skipped on reject.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="agent role", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ) + } + + async def fake_tailor(_): + return {"tailored_paragraph": "lead with MCP."} + + written = {"calls": 0, "with_tailor": 0} + + async def fake_vault_write(state): + written["calls"] += 1 + if state.get("tailored_paragraph"): + written["with_tailor"] += 1 + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + return written + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_approve_runs_tailor_then_vault_write(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + + pending = await state_store.list_pending() + tid = pending[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": True, "feedback": "LGTM"}) + assert final["vault_written"] is True + assert final["human_approved"] is True + assert stub_llm_nodes["with_tailor"] == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_reject_skips_tailor_writes_to_vault(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/2", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + final = await resume_pending(tid, decision={"approved": False}) + assert final["human_approved"] is False + assert stub_llm_nodes["with_tailor"] == 0 + assert stub_llm_nodes["calls"] == 1 # vault_write still fired (rejected branch) + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_unknown_thread_raises(): + from compass.hitl.resume import resume_pending + + with pytest.raises(LookupError, match="thread"): + await resume_pending("nope-1234", decision={"approved": True}) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_status_derives_from_final_state_not_input(stub_llm_nodes, monkeypatch): + """If the graph auto-rejects on resume (e.g. threshold edited mid-flight), + state_store must record 'rejected', not the input decision's 'approved'.""" + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/divergence-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the resume to auto-reject regardless of the input decision — + # simulates hitl_node short-circuiting on resume (e.g. SCORE_THRESHOLD + # edited between pause and resume). + async def auto_reject(state): + return {"human_approved": False} + + monkeypatch.setattr("compass.pipeline.graph.hitl_node", auto_reject) + + await resume_pending(tid, decision={"approved": True, "feedback": "human said yes"}) + + row = await state_store.get_pending(tid) + assert row["status"] == "rejected" # NOT "approved" — graph really auto-rejected + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_regenerates_counters(stub_llm_nodes, monkeypatch): + """A resumed thread writes a JobNote — gap_aggregator.regenerate() MUST run + so derived counters (CompanyNote.roles_seen, SkillNote.appears_in_jobs) + reflect the new JobNote. Phase 0 #12 / Phase 1.A #1 drift family.""" + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + regen_called = {"count": 0} + + def fake_regenerate(write: bool = False): + regen_called["count"] += 1 + return ([], {}) + + monkeypatch.setattr("compass.analysis.gap_aggregator.regenerate", fake_regenerate) + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/regen-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Baseline: run_pipeline did NOT regenerate (jobs_written==0 because the + # job paused before vault_write). Reset counter to isolate the resume call. + regen_called["count"] = 0 + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + assert regen_called["count"] == 1, "resume_pending must call gap_aggregator.regenerate" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_already_resolved_raises(stub_llm_nodes): + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/3", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + await resume_pending(tid, decision={"approved": True}) + + # Second resume call should not crash and should not re-fire vault_write + before = stub_llm_nodes["calls"] + with pytest.raises(ValueError, match="already resolved"): + await resume_pending(tid, decision={"approved": True}) + assert stub_llm_nodes["calls"] == before + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "temp_vault") +async def test_resume_purges_thread_checkpoint_blobs(stub_llm_nodes): + """After a thread resolves, its checkpoint rows in HITL_CHECKPOINT_DB must + be deleted — otherwise the DB grows unboundedly. Phase 1.B.1 I-2.""" + import aiosqlite + + from compass.config import HITL_CHECKPOINT_DB + from compass.hitl.resume import resume_pending + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="u://purge-probe", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 19), + ) + pre = await run_pipeline(raw_jobs=[job]) + assert pre["jobs_paused"] == 1 + tid = (await state_store.list_pending())[0]["thread_id"] + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (pre_count,) = await cur.fetchone() + assert pre_count > 0, f"setup: no checkpoint rows for {tid}" + + await resume_pending(tid, decision={"approved": True, "feedback": "test"}) + + async with aiosqlite.connect(HITL_CHECKPOINT_DB) as conn: + async with conn.execute( + "SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?", (tid,) + ) as cur: + (post_count,) = await cur.fetchone() + assert post_count == 0, f"checkpoint rows for resolved {tid} were not purged" diff --git a/tests/hitl/test_state_store.py b/tests/hitl/test_state_store.py new file mode 100644 index 0000000..355db43 --- /dev/null +++ b/tests/hitl/test_state_store.py @@ -0,0 +1,122 @@ +"""state_store CRUD: add/get/list/resolve + status transitions + serialisation.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.hitl import state_store + +pytestmark = pytest.mark.usefixtures("temp_hitl_db") + + +async def _add_one(thread_id: str = "tid-1", **overrides) -> None: + defaults = dict( + thread_id=thread_id, + job_url="https://jobs.example.com/abc", + company="Sierra", + title="Software Engineer, Agent", + score=4.2, + score_reasoning="Strong match on MCP + LangGraph.", + matched_skills=["MCP", "Python"], + missing_skills=["LangGraph"], + ) + defaults.update(overrides) + await state_store.add_pending(**defaults) + + +async def test_add_and_get_round_trips(frozen_now): + await _add_one() + row = await state_store.get_pending("tid-1") + assert row is not None + assert row["thread_id"] == "tid-1" + assert row["company"] == "Sierra" + assert row["score"] == pytest.approx(4.2) + assert row["matched_skills"] == ["MCP", "Python"] + assert row["missing_skills"] == ["LangGraph"] + assert row["status"] == "pending" + assert row["created_at"] == frozen_now.isoformat() + assert row["resolved_at"] is None + assert row["feedback"] is None + + +async def test_add_pending_is_idempotent_on_thread_id(): + """Re-running a pipeline that re-pauses the same thread_id is a no-op, not a crash.""" + await _add_one() + # Second call with same thread_id but different score should be a no-op + # (we don't overwrite — the resume must use the original checkpoint). + await _add_one(score=2.0) + row = await state_store.get_pending("tid-1") + assert row["score"] == pytest.approx(4.2) + + +async def test_list_pending_only_returns_pending_status(): + await _add_one(thread_id="tid-pending") + await _add_one(thread_id="tid-approved") + await state_store.mark_resolved("tid-approved", status="approved") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-pending"] + + +async def test_list_pending_orders_oldest_first(monkeypatch): + """The MCP UI shows the queue oldest-first so things don't get lost.""" + import compass.hitl.state_store as ss + + fixed = _dt.datetime(2026, 5, 19, 12, 0, 0, tzinfo=_dt.UTC) + monkeypatch.setattr(ss, "_now", lambda: fixed) + await _add_one(thread_id="tid-old") + monkeypatch.setattr(ss, "_now", lambda: fixed + _dt.timedelta(minutes=5)) + await _add_one(thread_id="tid-new") + rows = await state_store.list_pending() + assert [r["thread_id"] for r in rows] == ["tid-old", "tid-new"] + + +async def test_mark_resolved_records_status_and_timestamp(frozen_now): + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="LGTM") + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "LGTM" + assert row["resolved_at"] == frozen_now.isoformat() + + +async def test_mark_resolved_rejects_unknown_status(): + await _add_one() + with pytest.raises(ValueError, match="status"): + await state_store.mark_resolved("tid-1", status="bogus") + + +async def test_mark_resolved_unknown_thread_raises(): + with pytest.raises(LookupError): + await state_store.mark_resolved("tid-missing", status="approved") + + +async def test_list_timed_out_returns_only_old_pending(frozen_now, monkeypatch): + import compass.hitl.state_store as ss + + old = frozen_now - _dt.timedelta(hours=5) + young = frozen_now - _dt.timedelta(hours=1) + monkeypatch.setattr(ss, "_now", lambda: old) + await _add_one(thread_id="tid-old") + monkeypatch.setattr(ss, "_now", lambda: young) + await _add_one(thread_id="tid-young") + monkeypatch.setattr(ss, "_now", lambda: frozen_now) + rows = await state_store.list_timed_out(timeout_hours=4) + assert [r["thread_id"] for r in rows] == ["tid-old"] + + +async def test_get_pending_unknown_returns_none(): + assert await state_store.get_pending("nope") is None + + +async def test_mark_resolved_rejects_already_resolved_row(): + """Concurrent resolvers race: second one must see ValueError, not silent overwrite.""" + await _add_one() + await state_store.mark_resolved("tid-1", status="approved", feedback="first") + with pytest.raises(ValueError, match="already resolved"): + await state_store.mark_resolved("tid-1", status="timed_out") + # Confirm the first resolution stuck — no silent overwrite + row = await state_store.get_pending("tid-1") + assert row["status"] == "approved" + assert row["feedback"] == "first" diff --git a/tests/hitl/test_timeout_checker.py b/tests/hitl/test_timeout_checker.py new file mode 100644 index 0000000..df833b4 --- /dev/null +++ b/tests/hitl/test_timeout_checker.py @@ -0,0 +1,191 @@ +"""timeout_checker resumes pending rows older than HITL_TIMEOUT_HOURS with +{"approved": False}, marks them as 'timed_out' in the state store, appends +a row to _meta/agent-log.md (spec § DoD line 230), and leaves young +pending rows untouched.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from compass.hitl import state_store +from compass.pipeline.state import JobRequirements, JobScore, RawJob + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="x", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ) + } + + async def fake_vault_write(_): + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + async def fake_tailor(_): + return {"tailored_paragraph": "..."} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_resumes_old_pending_as_rejected(monkeypatch): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + tid = (await state_store.list_pending())[0]["thread_id"] + + # Force the row's created_at to look 5h old + import aiosqlite + + from compass.config import HITL_STATE_DB + + five_h_ago = (_dt.datetime.now(_dt.UTC) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute( + "UPDATE pending_approvals SET created_at = ? WHERE thread_id = ?", + (five_h_ago, tid), + ) + await conn.commit() + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 + row = await state_store.get_pending(tid) + assert row["status"] == "timed_out" + # Spec DoD line 230 — timeout must be logged to _meta/agent-log.md + from compass.config import AGENT_LOG_PATH + + log_text = AGENT_LOG_PATH.read_text(encoding="utf-8") + assert "hitl-timeout" in log_text + assert tid in log_text + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_leaves_young_pending_alone(): + from compass.hitl import timeout_checker + from compass.pipeline.graph import run_pipeline + + job = RawJob( + company="Sierra", + title="SWE", + url="https://x/2", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + await run_pipeline(raw_jobs=[job]) + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 0 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["status"] == "pending" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_timeout_checker_continues_after_one_resume_failure(monkeypatch): + """A single broken thread (e.g. checkpoint corrupted) must not block the + rest of the queue from timing out.""" + from compass.hitl import timeout_checker + + # Two rows, both stale + await state_store.add_pending( + thread_id="tid-broken", + job_url="x://1", + company="C", + title="T", + score=4.0, + score_reasoning="r", + matched_skills=[], + missing_skills=[], + ) + await state_store.add_pending( + thread_id="tid-ok", + job_url="x://2", + company="C", + title="T", + score=4.0, + score_reasoning="r", + matched_skills=[], + missing_skills=[], + ) + # Backdate both + import aiosqlite + + from compass.config import HITL_STATE_DB + + five_h_ago = (_dt.datetime.now(_dt.UTC) - _dt.timedelta(hours=5)).isoformat() + async with aiosqlite.connect(HITL_STATE_DB) as conn: + await conn.execute("UPDATE pending_approvals SET created_at = ?", (five_h_ago,)) + await conn.commit() + + # Make resume_pending raise for tid-broken + async def flaky(thread_id, **kw): + if thread_id == "tid-broken": + raise RuntimeError("checkpoint missing") + from compass.hitl.state_store import mark_resolved + + await mark_resolved(thread_id, status="timed_out") + + monkeypatch.setattr("compass.hitl.timeout_checker.resume_pending", flaky) + + n = await timeout_checker.check_and_resume_timeouts(timeout_hours=4) + assert n == 1 # one succeeded, one failed + row_broken = await state_store.get_pending("tid-broken") + assert row_broken["status"] == "error" + + +async def test_timeout_feedback_string_matches_vault_write_prefix(): + """Lock the contract: timeout_checker's feedback string MUST be matchable by + vault_write._derive_hitl_decision's startswith() check. Otherwise the + JobNote.hitl_decision='timed_out' mapping silently breaks.""" + from compass.hitl import HITL_TIMEOUT_FEEDBACK_PREFIX + + feedback = f"{HITL_TIMEOUT_FEEDBACK_PREFIX} 4h timeout" + assert feedback.lower().startswith(HITL_TIMEOUT_FEEDBACK_PREFIX.lower()) diff --git a/tests/mcp_server/__init__.py b/tests/mcp_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/mcp_server/test_pending_approvals.py b/tests/mcp_server/test_pending_approvals.py new file mode 100644 index 0000000..0c51efb --- /dev/null +++ b/tests/mcp_server/test_pending_approvals.py @@ -0,0 +1,75 @@ +"""MCP wrappers around state_store + resume_pending — JSON-serializable +return shapes (no datetime objects, no Pydantic instances).""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from compass.hitl import state_store + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_pending_approvals_tool_returns_jsonable_rows(): + from compass.mcp_server.server import pending_approvals + + await state_store.add_pending( + thread_id="tid-1", + job_url="https://x/1", + company="Sierra", + title="SWE Agent", + score=4.2, + score_reasoning="solid", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + ) + rows = await pending_approvals() + assert len(rows) == 1 + assert rows[0]["thread_id"] == "tid-1" + # Must be plain JSON-serialisable + json.dumps(rows) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_invokes_resume(monkeypatch): + from compass.mcp_server import server as mcp_server + + captured = {} + + async def fake_resume(thread_id, *, decision, status_override=None): + captured["thread_id"] = thread_id + captured["decision"] = decision + captured["status_override"] = status_override + return {"vault_written": True, "human_approved": decision["approved"]} + + monkeypatch.setattr(mcp_server, "_resume_pending", fake_resume) + + result = await mcp_server.approve(thread_id="tid-x", approved=True, feedback="LGTM") + assert result == {"vault_written": True, "human_approved": True, "human_feedback": None} + assert captured == { + "thread_id": "tid-x", + "decision": {"approved": True, "feedback": "LGTM"}, + "status_override": None, + } + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db") +async def test_approve_tool_handles_unknown_thread_as_error_dict(): + from compass.mcp_server.server import approve + + result = await approve(thread_id="missing", approved=False) + assert "error" in result and "missing" in result["error"] diff --git a/tests/pipeline/test_add_url.py b/tests/pipeline/test_add_url.py new file mode 100644 index 0000000..b72ca2b --- /dev/null +++ b/tests/pipeline/test_add_url.py @@ -0,0 +1,81 @@ +"""add_job_from_url unit tests — provider detection + generic fetch fallback.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest + +from compass.pipeline.add_url import _detect_provider, fetch_rawjob_from_url + + +def test_detect_provider_greenhouse(): + assert _detect_provider("https://boards.greenhouse.io/databricks/jobs/123") == "greenhouse" + assert _detect_provider("https://job-boards.greenhouse.io/anthropic/jobs/x") == "greenhouse" + + +def test_detect_provider_lever(): + assert _detect_provider("https://jobs.lever.co/company/abc") == "lever" + + +def test_detect_provider_ashby(): + assert _detect_provider("https://jobs.ashbyhq.com/sierra/agent-eng") == "ashby" + + +def test_detect_provider_workday(): + assert _detect_provider("https://citi.wd5.myworkdayjobs.com/2/job/123") == "workday" + + +def test_detect_provider_oracle_cloud_is_generic(): + """JPM and other Oracle Cloud careers pages don't have a structured API + we can probe — they fall to the generic static-fetch path.""" + assert _detect_provider("https://jpmc.fa.oraclecloud.com/hcmUI/...") == "generic" + + +def test_detect_provider_linkedin_is_generic(): + assert _detect_provider("https://www.linkedin.com/jobs/view/123") == "generic" + + +@pytest.mark.asyncio +async def test_fetch_too_short_body_returns_none(): + """JS-rendered pages return near-empty bodies (Oracle Cloud / LinkedIn). + The fetcher returns None rather than building a bogus RawJob — caller is + expected to fall back to `add_job_from_text` with a manual paste.""" + with patch( + "compass.pipeline.add_url._fetch_generic", + new=AsyncMock(return_value=("Page Title", "tiny body")), + ): + rj = await fetch_rawjob_from_url("https://jpmc.fa.oraclecloud.com/x") + assert rj is None + + +@pytest.mark.asyncio +async def test_fetch_real_body_builds_rawjob(): + long_body = "Build LangGraph agents in production. " * 30 + with patch( + "compass.pipeline.add_url._fetch_generic", + new=AsyncMock(return_value=("Agent Engineer — Acme Co", long_body)), + ): + rj = await fetch_rawjob_from_url("https://acme.com/careers/abc") + assert rj is not None + assert rj.title == "Agent Engineer — Acme Co" + assert rj.description == long_body + assert rj.url == "https://acme.com/careers/abc" + assert rj.source == "manual" + + +@pytest.mark.asyncio +async def test_fetch_explicit_company_and_title_overrides(): + long_body = "Build agents. " * 50 + with patch( + "compass.pipeline.add_url._fetch_generic", + new=AsyncMock(return_value=("Wrong Page Title", long_body)), + ): + rj = await fetch_rawjob_from_url( + "https://jpmc.fa.oraclecloud.com/x", + company="JPMorgan", + title="AI Engineer, LLM Suite", + ) + assert rj is not None + assert rj.company == "JPMorgan" + assert rj.title == "AI Engineer, LLM Suite" diff --git a/tests/pipeline/test_add_url_scheme.py b/tests/pipeline/test_add_url_scheme.py new file mode 100644 index 0000000..d1455de --- /dev/null +++ b/tests/pipeline/test_add_url_scheme.py @@ -0,0 +1,58 @@ +"""Regression tests for the URL scheme allowlist — added 2026-05-19 after the +adversarial review found that javascript:/ftp:/file:/data: URLs reached httpx +unchecked.""" + +from __future__ import annotations + +import pytest + +from compass.pipeline.add_url import fetch_rawjob_from_url + + +@pytest.mark.asyncio +async def test_javascript_scheme_rejected(): + """javascript: URLs must never be fetched — they can't possibly contain + a JD body and they may indicate copy-paste from a malicious source.""" + assert await fetch_rawjob_from_url("javascript:alert(1)") is None + + +@pytest.mark.asyncio +async def test_file_scheme_rejected(): + """file:// would expose local filesystem to the fetch path. Block.""" + assert await fetch_rawjob_from_url("file:///etc/passwd") is None + + +@pytest.mark.asyncio +async def test_ftp_scheme_rejected(): + """ftp:// isn't where JDs live. Block.""" + assert await fetch_rawjob_from_url("ftp://example.com/job") is None + + +@pytest.mark.asyncio +async def test_data_scheme_rejected(): + """data: URLs encode payloads inline — not a JD source.""" + assert await fetch_rawjob_from_url("data:text/plain,fake-jd") is None + + +@pytest.mark.asyncio +async def test_url_with_no_hostname_rejected(): + """Empty / fragment-only URLs have no host to fetch from.""" + assert await fetch_rawjob_from_url("https://") is None + assert await fetch_rawjob_from_url("https:///path") is None + assert await fetch_rawjob_from_url("") is None + + +@pytest.mark.asyncio +async def test_http_and_https_allowed(): + """http and https are the only allowed schemes. Body-length check still + applies; mocking the fetcher to return a long body confirms scheme passes.""" + from unittest.mock import AsyncMock, patch + + with patch( + "compass.pipeline.add_url._fetch_generic", + new=AsyncMock(return_value=("Title", "Real JD body. " * 100)), + ): + rj_https = await fetch_rawjob_from_url("https://example.com/job") + rj_http = await fetch_rawjob_from_url("http://example.com/job") + assert rj_https is not None + assert rj_http is not None diff --git a/tests/pipeline/test_auto_tags.py b/tests/pipeline/test_auto_tags.py new file mode 100644 index 0000000..2d73c60 --- /dev/null +++ b/tests/pipeline/test_auto_tags.py @@ -0,0 +1,82 @@ +"""P3 Obsidian leverage: vault_write_node auto-generates Obsidian tag-pane +filterable tags from JobNote fields so `#fit/strong AND #role/agent-engineer` +queries work naturally in the tag pane and Dataview.""" + +from __future__ import annotations + +from compass.pipeline.nodes.vault_write import _build_auto_tags + + +def test_build_auto_tags_strong_fit_approved(): + tags = _build_auto_tags( + tier="apply-now", + match_score=4.2, + role_family="agent-engineer", + hitl_decision="approved", + ) + assert tags == [ + "#tier/apply-now", + "#fit/strong", + "#role/agent-engineer", + "#decision/approved", + ] + + +def test_build_auto_tags_fit_buckets(): + assert "#fit/strong" in _build_auto_tags( + tier="apply-now", + match_score=4.0, + role_family="x", + hitl_decision=None, + ) + assert "#fit/decent" in _build_auto_tags( + tier="apply-now", + match_score=3.5, + role_family="x", + hitl_decision=None, + ) + assert "#fit/stretch" in _build_auto_tags( + tier="apply-now", + match_score=2.5, + role_family="x", + hitl_decision=None, + ) + assert "#fit/weak" in _build_auto_tags( + tier="apply-now", + match_score=1.5, + role_family="x", + hitl_decision=None, + ) + + +def test_build_auto_tags_omits_none_decision(): + """When hitl never ran (extract errored, etc.) no #decision/ tag is emitted.""" + tags = _build_auto_tags( + tier="opportunistic", + match_score=3.0, + role_family="agent-engineer", + hitl_decision=None, + ) + assert not any(t.startswith("#decision/") for t in tags) + + +def test_build_auto_tags_omits_empty_role_family(): + """role_family can be the empty string when classification deferred. No tag in that case.""" + tags = _build_auto_tags( + tier="apply-now", + match_score=4.0, + role_family="", + hitl_decision="approved", + ) + assert not any(t.startswith("#role/") for t in tags) + + +def test_build_auto_tags_includes_all_tiers(): + for tier in ["apply-now", "opportunistic", "backend-prep", "stretch", "skip", "unknown"]: + tags = _build_auto_tags( + tier=tier, + match_score=3.0, + role_family="agent-engineer", + hitl_decision=None, + ) + assert f"#tier/{tier}" in tags diff --git a/tests/pipeline/test_checkpoint_serde.py b/tests/pipeline/test_checkpoint_serde.py new file mode 100644 index 0000000..14bc012 --- /dev/null +++ b/tests/pipeline/test_checkpoint_serde.py @@ -0,0 +1,147 @@ +"""LangGraph checkpoint serde — Pydantic state classes on the msgpack allowlist +so checkpoint round-trip neither warns nor blocks under future STRICT mode.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +async def test_build_checkpoint_serde_allowlists_state_module(): + """_build_checkpoint_serde must return a serde whose allowlist suppresses + the 'unregistered type' warning AND preserves Pydantic classes on round-trip.""" + import langgraph.checkpoint.serde.jsonplus as _jp + from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer + + from compass.pipeline.graph import _build_checkpoint_serde + + _jp._warned_unregistered_types.clear() + + serde = _build_checkpoint_serde() + assert isinstance(serde, JsonPlusSerializer) + + import datetime + import logging + + from compass.pipeline.state import RawJob + + captured = [] + + class _H(logging.Handler): + def emit(self, r): + msg = r.getMessage() + if "unregistered type" in msg or "fingerprint" in msg.lower(): + captured.append(msg) + + h = _H() + logging.getLogger("langgraph").addHandler(h) + try: + rj = RawJob( + company="C", + title="T", + url="u://x", + source="ashby", + description="d", + date_posted=datetime.date(2026, 5, 19), + ) + ser = serde.dumps_typed(rj) + restored = serde.loads_typed(ser) + assert isinstance(restored, RawJob) + finally: + logging.getLogger("langgraph").removeHandler(h) + assert not captured, f"serde still emitted unregistered-type warnings: {captured}" + + +async def test_run_pipeline_emits_no_unregistered_warnings(checkpoint_db, monkeypatch, temp_vault): + """End-to-end smoke: run_pipeline mounts AsyncSqliteSaver with our serde. + No 'unregistered type' warnings during a full graph round-trip.""" + import datetime as _dt + import logging + + import langgraph.checkpoint.serde.jsonplus as _jp + + from compass.pipeline.state import JobRequirements, JobScore, RawJob + + _jp._warned_unregistered_types.clear() + + captured = [] + + class _Handler(logging.Handler): + def emit(self, record): + if "unregistered type" in record.getMessage(): + captured.append(record.getMessage()) + + handler = _Handler() + logging.getLogger("langgraph").addHandler(handler) + + async def fake_intake_filter(_): + return {"in_scope": True, "role_family": "agent-engineer"} + + async def fake_extract(_): + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP"], + nice_to_have_skills=[], + seniority="mid", + remote_policy="remote", + summary="x", + ) + } + + async def fake_score(_): + return { + "score_result": JobScore( + score=4.5, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=[], + tailoring_notes="", + ), + "score_threshold": 3.5, + } + + async def fake_tailor(_): + return {"tailored_paragraph": "..."} + + async def fake_vault_write(_): + return {"vault_written": True, "jobs_written": 1} + + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", lambda _: {"approved": True, "feedback": None} + ) + + from compass.pipeline.graph import run_pipeline + + await run_pipeline( + raw_jobs=[ + RawJob( + company="Sierra", + title="SWE", + url="u://x/1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 19), + ) + ] + ) + + logging.getLogger("langgraph").removeHandler(handler) + assert not captured, f"Expected no 'unregistered type' warnings, got: {captured}" diff --git a/tests/pipeline/test_graph_checkpointing.py b/tests/pipeline/test_graph_checkpointing.py new file mode 100644 index 0000000..c50d93e --- /dev/null +++ b/tests/pipeline/test_graph_checkpointing.py @@ -0,0 +1,141 @@ +"""End-to-end: graph pauses at hitl, state_store gets the row, +resume_pending finishes the run. + +These tests STUB the LLM-touching nodes (extract, score, tailor) — we only +exercise the graph machinery + interrupt + checkpointer + state_store +interaction. Real LLM calls live in the live-smoke check in Task 7.""" + +from __future__ import annotations + +import datetime as _dt +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from compass.hitl import state_store +from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob + + +@pytest.fixture +def stub_llm_nodes(monkeypatch): + """Replace extract / score / tailor / vault_write / intake_filter with deterministic stubs.""" + + async def fake_extract(state: CompassState) -> dict: + return { + "extracted_requirements": JobRequirements( + required_skills=["MCP", "Python"], + nice_to_have_skills=["LangGraph"], + seniority="mid", + remote_policy="remote", + summary="An agent role.", + ) + } + + async def fake_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=4.2, + reasoning="Strong match.", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="Lead with MCP.", + ) + } + + async def fake_tailor(state: CompassState) -> dict: + return {"tailored_paragraph": "Tailored: lead with MCP work."} + + async def fake_vault_write(state: CompassState) -> dict: + return {"vault_written": True, "jobs_written": 1} + + async def fake_intake_filter(state: CompassState) -> dict: + return {"in_scope": True, "role_family": "agent-engineer"} + + monkeypatch.setattr("compass.pipeline.graph.extract_node", fake_extract) + monkeypatch.setattr("compass.pipeline.graph.score_node", fake_score) + monkeypatch.setattr("compass.pipeline.graph.tailor_node", fake_tailor) + monkeypatch.setattr("compass.pipeline.graph.vault_write_node", fake_vault_write) + monkeypatch.setattr("compass.pipeline.graph.intake_filter_node", fake_intake_filter) + + +@pytest.fixture +def checkpoint_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db = tmp_path / "checkpoints.db" + import compass.config as cfg + + monkeypatch.setattr(cfg, "HITL_CHECKPOINT_DB", db) + return db + + +def _job(url: str = "https://jobs.example.com/sierra-1") -> RawJob: + return RawJob( + company="Sierra", + title="SWE, Agent", + url=url, + source="ashby", + description="An agent engineering role at Sierra.", + date_posted=_dt.date(2026, 5, 18), + ) + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_above_threshold_job_pauses_and_registers_in_state_store(monkeypatch): + """run_pipeline with a single above-threshold job: paused=1, written=0, + state_store has a row.""" + from compass.pipeline.graph import run_pipeline + + # No real scraping — pass the job in directly. + result = await run_pipeline(raw_jobs=[_job()]) + + assert result["jobs_processed"] == 1 + assert result["jobs_written"] == 0 # paused before vault_write + assert result["jobs_paused"] == 1 + rows = await state_store.list_pending() + assert len(rows) == 1 + assert rows[0]["company"] == "Sierra" + assert rows[0]["score"] == pytest.approx(4.2) + assert rows[0]["job_url"] == "https://jobs.example.com/sierra-1" + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_below_threshold_job_runs_to_completion_no_state_store_row(monkeypatch): + """Below-threshold path is unchanged from 1.A — no interrupt, vault_write fires.""" + + async def low_score(state: CompassState) -> dict: + return { + "score_result": JobScore( + score=2.0, + reasoning="Mismatched.", + matched_skills=[], + missing_skills=["A", "B"], + tailoring_notes="", + ) + } + + monkeypatch.setattr("compass.pipeline.graph.score_node", low_score) + from compass.pipeline.graph import run_pipeline + + result = await run_pipeline(raw_jobs=[_job()]) + assert result["jobs_written"] == 1 + assert result["jobs_paused"] == 0 + assert await state_store.list_pending() == [] + + +@pytest.mark.usefixtures("temp_hitl_db", "checkpoint_db", "stub_llm_nodes", "temp_vault") +async def test_thread_id_is_deterministic_for_same_url_and_batch_same_process(monkeypatch): + """Re-running the SAME batch (same start_wall) within the same process + reuses the same thread_id — second run hits the idempotent INSERT OR IGNORE path.""" + import os + + from compass.pipeline.graph import _thread_id_for + + monkeypatch.setattr(os, "getpid", lambda: 12345) + tid_a = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_b = _thread_id_for("https://jobs.example.com/x", _dt.datetime(2026, 5, 19, 9, 0, 0)) + tid_c = _thread_id_for("https://jobs.example.com/y", _dt.datetime(2026, 5, 19, 9, 0, 0)) + assert tid_a == tid_b + assert tid_a != tid_c + assert len(tid_a) == 16 diff --git a/tests/pipeline/test_graph_integration.py b/tests/pipeline/test_graph_integration.py index 5188bd8..32d568f 100644 --- a/tests/pipeline/test_graph_integration.py +++ b/tests/pipeline/test_graph_integration.py @@ -23,7 +23,7 @@ async def fake_extract(jd_text: str) -> JobRequirements: summary="Build agentic systems with LangGraph and MCP.", ) - async def fake_score(req, profile_text): + async def fake_score(req, profile_text, job=None): return JobScore( score=4.2, reasoning="Strong MCP + LangGraph match", @@ -40,7 +40,19 @@ async def fake_tailor(*args, **kwargs): monkeypatch.setattr(tailor, "_tailor", fake_tailor) -async def test_run_pipeline_end_to_end(temp_vault, mocked_llms): +@pytest.fixture +def auto_approve_hitl(monkeypatch): + """Stub the `interrupt()` call in hitl_node so the integration tests can + exercise the full extract -> score -> hitl -> tailor -> vault_write path + without needing a real human resume.""" + + def fake_interrupt(_payload): + return {"approved": True, "feedback": None} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + + +async def test_run_pipeline_end_to_end(temp_vault, mocked_llms, auto_approve_hitl): """Run a single fake job through the full graph; verify vault state.""" from compass.pipeline.graph import run_pipeline @@ -91,7 +103,7 @@ async def test_run_pipeline_skips_dedup_urls(temp_vault, mocked_llms): title="Agent Engineer", url="https://jobs.ashbyhq.com/sierra/test-uuid", source="ashby", - description="...", + description="Build LangGraph agents with MCP tool calling.", date_posted=date(2026, 5, 17), ), ] @@ -100,7 +112,7 @@ async def test_run_pipeline_skips_dedup_urls(temp_vault, mocked_llms): assert state["jobs_written"] == 0 -async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms): +async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms, auto_approve_hitl): """After processing, master-gap-plan.md should be regenerated.""" from compass.pipeline.graph import run_pipeline @@ -110,7 +122,7 @@ async def test_run_pipeline_regenerates_gap_plan(temp_vault, mocked_llms): title="Agent Engineer", url="https://jobs.ashbyhq.com/sierra/test-uuid", source="ashby", - description="...", + description="Build LangGraph agents with MCP tool calling.", date_posted=date(2026, 5, 17), ), ] @@ -142,7 +154,7 @@ async def fake_extract(jd_text): summary="Build agents.", ) - async def fake_score(req, profile_text): + async def fake_score(req, profile_text, job=None): return JobScore( score=2.0, reasoning="weak match against requirements.", @@ -177,7 +189,7 @@ async def fake_tailor(*args, **kwargs): assert tailor_calls["count"] == 0 -async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms): +async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms, auto_approve_hitl): """Every run_pipeline invocation appends a row to _meta/pipeline-runs.md.""" from compass.pipeline.graph import run_pipeline @@ -197,3 +209,45 @@ async def test_run_pipeline_appends_to_run_log(temp_vault, mocked_llms): log_text = log_path.read_text() assert "| Timestamp |" in log_text # header was written assert "| 1 |" in log_text # one job processed + + +async def test_run_log_migrates_5col_to_6col_on_first_post_1b1_append(temp_vault): + """A pre-1.B.1 pipeline-runs.md with 5-col rows gets migrated to 6-col on + the next append. Existing data preserved with Paused=0.""" + log_path = temp_vault / "_meta" / "pipeline-runs.md" + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text( + "# Pipeline Run Log\n\n" + "| Timestamp | Processed | Written | Errors | Duration |\n" + "|---|---|---|---|---|\n" + "| 2026-05-17T10:00:00 | 20 | 5 | 0 | 7.2s |\n" + "| 2026-05-18T10:00:00 | 15 | 3 | 1 | 5.4s |\n", + encoding="utf-8", + ) + + from compass.pipeline.graph import _append_run_log + + state = { + "raw_jobs": [], + "current_job": None, + "extracted_requirements": None, + "score_result": None, + "in_scope": None, + "role_family": None, + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 10, + "jobs_written": 2, + "errors": [], + "thread_id": None, + "score_threshold": None, + } + state["jobs_paused"] = 1 # type: ignore[typeddict-unknown-key] + _append_run_log(state, 4.1) + + text = log_path.read_text(encoding="utf-8") + assert "| Timestamp | Processed | Written | Paused | Errors | Duration |" in text + assert "| 2026-05-17T10:00:00 | 20 | 5 | 0 | 0 | 7.2s |" in text # migrated + assert "| 2026-05-18T10:00:00 | 15 | 3 | 0 | 1 | 5.4s |" in text diff --git a/tests/pipeline/test_hitl_node_interrupt.py b/tests/pipeline/test_hitl_node_interrupt.py new file mode 100644 index 0000000..f8ca5ae --- /dev/null +++ b/tests/pipeline/test_hitl_node_interrupt.py @@ -0,0 +1,142 @@ +"""hitl_node calls interrupt() above threshold; auto-rejects below threshold.""" + +from __future__ import annotations + +import datetime as _dt + +import pytest + +from compass.pipeline.nodes.hitl import hitl_node +from compass.pipeline.state import CompassState, JobScore, RawJob + + +def _state(score: float | None) -> CompassState: + job = RawJob( + company="Sierra", + title="SWE, Agent", + url="https://jobs.example.com/sierra-1", + source="ashby", + description="...", + date_posted=_dt.date(2026, 5, 18), + ) + sr = ( + None + if score is None + else JobScore( + score=score, + reasoning="ok", + matched_skills=["MCP"], + missing_skills=["LangGraph"], + tailoring_notes="", + ) + ) + return { + "raw_jobs": [], + "current_job": job, + "extracted_requirements": None, + "score_result": sr, + "in_scope": True, + "role_family": "agent-engineer", + "human_approved": None, + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": "tid-test", + } + + +async def test_below_threshold_auto_rejects_without_interrupt(monkeypatch): + """Below SCORE_THRESHOLD short-circuits — interrupt MUST NOT fire (no human prompt cost).""" + called = {"interrupt": 0} + + def boom(_payload): + called["interrupt"] += 1 + raise AssertionError("interrupt should not have been called") + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", boom) + result = await hitl_node(_state(score=2.0)) + assert result == {"human_approved": False} + assert called["interrupt"] == 0 + + +async def test_missing_score_auto_rejects(): + result = await hitl_node(_state(score=None)) + assert result == {"human_approved": False} + + +async def test_above_threshold_calls_interrupt_with_payload(monkeypatch): + captured = {} + + def fake_interrupt(payload): + captured.update(payload) + # Simulate resume value (this is what Command(resume=...) sends back) + return {"approved": True, "feedback": "Strong fit"} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + result = await hitl_node(_state(score=4.2)) + assert captured["kind"] == "approval_request" + assert captured["company"] == "Sierra" + assert captured["score"] == pytest.approx(4.2) + assert captured["matched_skills"] == ["MCP"] + assert result == {"human_approved": True, "human_feedback": "Strong fit"} + + +async def test_resume_rejection_propagates(monkeypatch): + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda _p: {"approved": False, "feedback": "Not a fit"}, + ) + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False, "human_feedback": "Not a fit"} + + +async def test_malformed_resume_value_defaults_to_rejected(monkeypatch): + """If Command(resume=...) somehow sends a non-dict, treat as reject not crash.""" + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", lambda _p: "bogus") + result = await hitl_node(_state(score=4.2)) + assert result == {"human_approved": False} + + +async def test_below_threshold_uses_state_score_threshold_when_present(monkeypatch): + """If state['score_threshold'] is set (captured at score time), hitl uses + it — not the live config value. Prevents resume-time SCORE_THRESHOLD + edits from silently auto-rejecting a pre-approved thread.""" + captured = {} + + def fake_interrupt(_p): + captured["called"] = True + return {"approved": True} + + monkeypatch.setattr("compass.pipeline.nodes.hitl.interrupt", fake_interrupt) + + state = _state(score=3.0) + state["score_threshold"] = 2.0 # captured at score time; less than 3.0 + # Live config: SCORE_THRESHOLD=3.5 (default) + import compass.pipeline.nodes.hitl as hitl_mod + + monkeypatch.setattr(hitl_mod, "SCORE_THRESHOLD", 3.5) + + result = await hitl_node(state) + assert captured.get("called") is True # interrupt fired despite config threshold > score + assert result["human_approved"] is True + + +async def test_below_threshold_falls_back_to_config_when_state_threshold_missing(monkeypatch): + """Backward-compat: paused threads from before this fix have no + state['score_threshold']; hitl falls back to config.""" + monkeypatch.setattr( + "compass.pipeline.nodes.hitl.interrupt", + lambda p: pytest.fail("interrupt should not fire when score < config threshold"), + ) + + state = _state(score=2.0) + # state['score_threshold'] not set — defaults to None via .get() + import compass.pipeline.nodes.hitl as hitl_mod + + monkeypatch.setattr(hitl_mod, "SCORE_THRESHOLD", 3.5) + + result = await hitl_node(state) + assert result == {"human_approved": False} diff --git a/tests/pipeline/test_intake_filter.py b/tests/pipeline/test_intake_filter.py index 3b76562..3f7e24f 100644 --- a/tests/pipeline/test_intake_filter.py +++ b/tests/pipeline/test_intake_filter.py @@ -4,7 +4,10 @@ from compass.pipeline.state import CompassState, RawJob -def _state(title: str, description: str = "We build agents.") -> CompassState: +def _state(title: str, description: str = "We build LangGraph agents with MCP.") -> CompassState: + # Default description carries a STRONG agent signal so tests that don't + # care about body content (most of them) still pass the agent-signal gate. + # Tests that DO care override the description explicitly. job = RawJob( company="Acme", title=title, @@ -144,8 +147,220 @@ async def fake_llm(title, body): monkeypatch.setattr(mod, "llm_classify", fake_llm) + # `Solutions Architect` is borderline — the keyword classifier returns + # None (LLM stage decides). MTS used to be borderline but as of the + # 3-month-pivot title expansion routes directly to agent-engineer, so + # it no longer exercises the LLM-failure code path. out = await mod.intake_filter_node( - _state("Member of Technical Staff", "writes Python systems code") + _state("Solutions Architect", "writes Python systems code") ) assert out["in_scope"] is True assert out["role_family"] == "other-eng" + + +class TestRejectRules: + """preferences.md `reject_if_title_contains` + `reject_if_jd_contains` are + enforced at intake before any LLM call. Saves ~40-60% LLM cost on a wide + scrape and keeps senior/staff/principal noise out of the vault.""" + + def _write_prefs(self, temp_vault): + prefs = temp_vault / "_profile" / "preferences.md" + prefs.write_text( + "---\ntype: profile\n---\n" + "## Role filters\n" + "```yaml\n" + "reject_if_title_contains:\n" + " - Senior\n" + " - Sr.\n" + " - Staff\n" + " - Principal\n" + "reject_if_jd_contains:\n" + " - 5+ years\n" + " - PhD required\n" + "```\n", + encoding="utf-8", + ) + + async def test_senior_in_title_dropped_before_llm(self, monkeypatch, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + self._write_prefs(temp_vault) + mock_llm = AsyncMock() + monkeypatch.setattr(mod, "llm_classify", mock_llm) + out = await mod.intake_filter_node(_state("Senior Software Engineer, Agents")) + assert out["in_scope"] is False + assert out["role_family"] == "out-of-scope" + mock_llm.assert_not_called() + log = (temp_vault / "_meta" / "filtered-jobs.md").read_text() + assert "title rejects" in log + assert "senior" in log.lower() + + async def test_yoe_in_jd_dropped_before_llm(self, monkeypatch, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + self._write_prefs(temp_vault) + mock_llm = AsyncMock() + monkeypatch.setattr(mod, "llm_classify", mock_llm) + out = await mod.intake_filter_node( + _state("Engineer", "Looking for someone with 5+ years of LLM experience.") + ) + assert out["in_scope"] is False + assert out["role_family"] == "out-of-scope" + mock_llm.assert_not_called() + log = (temp_vault / "_meta" / "filtered-jobs.md").read_text() + assert "jd rejects" in log + + async def test_reject_rules_dont_affect_clean_jds(self, monkeypatch, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + self._write_prefs(temp_vault) + mock_llm = AsyncMock() + monkeypatch.setattr(mod, "llm_classify", mock_llm) + out = await mod.intake_filter_node( + _state("Agent Engineer", "Build LangGraph agents in Python with MCP.") + ) + assert out["in_scope"] is True + + async def test_missing_preferences_file_doesnt_crash(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + # Delete the seed preferences file so load_reject_rules returns empties + (temp_vault / "_profile" / "preferences.md").unlink() + out = await mod.intake_filter_node(_state("Agent Engineer")) + assert out["in_scope"] is True + assert out["role_family"] == "agent-engineer" + + +class TestAgentSignalGate: + """JD-body agent-signal check: AI-oriented title with ZERO agent terms in + the body is dropped. Other role families pass through unfiltered.""" + + async def test_agent_title_with_no_body_signal_dropped(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("AI Engineer", "We build dashboards using React and Postgres.") + ) + assert out["in_scope"] is False + assert out["role_family"] == "out-of-scope" + assert out["agent_signal_count"] == 0 + log = (temp_vault / "_meta" / "filtered-jobs.md").read_text() + assert "no-strong-agent-signal" in log + + async def test_agent_title_with_body_signal_kept(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state( + "AI Engineer", + "Build LangGraph agents with MCP tool calling for production users.", + ) + ) + assert out["in_scope"] is True + assert out["agent_signal_count"] >= 3 # langgraph + agents + mcp + tool calling + + async def test_swe_backend_passes_through_without_body_signal(self, temp_vault): + """Generic SWE titles at AI-native companies sometimes describe agent + work — don't gate them on body signal. The score node sees it later.""" + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("Backend Engineer", "Build distributed systems with Postgres.") + ) + assert out["in_scope"] is True + assert out["role_family"] == "swe-backend" + assert out["agent_signal_count"] == 0 # tracked but not gating + + async def test_agent_engineer_title_with_one_signal_kept(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("Agent Engineer", "Build and ship multi-agent systems.") + ) + assert out["in_scope"] is True + assert out["agent_signal_count"] >= 1 + + +class TestAgentSignalFalsePositives: + """Regression tests for 2026-05-19 adversarial review: weak-signal words + ('agent' alone in non-agentic context) must NOT pass the gate.""" + + async def test_change_agent_marketing_speak_dropped(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state( + "AI Engineer", + "We're looking for a change agent who can transform our team.", + ) + ) + assert out["in_scope"] is False, "marketing 'change agent' must not pass the gate" + assert out["agent_signal_count"] >= 1, "but weak signal count should still reflect the hit" + + async def test_user_agent_http_header_context_dropped(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("AI Engineer", "Set the User-Agent header in your requests to the API.") + ) + assert out["in_scope"] is False + assert out["agent_signal_count"] >= 1 + + async def test_real_strong_signal_passes(self, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("AI Engineer", "Build LangGraph multi-agent systems with MCP.") + ) + assert out["in_scope"] is True + assert out["agent_signal_count"] >= 3 # langgraph + multi-agent + mcp + + async def test_strong_signal_alone_passes_even_without_agent_word(self, temp_vault): + """A JD that mentions only LangGraph + tool-calling should pass — those + are unambiguous agent-eng terms even without the word 'agent'.""" + from compass.pipeline.nodes import intake_filter as mod + + out = await mod.intake_filter_node( + _state("AI Engineer", "Implement function-calling with LangGraph for our platform.") + ) + assert out["in_scope"] is True + + +class TestAgentSignalCountConsistency: + """Regression: every dropped JD should set agent_signal_count to a number, + not None. Downstream readers should never have to handle None for that key.""" + + async def test_title_reject_sets_signal_count_zero(self, monkeypatch, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + TestRejectRules()._write_prefs(temp_vault) + out = await mod.intake_filter_node(_state("Senior Software Engineer")) + assert out["in_scope"] is False + assert out["agent_signal_count"] == 0 + + async def test_jd_reject_sets_signal_count_zero(self, monkeypatch, temp_vault): + from compass.pipeline.nodes import intake_filter as mod + + TestRejectRules()._write_prefs(temp_vault) + out = await mod.intake_filter_node( + _state("AI Engineer", "Looking for 5+ years experience with agents.") + ) + assert out["in_scope"] is False + assert out["agent_signal_count"] == 0 + + +def test_agent_signal_count_helper(): + """Counts distinct term hits, case-insensitive, word-boundary safe.""" + from compass.pipeline.nodes.intake_filter import _agent_signal_count + + assert _agent_signal_count("") == 0 + assert _agent_signal_count("This role is about React and dashboards.") == 0 + assert _agent_signal_count("Build LangGraph agents.") == 2 # langgraph + agents + # repeated terms count once + assert _agent_signal_count("agents agents agents") == 1 + # word-boundary: "agenda" should not match "agent" + assert _agent_signal_count("Your daily agenda includes meetings.") == 0 + # "agentic" matches its own pattern, not "agent" + # "agentic AI" is a STRONG term (matches r"\bagentic\s+ai\b"); "agentic" + # alone is also a WEAK term — both fire, distinct hits. + assert _agent_signal_count("We are an agentic AI company.") == 2 diff --git a/tests/pipeline/test_role_family.py b/tests/pipeline/test_role_family.py index c225dfb..2b2d348 100644 --- a/tests/pipeline/test_role_family.py +++ b/tests/pipeline/test_role_family.py @@ -40,8 +40,39 @@ def test_borderline_solutions_architect_is_none(self): def test_borderline_fde_is_none(self): assert keyword_classify("Forward Deployed Engineer")[0] is None - def test_borderline_random_title(self): - assert keyword_classify("Member of Technical Staff")[0] is None + def test_master_boolean_agent_titles_route_to_agent_engineer(self): + """Titles named in _profile/target-roles.md::JD-master-boolean should + all classify as in-scope agent-engineer without hitting the LLM.""" + for title in [ + "AI Agent Engineer", + "Agentic AI Engineer", + "Software Engineer, Agents", + "Software Engineer - Agentic", + "AI Native Engineer", + ]: + in_scope, family = keyword_classify(title) + assert in_scope is True, f"{title!r} should be IN" + assert family == "agent-engineer", f"{title!r} → {family!r}" + + def test_master_boolean_applied_ai_titles_route_to_applied_ai(self): + for title in [ + "GenAI Engineer", + "AI Enablement Engineer", + "AI/ML Engineer", + ]: + in_scope, family = keyword_classify(title) + assert in_scope is True, f"{title!r} should be IN" + assert family == "applied-ai", f"{title!r} → {family!r}" + + def test_member_of_technical_staff_routes_to_agent_engineer(self): + """MTS is a frontier-startup flat-hierarchy signal — Sierra / Decagon / + Cognition / Cursor / xAI / Mistral all use it for agent-eng ICs. + Per _profile/target-roles.md it's in-range, so the keyword classifier + routes it to agent-engineer; the body-signal upgrader can move it + elsewhere if the JD is research-flavored.""" + in_scope, family = keyword_classify("Member of Technical Staff") + assert in_scope is True + assert family == "agent-engineer" def test_out_keyword_beats_in_keyword(self): in_scope, family = keyword_classify("Sales Engineer") diff --git a/tests/pipeline/test_role_family_no_mobile_upgrade.py b/tests/pipeline/test_role_family_no_mobile_upgrade.py new file mode 100644 index 0000000..93921e3 --- /dev/null +++ b/tests/pipeline/test_role_family_no_mobile_upgrade.py @@ -0,0 +1,45 @@ +"""Regression: a React Native job at an LLM-native startup whose JD body +mentions LangGraph + multi-agent in passing used to get upgraded from +swe-mobile to agent-engineer (and same for swe-frontend). Mobile and +frontend specialists aren't who Akash is — they're not promotion-eligible +anymore. 2026-05-19 adversarial review (wave 2).""" + +from __future__ import annotations + +from compass.pipeline.role_family import GENERIC_FAMILIES, upgrade_family + + +def test_swe_mobile_not_promotion_eligible(): + assert "swe-mobile" not in GENERIC_FAMILIES + + +def test_swe_frontend_not_promotion_eligible(): + assert "swe-frontend" not in GENERIC_FAMILIES + + +def test_swe_mobile_stays_mobile_even_with_strong_agent_body(): + """A mobile job whose JD mentions LangGraph + multi-agent should still + be classified as swe-mobile — Akash isn't a mobile dev.""" + body = ( + "Build React Native iOS apps. Backend team uses LangGraph for " + "multi-agent orchestration but you'll integrate via REST API." + ) + result = upgrade_family("swe-mobile", body) + assert result == "swe-mobile" + + +def test_swe_frontend_stays_frontend_even_with_strong_agent_body(): + body = "React frontend for an LangGraph multi-agent platform. TypeScript." + result = upgrade_family("swe-frontend", body) + assert result == "swe-frontend" + + +def test_swe_backend_still_promotion_eligible(): + """Backend SWE titles at agent-native companies (Sierra 'Software Engineer, + Product') ARE legitimately agent-eng IC roles. Keep them eligible.""" + body = ( + "Build LangGraph agents. Tool calling, multi-agent orchestration, " + "MCP integration. Python/FastAPI backend." + ) + result = upgrade_family("swe-backend", body) + assert result == "agent-engineer" diff --git a/tests/pipeline/test_routing.py b/tests/pipeline/test_routing.py index 9aa5349..5a87e34 100644 --- a/tests/pipeline/test_routing.py +++ b/tests/pipeline/test_routing.py @@ -1,8 +1,7 @@ -"""Tests for reflect_node + hitl_node — control-flow nodes (no LLM in 0.B).""" +"""Tests for reflect_node — control-flow nodes (no LLM).""" from datetime import date -from compass.config import SCORE_THRESHOLD from compass.pipeline.state import CompassState, JobScore, RawJob @@ -43,29 +42,6 @@ async def test_reflect_node_is_passthrough(): assert result == {} -async def test_hitl_node_approves_when_score_meets_threshold(): - from compass.pipeline.nodes.hitl import hitl_node - - result = await hitl_node(_state(SCORE_THRESHOLD)) - assert result["human_approved"] is True - - -async def test_hitl_node_rejects_when_score_below_threshold(): - from compass.pipeline.nodes.hitl import hitl_node - - result = await hitl_node(_state(SCORE_THRESHOLD - 0.5)) - assert result["human_approved"] is False - - -async def test_hitl_node_handles_missing_score(): - from compass.pipeline.nodes.hitl import hitl_node - - state = _state(0.0) - state["score_result"] = None - result = await hitl_node(state) - assert result["human_approved"] is False - - class TestIntakeFilterRouting: def test_out_of_scope_yields_in_scope_false_and_no_write(self, temp_vault): """An out-of-scope job must short-circuit to END without invoking diff --git a/tests/pipeline/test_score.py b/tests/pipeline/test_score.py index 94de527..9e42262 100644 --- a/tests/pipeline/test_score.py +++ b/tests/pipeline/test_score.py @@ -31,7 +31,7 @@ def _state(req): async def test_score_node_returns_jobscore(monkeypatch, temp_vault): from compass.pipeline.nodes import score - async def fake_score(req, profile_text: str) -> JobScore: + async def fake_score(req, profile_text: str, job=None) -> JobScore: return JobScore( score=4.2, reasoning="Strong MCP match", @@ -64,20 +64,25 @@ async def test_score_node_missing_requirements_errors(monkeypatch, temp_vault): async def test_score_node_passes_profile_to_llm(monkeypatch, temp_vault): - """The profile_text passed to _score must include resume + skill-inventory content.""" + """The profile_text passed to _score includes resume + RAG-retrieved skill chunks.""" from compass.pipeline.nodes import score + from compass.rag.retriever import RetrievedChunk captured = {} - async def fake_score(req, profile_text: str) -> JobScore: + async def fake_score(req, profile_text: str, job=None) -> JobScore: captured["profile_text"] = profile_text return JobScore( score=3.0, reasoning="", matched_skills=[], missing_skills=[], tailoring_notes="" ) + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 3.", score=0.9)] + monkeypatch.setattr(score, "_score", fake_score) + monkeypatch.setattr(score, "rag_retrieve", fake_retrieve) req = JobRequirements( - required_skills=[], + required_skills=["Python"], nice_to_have_skills=[], years_experience=None, seniority="mid", @@ -86,7 +91,7 @@ async def fake_score(req, profile_text: str) -> JobScore: ) await score.score_node(_state(req)) assert "Fake resume body" in captured["profile_text"] - assert "Python: 3" in captured["profile_text"] + assert "## Python\nLevel 3." in captured["profile_text"] async def test_score_node_drops_skills_outside_jd_universe(monkeypatch, temp_vault): @@ -95,7 +100,7 @@ async def test_score_node_drops_skills_outside_jd_universe(monkeypatch, temp_vau them so gap_aggregator never sees JD-irrelevant skills.""" from compass.pipeline.nodes import score - async def hallucinating_score(req, profile_text): + async def hallucinating_score(req, profile_text, job=None): # Simulate the live-run bug: LLM listed every profile skill as "matched" # ignoring the empty required_skills. return JobScore( @@ -124,7 +129,7 @@ async def test_score_node_keeps_only_jd_skills(monkeypatch, temp_vault): """When LLM mixes JD skills with hallucinated ones, only JD skills survive.""" from compass.pipeline.nodes import score - async def mixed_score(req, profile_text): + async def mixed_score(req, profile_text, job=None): return JobScore( score=4.0, reasoning="...", @@ -154,7 +159,7 @@ async def test_score_node_resolves_matched_missing_overlap(monkeypatch, temp_vau would count a matched skill as a gap.""" from compass.pipeline.nodes import score - async def overlapping_score(req, profile_text): + async def overlapping_score(req, profile_text, job=None): return JobScore( score=4.0, reasoning="...", @@ -190,7 +195,7 @@ async def test_score_node_retries_on_truncated_reasoning(monkeypatch, temp_vault call_count = {"n": 0} - async def flaky_score(req, profile_text): + async def flaky_score(req, profile_text, job=None): call_count["n"] += 1 if call_count["n"] == 1: return JobScore( @@ -228,7 +233,7 @@ async def test_score_node_no_retry_on_complete_reasoning(monkeypatch, temp_vault call_count = {"n": 0} - async def good_score(req, profile_text): + async def good_score(req, profile_text, job=None): call_count["n"] += 1 return JobScore( score=4.0, diff --git a/tests/pipeline/test_score_node_rag.py b/tests/pipeline/test_score_node_rag.py new file mode 100644 index 0000000..4b18db0 --- /dev/null +++ b/tests/pipeline/test_score_node_rag.py @@ -0,0 +1,87 @@ +"""score_node uses RAG retrieval to build profile context, not full inventory.""" + +from __future__ import annotations + +import pytest + +from compass.pipeline.state import JobRequirements + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +def _stub_req(**overrides) -> JobRequirements: + base = dict( + required_skills=["Python", "LangGraph"], + nice_to_have_skills=["MCP"], + seniority="senior", + remote_policy="remote", + summary="Build agentic systems.", + ) + base.update(overrides) + return JobRequirements(**base) + + +async def test_profile_text_includes_resume_and_retrieved_chunks(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME TEXT") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME TEXT" in text + assert "## Python\nLevel 4." in text + + +async def test_retrieval_query_carries_jd_skills_and_summary(monkeypatch): + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + captured: dict[str, object] = {} + + async def fake_retrieve(query, k=8): + captured["query"] = query + return [RetrievedChunk(skill="Python", document="## Python", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "") + + await score_mod._profile_text(_stub_req()) + q = captured["query"] + assert "Python" in q and "LangGraph" in q and "MCP" in q + assert "Build agentic systems" in q + + +async def test_score_node_handles_empty_jd_skills(monkeypatch): + """JD with no required/nice-to-have skills falls back to summary-only query.""" + from compass.pipeline.nodes import score as score_mod + from compass.rag.retriever import RetrievedChunk + + async def fake_retrieve(query, k=8): + return [RetrievedChunk(skill="Python", document="## Python\nLevel 4.", score=0.9)] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME") + + text = await score_mod._profile_text( + _stub_req(required_skills=[], nice_to_have_skills=[], summary="Just a summary.") + ) + assert "RESUME" in text + assert "## Python\nLevel 4." in text + + +async def test_score_node_handles_empty_retrieval(monkeypatch): + """If retriever returns [], profile context still includes the resume.""" + from compass.pipeline.nodes import score as score_mod + + async def fake_retrieve(query, k=8): + return [] + + monkeypatch.setattr("compass.pipeline.nodes.score.rag_retrieve", fake_retrieve) + monkeypatch.setattr("compass.pipeline.nodes.score.read_resume", lambda: "RESUME ONLY") + + text = await score_mod._profile_text(_stub_req()) + assert "RESUME ONLY" in text diff --git a/tests/pipeline/test_scrape_recency.py b/tests/pipeline/test_scrape_recency.py new file mode 100644 index 0000000..2d87dc0 --- /dev/null +++ b/tests/pipeline/test_scrape_recency.py @@ -0,0 +1,52 @@ +"""Recency handling in _scrape_all: drop >30d-old postings, sort each board +by date_posted DESC, None-dated jobs sink to the bottom.""" + +from __future__ import annotations + +from datetime import date, timedelta + +from compass.pipeline.graph import MAX_POSTING_AGE_DAYS, _filter_and_sort_by_recency +from compass.pipeline.state import RawJob + + +def _job(name: str, days_ago: int | None) -> RawJob: + return RawJob( + company="Acme", + title=f"role-{name}", + url=f"https://x/{name}", + source="manual", + description="t", + date_posted=(date.today() - timedelta(days=days_ago)) if days_ago is not None else None, + ) + + +def test_drops_jobs_older_than_cutoff(): + cutoff = MAX_POSTING_AGE_DAYS + jobs = [_job("fresh", 1), _job("borderline", cutoff), _job("stale", cutoff + 1)] + kept = _filter_and_sort_by_recency(jobs) + names = [j.title for j in kept] + assert "role-fresh" in names + assert "role-borderline" in names # exactly on the boundary is kept + assert "role-stale" not in names + + +def test_undated_jobs_are_kept_and_sink_to_bottom(): + """ATSes that don't expose date_posted shouldn't have their jobs dropped — + we can't distinguish stale from undated. But dated jobs are preferred.""" + jobs = [_job("undated", None), _job("recent", 1), _job("old", 14)] + kept = _filter_and_sort_by_recency(jobs) + assert len(kept) == 3 + # recent (1d ago) > old (14d ago) > undated + assert kept[0].title == "role-recent" + assert kept[1].title == "role-old" + assert kept[2].title == "role-undated" + + +def test_freshest_first(): + jobs = [_job("oldish", 20), _job("today", 0), _job("yesterday", 1)] + kept = _filter_and_sort_by_recency(jobs) + assert [j.title for j in kept] == ["role-today", "role-yesterday", "role-oldish"] + + +def test_empty_list_returns_empty(): + assert _filter_and_sort_by_recency([]) == [] diff --git a/tests/pipeline/test_tailor_score_gate.py b/tests/pipeline/test_tailor_score_gate.py new file mode 100644 index 0000000..c24a992 --- /dev/null +++ b/tests/pipeline/test_tailor_score_gate.py @@ -0,0 +1,87 @@ +"""Regression: tailor_node used to fire on any `human_approved=True` job +regardless of score. A mis-click in the HiTL UI on a score=1.0 row would +burn a Sonnet call. Fixed 2026-05-19 wave-2 review.""" + +from __future__ import annotations + +from datetime import date +from unittest.mock import AsyncMock + +import pytest + +from compass.pipeline.state import JobScore, RawJob + + +def _state(score_value: float, threshold: float = 3.5): + return { + "raw_jobs": [], + "current_job": RawJob( + company="X", + title="Y", + url="https://x", + source="manual", + description="t", + date_posted=date.today(), + ), + "extracted_requirements": None, + "score_result": JobScore( + score=score_value, + reasoning="reason ending properly.", + matched_skills=[], + missing_skills=[], + tailoring_notes="", + ), + "in_scope": True, + "role_family": "agent-engineer", + "agent_signal_count": 2, + "human_approved": True, # human clicked approve + "human_feedback": None, + "tailored_paragraph": None, + "vault_written": False, + "jobs_processed": 0, + "jobs_written": 0, + "errors": [], + "thread_id": None, + "score_threshold": threshold, + } + + +@pytest.mark.asyncio +async def test_tailor_skipped_when_score_below_threshold(monkeypatch, temp_vault): + from compass.pipeline.nodes import tailor + + called = AsyncMock() + monkeypatch.setattr(tailor, "_tailor", called) + + # Score 1.0, threshold 3.5 — human_approved=True doesn't override + out = await tailor.tailor_node(_state(score_value=1.0)) + called.assert_not_called() + assert out == {} + + +@pytest.mark.asyncio +async def test_tailor_runs_when_score_meets_threshold(monkeypatch, temp_vault): + from compass.pipeline.nodes import tailor + + async def fake(*args, **kwargs): + return "tailored paragraph" + + monkeypatch.setattr(tailor, "_tailor", fake) + + out = await tailor.tailor_node(_state(score_value=4.0, threshold=3.5)) + assert out.get("tailored_paragraph") == "tailored paragraph" + + +@pytest.mark.asyncio +async def test_tailor_uses_state_threshold_over_config(monkeypatch, temp_vault): + """If `score_threshold` is in state (sticky from score_node), use it + rather than the live SCORE_THRESHOLD config — supports threshold drift.""" + from compass.pipeline.nodes import tailor + + called = AsyncMock() + monkeypatch.setattr(tailor, "_tailor", called) + + # Sticky threshold 4.5, score 4.0 — score below sticky threshold, skip + out = await tailor.tailor_node(_state(score_value=4.0, threshold=4.5)) + called.assert_not_called() + assert out == {} diff --git a/tests/pipeline/test_vault_write.py b/tests/pipeline/test_vault_write.py index 7d097ef..dcb6995 100644 --- a/tests/pipeline/test_vault_write.py +++ b/tests/pipeline/test_vault_write.py @@ -7,9 +7,16 @@ from compass.pipeline.state import CompassState, JobRequirements, JobScore, RawJob -def _state( - skills_required: list[str], skills_matched: list[str], score: float = 4.2 +def _build_state_for_score( + *, + score: float = 4.2, + skills_required: list[str] | None = None, + skills_matched: list[str] | None = None, + human_approved: bool = True, + human_feedback: str | None = None, ) -> CompassState: + skills_required = skills_required if skills_required is not None else ["MCP", "LangGraph"] + skills_matched = skills_matched if skills_matched is not None else ["MCP"] return { "raw_jobs": [], "current_job": RawJob( @@ -38,8 +45,8 @@ def _state( ), "in_scope": True, "role_family": "agent-engineer", - "human_approved": True, - "human_feedback": None, + "human_approved": human_approved, + "human_feedback": human_feedback, "tailored_paragraph": None, "vault_written": False, "jobs_processed": 0, @@ -51,7 +58,9 @@ def _state( async def test_vault_write_node_writes_jobnote(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - result = await vault_write_node(_state(["MCP", "LangGraph"], ["MCP"])) + result = await vault_write_node( + _build_state_for_score(skills_required=["MCP", "LangGraph"], skills_matched=["MCP"]) + ) assert result["vault_written"] is True assert result["jobs_written"] == 1 job_files = list((temp_vault / "jobs").glob("*.md")) @@ -71,7 +80,9 @@ async def test_vault_write_node_records_skills_on_jobnote(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - await vault_write_node(_state(["MCP", "LangGraph"], ["MCP"])) + await vault_write_node( + _build_state_for_score(skills_required=["MCP", "LangGraph"], skills_matched=["MCP"]) + ) job_files = list((temp_vault / "jobs").glob("*.md")) assert len(job_files) == 1 loaded = frontmatter.load(job_files[0]) @@ -82,7 +93,7 @@ async def test_vault_write_node_records_skills_on_jobnote(temp_vault): async def test_vault_write_node_writes_company_note(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - await vault_write_node(_state(["MCP"], ["MCP"])) + await vault_write_node(_build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"])) company_path = temp_vault / "companies" / "Sierra.md" assert company_path.exists() @@ -90,7 +101,7 @@ async def test_vault_write_node_writes_company_note(temp_vault): async def test_vault_write_node_handles_missing_state(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"]) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"]) state["score_result"] = None result = await vault_write_node(state) assert result["vault_written"] is False @@ -101,7 +112,7 @@ async def test_vault_write_node_persists_tailored_paragraph(temp_vault): """When state has tailored_paragraph, it lands on the JobNote.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"]) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"]) state["tailored_paragraph"] = "Lead with your Cisco MCP server work." await vault_write_node(state) job_files = list((temp_vault / "jobs").glob("*.md")) @@ -114,7 +125,7 @@ async def test_vault_write_node_persists_full_jd_body(temp_vault): raw JD is preserved in the JobNote body, not just the LLM summary.""" import compass.pipeline.nodes.vault_write as vw - state = _state(["MCP"], ["MCP"], score=4.2) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.2) state["current_job"] = state["current_job"].model_copy( update={"description": "VERY_DISTINCTIVE_RAW_JD_MARKER agentic engineering."} ) @@ -129,7 +140,7 @@ async def test_low_score_in_scope_still_writes(temp_vault): regardless of match_score. Pre-1.A this returned vault_written=False.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["Python"], ["Python"], score=1.5) + state = _build_state_for_score(skills_required=["Python"], skills_matched=["Python"], score=1.5) state["in_scope"] = True state["role_family"] = "swe-backend" result = await vault_write_node(state) @@ -142,7 +153,9 @@ async def test_role_family_threaded_to_jobnote(temp_vault): """role_family from state lands in JobNote frontmatter.""" from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["LangGraph"], ["LangGraph"], score=4.0) + state = _build_state_for_score( + skills_required=["LangGraph"], skills_matched=["LangGraph"], score=4.0 + ) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -162,7 +175,7 @@ async def test_tier_resolved_from_target_companies(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -179,7 +192,7 @@ async def test_unknown_company_tier_remains_unknown(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" state["current_job"] = state["current_job"].model_copy(update={"company": "RandomCo"}) @@ -207,7 +220,7 @@ async def test_human_edited_company_tier_preserved(temp_vault): from compass.pipeline.nodes.vault_write import vault_write_node - state = _state(["MCP"], ["MCP"], score=4.5) + state = _build_state_for_score(skills_required=["MCP"], skills_matched=["MCP"], score=4.5) state["in_scope"] = True state["role_family"] = "agent-engineer" await vault_write_node(state) @@ -217,3 +230,56 @@ async def test_human_edited_company_tier_preserved(temp_vault): job_path = next((temp_vault / "jobs").glob("*Sierra*.md")) assert frontmatter.load(job_path).metadata["tier"] == "apply-now" + + +async def test_vault_write_records_approved_decision(temp_vault): + """state['human_approved'] = True -> hitl_decision='approved', hitl_at set.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.5, + human_approved=True, + human_feedback="LGTM", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "approved" + assert "hitl_at" in md and md["hitl_at"] is not None + + +async def test_vault_write_records_auto_rejected_for_low_score(temp_vault): + """Below-threshold path: hitl never interrupts; decision is 'auto_rejected'.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=2.0, human_approved=False, human_feedback=None) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "auto_rejected" + + +async def test_vault_write_records_rejected_when_human_said_no(temp_vault): + """Above-threshold path with human_approved=False = explicit reject.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score(score=4.2, human_approved=False, human_feedback="not a fit") + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "rejected" + + +async def test_vault_write_records_timed_out(temp_vault): + """Timeout-checker resume sets feedback='auto-cancelled after Xh timeout'.""" + from compass.pipeline.nodes.vault_write import vault_write_node + + state = _build_state_for_score( + score=4.2, + human_approved=False, + human_feedback="auto-cancelled after 4h timeout", + ) + await vault_write_node(state) + job_file = next((temp_vault / "jobs").glob("*.md")) + md = frontmatter.load(job_file).metadata + assert md["hitl_decision"] == "timed_out" diff --git a/tests/rag/__init__.py b/tests/rag/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/rag/conftest.py b/tests/rag/conftest.py new file mode 100644 index 0000000..e263540 --- /dev/null +++ b/tests/rag/conftest.py @@ -0,0 +1,40 @@ +"""Shared fixtures for RAG tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def temp_chroma_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Per-test Chroma path so the persistent client doesn't leak between tests.""" + p = tmp_path / "chroma" + import compass.config as cfg + + monkeypatch.setattr(cfg, "CHROMA_PATH", p) + return p + + +@pytest.fixture +def tiny_inventory(temp_vault) -> Path: + """Small 3-section inventory for fast retrieval tests.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "# Skill Inventory\n\n" + "## Python\n\n" + "Level 4. 5 years building backends and agents in Python. " + "Cisco MCP server, LangGraph pipelines, FastAPI services.\n\n" + "## LangGraph\n\n" + "Level 3. Built Compass pipeline with stateful graph, conditional " + "edges, interrupt()+AsyncSqliteSaver checkpointing for HITL.\n\n" + "## React\n\n" + "Level 1. Touched in school projects. Not a primary skill.\n", + encoding="utf-8", + ) + return inv diff --git a/tests/rag/test_indexer.py b/tests/rag/test_indexer.py new file mode 100644 index 0000000..d59c748 --- /dev/null +++ b/tests/rag/test_indexer.py @@ -0,0 +1,96 @@ +"""indexer parses skill-inventory.md by ## SkillName, embeds, persists. + +Idempotent on rebuild (upsert by stable kebab id). Uses real sentence- +transformers (model cached at session scope — ~90MB on first run only).""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_indexer_builds_one_chunk_per_section(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + assert await indexer.build_index() == 3 + + +async def test_indexer_uses_kebab_case_ids(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python", "langgraph", "react"} + + +async def test_indexer_metadata_carries_skill_and_source(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + col = indexer._collection(indexer._client()) + res = col.get(ids=["python"]) + assert res["metadatas"][0]["skill"] == "Python" + assert res["metadatas"][0]["source"] == "skill-inventory.md" + + +async def test_indexer_is_idempotent_on_rebuild(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + await indexer.build_index() + col = indexer._collection(indexer._client()) + assert col.count() == 3 + + +async def test_indexer_force_rebuild_clears_stale_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer + + await indexer.build_index() + tiny_inventory.write_text( + "# Skill Inventory\n\n## Python\n\nLevel 4.\n", + encoding="utf-8", + ) + await indexer.build_index(force_rebuild=True) + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_repairs_stale_l2_collection(temp_vault, temp_chroma_path): + """Pre-1.B.2 installs may have an L2 collection — must be dropped+recreated + so the retriever's cosine-score formula stays valid.""" + from chromadb import PersistentClient + + from compass.rag import indexer + + client = PersistentClient(path=str(temp_chroma_path)) + client.create_collection(name="skill_inventory") # default = L2 + + col = indexer._collection(client) + assert (col.metadata or {}).get("hnsw:space") == "cosine" + + +async def test_indexer_excludes_assessor_grades_heading(temp_vault, temp_chroma_path): + """Auto-generated `## Assessor-current grades` is metadata, not a skill — + must NOT land in the index or it pollutes retrieval with every skill name.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text( + "## Python\n\nLevel 4.\n\n" + "## Assessor-current grades\n\nPython: 4 / LangGraph: 3 / MCP: 5\n", + encoding="utf-8", + ) + from compass.rag import indexer + + assert await indexer.build_index() == 1 + col = indexer._collection(indexer._client()) + assert set(col.get()["ids"]) == {"python"} + + +async def test_indexer_handles_empty_inventory_gracefully(temp_vault, temp_chroma_path): + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Skill Inventory\n\nNo skills yet.\n", encoding="utf-8") + from compass.rag import indexer + + assert await indexer.build_index() == 0 diff --git a/tests/rag/test_retriever.py b/tests/rag/test_retriever.py new file mode 100644 index 0000000..b2bf87d --- /dev/null +++ b/tests/rag/test_retriever.py @@ -0,0 +1,51 @@ +"""retriever.retrieve(query, k) — top-k chunks by cosine similarity. + +Lazy-init builds the index on first call so scoring works before any +explicit indexer invocation.""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.usefixtures("embedding_model_cached") + + +async def test_retrieve_returns_top_k_relevant_chunks(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + + hits = await retriever.retrieve("Python backend agent work", k=2) + assert len(hits) == 2 + skills = [h.skill for h in hits] + assert "Python" in skills + assert any("Cisco MCP" in h.document for h in hits) + assert all(0.0 <= h.score <= 1.0 for h in hits) + + +async def test_retrieve_lazy_inits_index_on_miss(tiny_inventory, temp_chroma_path): + """If no index exists yet, retrieve() builds one before querying.""" + from compass.rag import retriever + + hits = await retriever.retrieve("LangGraph stateful pipeline", k=1) + assert len(hits) == 1 + assert hits[0].skill == "LangGraph" + + +async def test_retrieve_returns_empty_when_inventory_empty(temp_vault, temp_chroma_path): + """No sections in inventory → retrieve returns [] without crashing.""" + inv = temp_vault / "_profile" / "skill-inventory.md" + inv.parent.mkdir(parents=True, exist_ok=True) + inv.write_text("# Empty\n", encoding="utf-8") + from compass.rag import retriever + + hits = await retriever.retrieve("anything", k=5) + assert hits == [] + + +async def test_retrieve_k_larger_than_corpus_returns_all(tiny_inventory, temp_chroma_path): + from compass.rag import indexer, retriever + + await indexer.build_index() + hits = await retriever.retrieve("python or react", k=99) + assert 1 <= len(hits) <= 3 diff --git a/tests/scrapers/test_workday.py b/tests/scrapers/test_workday.py new file mode 100644 index 0000000..53c7689 --- /dev/null +++ b/tests/scrapers/test_workday.py @@ -0,0 +1,99 @@ +"""Workday scraper unit tests — mocked HTTP responses, no network.""" + +from __future__ import annotations + +from datetime import date, timedelta +from unittest.mock import patch + +import httpx +import pytest + +from compass.scrapers.workday import ( + _parse_posted_on, + _parse_slug, + _to_rawjob, + scrape_workday, +) + + +class TestParseSlug: + def test_well_formed(self): + assert _parse_slug("wd5/citi/2") == ("wd5", "citi", "2") + assert _parse_slug("wd1/wf/WellsFargoJobs") == ("wd1", "wf", "WellsFargoJobs") + + def test_missing_part_returns_none(self): + assert _parse_slug("wd5/citi") is None + assert _parse_slug("citi/2") is None + + def test_invalid_subdomain_returns_none(self): + assert _parse_slug("wrong/citi/2") is None + + +class TestParsePostedOn: + def test_today(self): + assert _parse_posted_on("Posted Today") == date.today() + + def test_yesterday(self): + assert _parse_posted_on("Posted Yesterday") == date.today() - timedelta(days=1) + + def test_n_days_ago(self): + assert _parse_posted_on("Posted 5 Days Ago") == date.today() - timedelta(days=5) + + def test_30_plus_days_ago(self): + assert _parse_posted_on("Posted 30+ Days Ago") == date.today() - timedelta(days=30) + + def test_n_months_ago(self): + # Approximate: ~30 days per month + assert _parse_posted_on("Posted 2 Months Ago") == date.today() - timedelta(days=60) + + def test_unparseable_returns_none(self): + assert _parse_posted_on(None) is None + assert _parse_posted_on("") is None + assert _parse_posted_on("Posted Recently") is None + + +class TestToRawJob: + def test_builds_rawjob_with_body(self): + raw = { + "title": "AI Engineer, LLM Suite", + "locationsText": "Plano, TX", + "postedOn": "Posted 3 Days Ago", + "externalPath": "/job/123", + } + rj = _to_rawjob("JPMorgan", raw, "Build GenAI agents.", "https://x/apply") + assert rj is not None + assert rj.company == "JPMorgan" + assert rj.title == "AI Engineer, LLM Suite" + assert rj.url == "https://x/apply" + assert rj.source == "workday" + assert rj.location == "Plano, TX" + assert rj.description == "Build GenAI agents." + assert rj.date_posted == date.today() - timedelta(days=3) + + def test_drops_empty_body(self): + """Workday detail endpoint sometimes rate-limits and we get no body. + Don't ship empty descriptions downstream — extract LLM will hallucinate.""" + rj = _to_rawjob("Citi", {"title": "Engineer"}, "", None) + assert rj is None + + def test_drops_missing_title(self): + rj = _to_rawjob("Citi", {}, "body", None) + assert rj is None + + +@pytest.mark.asyncio +async def test_scrape_workday_malformed_slug_returns_empty(): + jobs = await scrape_workday("not-a-slug") + assert jobs == [] + + +@pytest.mark.asyncio +async def test_scrape_workday_handles_http_error(): + """Network error MUST NOT raise — the pipeline keeps running.""" + + async def boom(*a, **kw): + raise httpx.ConnectError("simulated") + + with patch("httpx.AsyncClient.post", side_effect=boom): + jobs = await scrape_workday("wd5/citi/2") + assert jobs == [] diff --git a/tests/scrapers/test_workday_url_resolution.py b/tests/scrapers/test_workday_url_resolution.py new file mode 100644 index 0000000..17e64fc --- /dev/null +++ b/tests/scrapers/test_workday_url_resolution.py @@ -0,0 +1,52 @@ +"""Regression: Workday _to_rawjob used to emit relative paths like +`/job/abc` as the RawJob.url, which then mangled through url_dedup into +the invalid string `https:///job/abc`. Fixed in 2026-05-19 wave-2 review.""" + +from __future__ import annotations + +from compass.scrapers.workday import _to_rawjob + + +def test_absolute_apply_url_is_kept(): + raw = {"title": "AI Eng", "externalPath": "/job/123", "postedOn": "Posted Today"} + rj = _to_rawjob("Citi", raw, "Build agents.", "https://example.com/apply/123") + assert rj is not None + assert rj.url == "https://example.com/apply/123" + + +def test_relative_externalpath_absolutized_against_base(): + """When the detail endpoint didn't return an externalUrl (apply_url=None), + the relative `externalPath` must be combined with the tenant base URL + instead of stored bare.""" + raw = {"title": "AI Eng", "externalPath": "/job/abc", "postedOn": "Posted Today"} + rj = _to_rawjob( + "Citi", + raw, + "Build agents.", + None, + base_url="https://citi.wd5.myworkdayjobs.com", + ) + assert rj is not None + assert rj.url == "https://citi.wd5.myworkdayjobs.com/job/abc" + assert not rj.url.startswith("https:///") + + +def test_no_url_at_all_drops_job(): + """Without any URL source the job can't be dedup'd or applied to — + drop with a warning rather than persist a broken vault row.""" + raw = {"title": "AI Eng", "externalPath": "", "postedOn": "Posted Today"} + rj = _to_rawjob("Citi", raw, "Build agents.", None, base_url="https://x") + assert rj is None + + +def test_externalpath_already_absolute_kept_as_is(): + """Some Workday tenants return a fully-qualified externalPath. Don't + double-prefix the base URL.""" + raw = { + "title": "AI Eng", + "externalPath": "https://other.example.com/job/xyz", + "postedOn": "Posted Today", + } + rj = _to_rawjob("Citi", raw, "Build agents.", None, base_url="https://x") + assert rj is not None + assert rj.url == "https://other.example.com/job/xyz" diff --git a/tests/test_llm.py b/tests/test_llm.py index 15271cb..d02dd22 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -5,8 +5,13 @@ def test_get_model_id_reads_env_at_call_time(monkeypatch): """Changing the env between calls must change the resolved model id.""" + import compass.config as cfg from compass.llm import get_model_id + # `compass.config` reads EXTRACT_MODEL from env at import time. To force a + # call-time lookup via env, clear the cfg constant so the os.environ + # fallback fires. + monkeypatch.setattr(cfg, "EXTRACT_MODEL", "") monkeypatch.setenv("EXTRACT_MODEL", "google/gemini-2.5-flash") assert get_model_id("extract") == "google/gemini-2.5-flash" @@ -14,6 +19,18 @@ def test_get_model_id_reads_env_at_call_time(monkeypatch): assert get_model_id("extract") == "anthropic/claude-haiku-4-5" +def test_get_model_id_reads_config_attr(monkeypatch): + """Regression for wave-3 fix: monkeypatching `compass.config.EXTRACT_MODEL` + must take effect — pre-fix the resolver bypassed cfg and read os.environ + directly, so cfg patches in tests silently used the real env value.""" + import compass.config as cfg + from compass.llm import get_model_id + + monkeypatch.setattr(cfg, "EXTRACT_MODEL", "test-only-model") + monkeypatch.delenv("EXTRACT_MODEL", raising=False) + assert get_model_id("extract") == "test-only-model" + + def test_get_model_id_unknown_node_raises(): from compass.llm import get_model_id @@ -22,8 +39,13 @@ def test_get_model_id_unknown_node_raises(): def test_get_model_id_missing_env_raises(monkeypatch): + import compass.config as cfg from compass.llm import get_model_id + # Must clear BOTH cfg constant and env var — the resolver falls back to + # os.environ if cfg is empty, so leaving cfg with its imported value + # would mask the env deletion. + monkeypatch.setattr(cfg, "EXTRACT_MODEL", "") monkeypatch.delenv("EXTRACT_MODEL", raising=False) with pytest.raises(ValueError, match="no model configured"): get_model_id("extract") diff --git a/tests/test_mcp_get_profile_traversal.py b/tests/test_mcp_get_profile_traversal.py new file mode 100644 index 0000000..ea4357d --- /dev/null +++ b/tests/test_mcp_get_profile_traversal.py @@ -0,0 +1,33 @@ +"""Regression — `get_profile` MCP tool must NOT leak files outside `_profile/`. + +Pre-fix (2026-05-19 wave 3) `get_profile("../.env")` resolved to the vault root +and returned the contents of any sibling .env-like file. Same threat class as +the `generate_cover_letter` path-traversal fix from wave 1. +""" + +from __future__ import annotations + + +def test_path_traversal_rejected(temp_vault): + from compass.mcp_server.server import get_profile + + out = get_profile("../.env") + assert "invalid section name" in out.lower() or "must be inside" in out.lower() + + +def test_absolute_path_rejected(temp_vault): + from compass.mcp_server.server import get_profile + + out = get_profile("/etc/passwd") + assert "invalid section name" in out.lower() or "must be inside" in out.lower() + + +def test_normal_section_works(temp_vault): + """The path-traversal fix shouldn't break the happy path. Write a known + file explicitly to avoid order-dependence on the conftest seed (other + tests may delete _profile/* files in the same session).""" + from compass.mcp_server.server import get_profile + + (temp_vault / "_profile" / "resume.md").write_text("# Resume\n\nReal content here.\n") + out = get_profile("resume") + assert "Real content here" in out diff --git a/tests/test_mcp_path_traversal.py b/tests/test_mcp_path_traversal.py new file mode 100644 index 0000000..2b2408c --- /dev/null +++ b/tests/test_mcp_path_traversal.py @@ -0,0 +1,36 @@ +"""Regression test for the path-traversal fix in generate_cover_letter — added +2026-05-19 after the adversarial review found that `job_filename` was joined +to VAULT_PATH without validation.""" + +from __future__ import annotations + +import pytest + + +@pytest.mark.asyncio +async def test_generate_cover_letter_rejects_path_traversal(temp_vault): + from compass.mcp_server.server import generate_cover_letter + + out = await generate_cover_letter("../_profile/resume.md") + assert "error" in out, "path traversal must be rejected" + assert "inside jobs/" in out["error"] + + +@pytest.mark.asyncio +async def test_generate_cover_letter_rejects_absolute_path(temp_vault): + from compass.mcp_server.server import generate_cover_letter + + out = await generate_cover_letter("/etc/passwd") + assert "error" in out + + +@pytest.mark.asyncio +async def test_generate_cover_letter_accepts_valid_filename(temp_vault): + """A real JobNote filename inside jobs/ — even if the file doesn't exist, + the path-traversal guard should pass; only the not-found error fires.""" + from compass.mcp_server.server import generate_cover_letter + + out = await generate_cover_letter("2026-05-19-Sierra-Engineer-abc.md") + # Path passes the traversal check but file doesn't exist + assert "error" in out + assert "JobNote not found" in out["error"] diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 978ceca..41fe980 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -27,7 +27,7 @@ async def fake_extract(jd_text): summary="Build agents.", ) - async def fake_score(req, profile_text): + async def fake_score(req, profile_text, job=None): return JobScore( score=4.0, reasoning="Strong Python + LangGraph evidence in profile.", diff --git a/tests/vault/test_learning_bridge_traversal.py b/tests/vault/test_learning_bridge_traversal.py new file mode 100644 index 0000000..a348500 --- /dev/null +++ b/tests/vault/test_learning_bridge_traversal.py @@ -0,0 +1,86 @@ +"""Regression — `learning_bridge.resolve()` must NOT escape LEARNING_VAULT_PATH. + +Pre-fix (2026-05-19 wave 3) a URI like `learning-vault://../../.env` resolved +through the bare `LEARNING_VAULT_PATH / rest` join (Path doesn't normalize `..` +components) and would return the contents of the secret file. Verified live +that the leak was real before the fix. +""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def isolated_learning_vault(tmp_path, monkeypatch): + lv = tmp_path / "learning-vault" + lv.mkdir() + (lv / "real.md").write_text("# Real evidence\n\nSafe content.") + # Place a sensitive sibling we'd want to NOT leak + secret = tmp_path / "secret.env" + secret.write_text("OPENROUTER_API_KEY=sk-leaked-DO-NOT-RETURN") + + import compass.config as cfg + import compass.vault.learning_bridge as lb + + monkeypatch.setattr(cfg, "LEARNING_VAULT_PATH", lv) + monkeypatch.setattr(lb, "LEARNING_VAULT_PATH", lv) + return lv + + +def test_normal_uri_resolves_within_vault(isolated_learning_vault): + from compass.vault.learning_bridge import resolve + + result = resolve("learning-vault://real.md") + assert result is not None + assert "Safe content" in result.snippet + + +def test_dot_dot_uri_rejected_returns_none(isolated_learning_vault): + """`../secret.env` escapes the vault root — `_parse_uri` raises ValueError + which `resolve` catches and returns None. The user-observable contract is + "no file contents leaked." Pre-fix, the .env contents WERE returned.""" + from compass.vault.learning_bridge import resolve + + # The critical assertion: secret content does NOT appear in any returned object + result = resolve("learning-vault://../secret.env") + assert result is None, "path-traversal must not return an artifact" + + +def test_dot_dot_uri_raises_at_parse(isolated_learning_vault): + """The lower-level `_parse_uri` raises ValueError on traversal — that's + the security guard. `resolve` catches it as part of its existing + None-on-error contract.""" + from compass.vault.learning_bridge import _parse_uri + + with pytest.raises(ValueError, match="resolves outside the vault root"): + _parse_uri("learning-vault://../secret.env") + + +def test_deep_dot_dot_uri_blocked(isolated_learning_vault): + from compass.vault.learning_bridge import _parse_uri, resolve + + assert resolve("learning-vault://../../etc/passwd") is None + with pytest.raises(ValueError): + _parse_uri("learning-vault://../../etc/passwd") + + +def test_absolute_path_attempt_blocked(isolated_learning_vault): + """An absolute path in the URI would resolve relative to / not LEARNING_VAULT_PATH.""" + from compass.vault.learning_bridge import _parse_uri, resolve + + assert resolve("learning-vault:///etc/passwd") is None + with pytest.raises(ValueError): + _parse_uri("learning-vault:///etc/passwd") + + +def test_url_encoded_traversal_still_blocked(isolated_learning_vault): + """Path components with literal `..` are caught by resolve+relative_to; + URL-encoded variants like %2e%2e/ would bypass that if we decoded. We + don't decode, so they're treated as literal filenames — not vulnerable + to encoding tricks. This test pins that behavior.""" + from compass.vault.learning_bridge import resolve + + # A literal "%2e%2e" filename inside the vault wouldn't exist; expect None. + result = resolve("learning-vault://%2e%2e/secret.env") + assert result is None diff --git a/tests/vault/test_normalize_defensive.py b/tests/vault/test_normalize_defensive.py new file mode 100644 index 0000000..a857617 --- /dev/null +++ b/tests/vault/test_normalize_defensive.py @@ -0,0 +1,35 @@ +"""Regression — `taxonomy.normalize` accepts defensively-bad inputs without +crashing. A JobNote with a YAML list-entry written as `- ` parses as `[None]`; +without the guard, gap_aggregator.aggregate() crashed the whole regenerate() +call on a single malformed JobNote. + +Wave-3 adversarial review, 2026-05-19. +""" + +from __future__ import annotations + +from compass.vault.taxonomy import normalize + + +def test_none_returns_none(): + assert normalize(None) is None # type: ignore[arg-type] + + +def test_empty_string_returns_none(): + assert normalize("") is None + + +def test_whitespace_string_returns_none(): + assert normalize(" ") is None + + +def test_non_str_returns_none(): + """Defensive: a dict/list/int slipping through YAML parsing shouldn't crash.""" + assert normalize(123) is None # type: ignore[arg-type] + assert normalize({}) is None # type: ignore[arg-type] + assert normalize([]) is None # type: ignore[arg-type] + + +def test_valid_skill_still_works(): + """Confirm the defensive guard didn't break the happy path.""" + assert normalize("Python") == "Python" diff --git a/tests/vault/test_target_companies_aliases.py b/tests/vault/test_target_companies_aliases.py new file mode 100644 index 0000000..617d443 --- /dev/null +++ b/tests/vault/test_target_companies_aliases.py @@ -0,0 +1,93 @@ +"""Regression tests for aliases + mtime-aware reload — both surfaced by the +2026-05-19 adversarial review. + +Aliases bug: JPM's Workday/Oracle tenant slug is "jpmc" but the YAML lists +the company as "JPMorgan". Pre-fix, `get_tier("JPMC")` returned "unknown" +because no substring match. With aliases, `JPMC` → "JPMorgan" → apply-now. + +Mtime-aware reload bug: long-running MCP server cached the YAML at startup; +edits to the YAML file were invisible until process restart. +""" + +from __future__ import annotations + +import time + +import pytest + +_BASE_YAML = """ +schema_version: 1 +companies: + - company: JPMorgan + aliases: [JPMC, JPM, JPMorgan Chase] + tier: apply-now + section: banks + ats: {provider: manual, slug: oracle-cloud} + interview_difficulty: hackerrank +""" + + +@pytest.fixture +def yaml_vault(temp_vault): + (temp_vault / "_profile" / "target-companies.yaml").write_text(_BASE_YAML, encoding="utf-8") + import compass.vault.target_companies as tc + + tc.refresh_yaml() + yield temp_vault + tc.refresh_yaml() + + +def test_alias_resolves_to_primary_tier(yaml_vault): + """Every alias in the aliases list should resolve to the same tier as the + primary name. This fixes the JPMC↔JPMorgan slug mismatch.""" + from compass.vault.target_companies import get_tier + + assert get_tier("JPMorgan") == "apply-now" + assert get_tier("JPMC") == "apply-now" + assert get_tier("JPM") == "apply-now" + assert get_tier("JPMorgan Chase") == "apply-now" + # Lowercase + whitespace shouldn't matter + assert get_tier("jpmc") == "apply-now" + assert get_tier("jp morgan chase") == "apply-now" + + +def test_alias_resolves_to_primary_meta(yaml_vault): + """get_company_meta should also resolve aliases to the primary entry.""" + from compass.vault.target_companies import get_company_meta + + assert get_company_meta("JPMC")["company"] == "JPMorgan" + assert get_company_meta("JPM")["company"] == "JPMorgan" + + +def test_yaml_reload_picks_up_disk_edits(yaml_vault): + """Long-running MCP server should see YAML edits within one accessor call.""" + from compass.vault.target_companies import get_tier + + assert get_tier("JPMorgan") == "apply-now" + + # Edit the YAML on disk — flip tier + yaml_path = yaml_vault / "_profile" / "target-companies.yaml" + edited = yaml_path.read_text().replace("tier: apply-now", "tier: opportunistic") + # Tiny sleep so the mtime resolution catches the change (some filesystems + # only have second-level mtime granularity) + time.sleep(0.01) + yaml_path.write_text(edited) + # Force mtime to definitely be newer + import os + + now = time.time() + os.utime(yaml_path, (now, now + 1)) + + # No explicit refresh — accessor should auto-reload via mtime check + assert get_tier("JPMorgan") == "opportunistic" + + +def test_yaml_deletion_clears_cache(yaml_vault): + """If the YAML file is removed (vault wipe), accessors should return + fallback values, not stale cache.""" + from compass.vault.target_companies import get_company_meta + + assert get_company_meta("JPMorgan") is not None + + (yaml_vault / "_profile" / "target-companies.yaml").unlink() + assert get_company_meta("JPMorgan") is None diff --git a/tests/vault/test_target_companies_yaml.py b/tests/vault/test_target_companies_yaml.py new file mode 100644 index 0000000..de7e746 --- /dev/null +++ b/tests/vault/test_target_companies_yaml.py @@ -0,0 +1,103 @@ +"""YAML-driven per-company metadata accessors. Coexist with the legacy +markdown-based `get_tier`.""" + +from __future__ import annotations + +import pytest + +_SAMPLE_YAML = """ +schema_version: 1 +companies: + - company: Databricks + tier: apply-now + section: data-ai-infra + ats: {provider: greenhouse, slug: databricks} + geos: [NYC, SF] + interview_difficulty: lc-medium-hard + notes: "Mosaic AI Agent Framework" + + - company: New Relic + tier: apply-now + section: observability-aiops + ats: {provider: greenhouse, slug: newrelic} + geos: [SF, Remote] + interview_difficulty: lc-medium + notes: "AI observability" + + - company: TypoCo + tier: apply-now + ats: {provider: greenhouse, slug: typoco} + interview_difficulty: super-easy # invalid Literal — must collapse to "unknown" +""" + + +@pytest.fixture +def yaml_vault(temp_vault, monkeypatch): + (temp_vault / "_profile" / "target-companies.yaml").write_text(_SAMPLE_YAML, encoding="utf-8") + import compass.vault.target_companies as tc + + tc.refresh_yaml() + yield temp_vault + # Reset module-level cache between tests + tc.refresh_yaml() + + +def test_get_company_meta_returns_full_entry(yaml_vault): + from compass.vault.target_companies import get_company_meta + + meta = get_company_meta("Databricks") + assert meta is not None + assert meta["tier"] == "apply-now" + assert meta["ats"]["slug"] == "databricks" + assert "NYC" in meta["geos"] + + +def test_get_interview_difficulty_known_value(yaml_vault): + from compass.vault.target_companies import get_interview_difficulty + + assert get_interview_difficulty("Databricks") == "lc-medium-hard" + assert get_interview_difficulty("New Relic") == "lc-medium" + + +def test_get_interview_difficulty_invalid_collapses_to_unknown(yaml_vault): + """Hand-edited YAML may carry typos. Accessor must collapse to "unknown" + so JobNote Literal validation never fails on human edits.""" + from compass.vault.target_companies import get_interview_difficulty + + assert get_interview_difficulty("TypoCo") == "unknown" + + +def test_get_interview_difficulty_unknown_company(yaml_vault): + from compass.vault.target_companies import get_interview_difficulty + + assert get_interview_difficulty("CompanyNotInYAML") == "unknown" + + +def test_get_ats_returns_tuple(yaml_vault): + from compass.vault.target_companies import get_ats + + assert get_ats("Databricks") == ("greenhouse", "databricks") + + +def test_get_ats_returns_none_for_unknown_company(yaml_vault): + from compass.vault.target_companies import get_ats + + assert get_ats("RandomCo") is None + + +def test_list_yaml_companies_tier_filter(yaml_vault): + from compass.vault.target_companies import list_yaml_companies + + assert len(list_yaml_companies()) == 3 + assert len(list_yaml_companies(tier_filter="apply-now")) == 3 + assert len(list_yaml_companies(tier_filter="stretch")) == 0 + + +def test_bidirectional_substring_match(yaml_vault): + """`get_company_meta("Databricks Inc")` should still find the "Databricks" + entry — same substring fallback as `get_tier`.""" + from compass.vault.target_companies import get_company_meta + + meta = get_company_meta("Databricks Inc") + assert meta is not None + assert meta["company"] == "Databricks" diff --git a/tests/vault/test_url_dedup.py b/tests/vault/test_url_dedup.py new file mode 100644 index 0000000..d851722 --- /dev/null +++ b/tests/vault/test_url_dedup.py @@ -0,0 +1,75 @@ +"""URL normalization regression tests — every case found in the adversarial +review of 2026-05-19 must collapse to the same canonical form.""" + +from __future__ import annotations + +from compass.vault.url_dedup import normalize_url + + +class TestNormalization: + """Cosmetic variants of the same JD URL must normalize to one string.""" + + def test_trailing_slash(self): + assert normalize_url("https://jobs.ashbyhq.com/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc/" + ) + + def test_utm_tracking_params_stripped(self): + assert normalize_url("https://jobs.ashbyhq.com/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc?utm_source=google&utm_medium=cpc" + ) + + def test_gclid_stripped(self): + assert normalize_url("https://jobs.ashbyhq.com/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc?gclid=ABCDEF" + ) + + def test_http_collapses_to_https(self): + assert normalize_url("http://jobs.ashbyhq.com/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc" + ) + + def test_host_case_insensitive(self): + assert normalize_url("https://JOBS.ASHBYHQ.COM/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc" + ) + + def test_fragment_dropped(self): + assert normalize_url("https://jobs.ashbyhq.com/sierra/abc#apply") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc" + ) + + def test_default_ports_dropped(self): + assert normalize_url("https://jobs.ashbyhq.com:443/sierra/abc") == normalize_url( + "https://jobs.ashbyhq.com/sierra/abc" + ) + + def test_non_default_port_preserved(self): + """If the URL really does use a non-standard port, keep it.""" + assert ":8080" in normalize_url("https://jobs.example.com:8080/role") + + def test_non_tracking_query_preserved(self): + """Tracking-only params are stripped; meaningful ones (job_id, etc.) stay.""" + out = normalize_url("https://jobs.example.com/list?job_id=42&utm_source=x") + assert "job_id=42" in out + assert "utm_source" not in out + + def test_query_params_sorted(self): + """Two URLs with the same params in different order canonicalize together.""" + assert normalize_url("https://example.com/?b=2&a=1") == normalize_url( + "https://example.com/?a=1&b=2" + ) + + def test_path_case_preserved(self): + """Servers MAY treat /Foo and /foo differently — don't fold.""" + assert normalize_url("https://example.com/Foo") != normalize_url("https://example.com/foo") + + def test_empty_url_returns_empty(self): + assert normalize_url("") == "" + + def test_malformed_url_returns_unchanged(self): + """If urlsplit raises, degrade to per-string matching for that one URL.""" + # Not strictly malformed but unusual: + out = normalize_url("notaurl") + # Should not crash and should produce some deterministic output + assert isinstance(out, str) diff --git a/tests/vault/test_writer.py b/tests/vault/test_writer.py index e32ce67..659d55b 100644 --- a/tests/vault/test_writer.py +++ b/tests/vault/test_writer.py @@ -59,6 +59,39 @@ def test_write_job_note_sanitizes_filename(temp_vault): assert "?" not in path.name +def test_write_job_note_strips_html_from_full_jd(temp_vault): + """Belt-and-suspenders: if a scraper leaks HTML, the writer normalizes it + before persisting. Archived JobNotes showed `</span></strong></p></div>` + cruft from a historical scrape path that bypassed `_strip_html` — this + guards the writer-side boundary regardless of upstream behavior.""" + from compass.vault.writer import write_job_note + + leaky = ( + "<p><strong>About Databricks</strong></p>" + '<p><span style="font-family: arial;">Databricks is the data and AI company.</span></p>' + "</div>" + ) + path = write_job_note(_make_job_note(), full_description=leaky) + body = path.read_text() + assert "<p>" not in body + assert "</span>" not in body + assert "</div>" not in body + assert "Databricks is the data and AI company." in body + assert "About Databricks" in body + + +def test_write_job_note_preserves_plain_text_with_angle_brackets(temp_vault): + """A JD containing literal `<` (e.g. `<200ms latency`) must pass through + untouched — only visibly-HTML inputs get stripped.""" + from compass.vault.writer import write_job_note + + plain = "Build agents with <200ms latency. Use Python 3.12 (>=3.12 OK)." + path = write_job_note(_make_job_note(), full_description=plain) + body = path.read_text() + assert "<200ms latency" in body + assert ">=3.12" in body + + def test_write_job_note_persists_full_jd_when_provided(temp_vault): """Regression: pre-fix JobNotes only carried the LLM-generated summary; the raw JD was discarded after extract+score. Humans then couldn't verify what @@ -83,6 +116,70 @@ def test_write_job_note_omits_full_jd_section_when_not_provided(temp_vault): assert "## Full JD" not in path.read_text() +def test_jobnote_body_has_skills_section_with_wikilinks(temp_vault): + """Day 1 Obsidian P1: JobNote body renders a `## Skills` block with + wikilinks so the graph view shows JobNote → SkillNote edges. Section + sits between the LLM summary and `## Full JD`.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["Python", "LangGraph"], + skills_nice_to_have=["FastAPI"], + skills_matched=["Python"], + skills_missing=["LangGraph"], + ) + path = write_job_note(note, full_description="raw jd text here") + body = path.read_text() + assert "## Skills" in body + assert "[[Python]]" in body + assert "[[LangGraph]]" in body + assert "[[FastAPI]]" in body + assert "**Required:**" in body + assert "**Nice to have:**" in body + assert "**Matched:**" in body + assert "**Missing:**" in body + # ## Skills must appear before ## Full JD + assert body.index("## Skills") < body.index("## Full JD") + + +def test_jobnote_body_omits_empty_skill_categories(temp_vault): + """Empty categories are omitted entirely — no '(none)' placeholder.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["Python"], + skills_nice_to_have=[], + skills_matched=[], + skills_missing=["Python"], + ) + path = write_job_note(note) + body = path.read_text() + assert "**Nice to have:**" not in body + assert "**Matched:**" not in body + assert "**Required:** [[Python]]" in body + assert "**Missing:** [[Python]]" in body + + +def test_jobnote_skill_wikilink_aliases_unsafe_filenames(temp_vault): + """Skills with spaces or punctuation resolve to a safe-segment filename; + the wikilink must point at the actual file via alias form so the link + resolves AND the display matches the user-facing skill name.""" + from compass.vault.writer import write_job_note + + note = _make_job_note( + skills_required=["AWS Bedrock", "C++"], + skills_matched=[], + skills_missing=[], + skills_nice_to_have=[], + ) + path = write_job_note(note) + body = path.read_text() + assert "[[AWS_Bedrock|AWS Bedrock]]" in body + # `_safe_segment` collapses non-word runs to `_` then strips trailing `_`, + # so "C++" becomes the file "C.md" — alias keeps the original display. + assert "[[C|C++]]" in body + + def test_write_job_note_idempotent_on_duplicate_url(temp_vault): """Writing the same URL twice should overwrite the same file, not create a second.""" from compass.vault.writer import write_job_note @@ -96,29 +193,6 @@ def test_write_job_note_idempotent_on_duplicate_url(temp_vault): assert loaded.metadata["match_score"] == 4.5 -def test_update_skill_note_increments_counter(temp_vault): - from compass.vault.writer import update_skill_note - - skill_path = temp_vault / "skills" / "LangGraph.md" - skill_path.write_text( - "---\ntype: skill\nskill: LangGraph\ncategory: agent-framework\nappears_in_jobs: 5\n---\n# LangGraph\n" - ) - update_skill_note("LangGraph", "https://example.com/jobs/x") - loaded = frontmatter.load(skill_path) - assert loaded.metadata["appears_in_jobs"] == 6 - - -def test_update_skill_note_creates_if_missing(temp_vault): - from compass.vault.writer import update_skill_note - - update_skill_note("Python", "https://example.com/jobs/x") - skill_path = temp_vault / "skills" / "Python.md" - assert skill_path.exists() - loaded = frontmatter.load(skill_path) - assert loaded.metadata["skill"] == "Python" - assert loaded.metadata["appears_in_jobs"] == 1 - - def test_write_company_note_creates_file(temp_vault): from compass.vault.schemas import CompanyNote from compass.vault.writer import write_company_note