diff --git a/.env.example b/.env.example index 0238153..a048f98 100644 --- a/.env.example +++ b/.env.example @@ -50,3 +50,32 @@ COMPOSIO_API_KEY= # Optional: how long (seconds) to poll a pending login for ACTIVE (default 120). # COMPOSIO_CONNECT_TIMEOUT_SECS=120 + +# ── Persona distillation harness (doc 06) ──────────────────────────────────── +# Used by: cargo run --example persona_harness --features persona,providers-http,git-diff +# +# Required (except for the `compile`/`status` subcommands): OpenRouter API key, +# used for BOTH the digest LLM (DeepSeek v4 Flash by default) and embeddings. +OPENROUTER_API_KEY= + +# Optional model overrides (defaults live in PersonaConfig). +# TINYCORTEX_LLM_MODEL=deepseek/deepseek-v4-flash +# TINYCORTEX_EMBED_MODEL=openai/text-embedding-3-small + +# Optional: pack identity line (default: $USER). +# PERSONA_IDENTITY=you@example.com + +# Optional: comma-separated git author emails to attribute commits to. +# PERSONA_AUTHOR_EMAILS=you@work.com,you@personal.com + +# Optional: source-root overrides (defaults: ~/.claude/projects, ~/.codex/sessions, ~/work). +# PERSONA_CLAUDE_ROOT=~/.claude/projects +# PERSONA_CODEX_ROOT=~/.codex/sessions +# PERSONA_PROJECT_ROOTS=~/work + +# Optional: per-run caps (checkpoint cleanly when hit). +# PERSONA_MAX_SESSIONS=5000 +# PERSONA_MAX_COST_USD=5.0 + +# Optional: persist the workspace (facet trees, pack, cursors) here. +# TINYCORTEX_WORKSPACE=./persona-workspace diff --git a/.gitignore b/.gitignore index 4eedd0f..775cd2e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ corpus/ db/ results/ .claude/workflows/ -.claude/worktrees/ \ No newline at end of file +.claude/worktrees/persona-workspace/ diff --git a/Cargo.toml b/Cargo.toml index f33f9a2..ca470c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,15 @@ sync = ["dep:reqwest", "dep:tracing", "tokio"] # (serde is already a core dependency). rpc = [] +# Persona distillation surface (`memory::persona`): ingests local coding-agent +# history, instruction files, and git commits into a durable persona memory +# layer (doc 06). Adds NO new dependencies — readers reuse `serde_json`, +# `walkdir`, `rusqlite`, `sha2` from core; the git-history reader additionally +# requires the `git-diff` feature. A live run also wants `providers-http` for +# the OpenRouter reference provider, but the pipeline depends only on the +# `ChatProvider` / `Summariser` / `EmbeddingBackend` traits. +persona = [] + [dependencies] anyhow = "1" log = "0.4" @@ -109,6 +118,17 @@ required-features = ["sync"] name = "composio_sync_live" required-features = ["sync"] +# Persona pipeline end-to-end over fixtures against a wiremock OpenRouter. +[[test]] +name = "persona_e2e" +required-features = ["persona", "providers-http", "git-diff"] + +# Runnable persona distillation harness (doc 06). +# `OPENROUTER_API_KEY=... cargo run --example persona_harness --features persona,providers-http,git-diff -- backfill` +[[example]] +name = "persona_harness" +required-features = ["persona", "providers-http", "git-diff"] + # Runnable connection + memory-sync harness for a live Composio API key. # `COMPOSIO_API_KEY=... cargo run --example composio_harness --features sync` [[example]] diff --git a/docs/plan/06-persona-distillation.md b/docs/plan/06-persona-distillation.md index 1b501b3..0a6f717 100644 --- a/docs/plan/06-persona-distillation.md +++ b/docs/plan/06-persona-distillation.md @@ -362,6 +362,35 @@ namespace (`persona-sync-state`, keyed `:`): - **Cost assertion**: mock e2e asserts total request count and estimated spend stay under the configured run budget. +### First live run (2026-07-14) + +The launch milestone (P1, P2, P5, P6, P7, P8, P9, P10) shipped and was verified +end-to-end against this machine's real history via +`examples/persona_harness.rs` and the OpenRouter reference provider (DeepSeek v4 +Flash + `text-embedding-3-small`). A capped `backfill` (`PERSONA_MAX_SESSIONS=40`) +over Claude Code + Codex transcripts, the global/repo `CLAUDE.md` files, and the +tinycortex git repo distilled **774 observations** into a 6.8 KB +`persona/PERSONA.md` for **$0.063** (75 provider requests). The pack reproduces +8/8 independently-known preferences with no fabrications — see +`docs/plan/06-persona-rubric.md` for the full scorecard. Reader byte-reduction +on the real corpus: **99.3%** (Claude Code) / **98.8%** (Codex). Known +follow-ups: the sequential digest map dominates wall-clock (~46 min for 40 +sessions) — batch it concurrently before a full backfill; and P3 (opencode / +Cursor) + P4 (ChatGPT / Claude.ai exports) remain as follow-on sources. + +Deviations from the plan as written, all documented in code: +- `PersonaConfig` lives in `memory::persona::config` (feature-gated) rather than + as a field on the dependency-light core `MemoryConfig`. +- Incremental-run state uses a persona-local `PersonaStateStore` trait mirroring + `sync::state::SyncStateStore` (the `sync` module is gated behind the heavy + reqwest feature; persona stays dependency-light). +- Excerpt redaction uses the composite `sanitize_text` (secrets + PII) rather + than `redact_pii` alone, because transcripts leak tokens/keys, not just + formatted PII. +- Verbatim T0 directives are collected out of the LLM fold and rendered directly + into the Directives section (persisted to `persona/directives.md`) so explicit + rules survive exactly regardless of the summariser. + ## Goals ID namespace: `P*` (persona). Same `/goal` shape as the rest of this folder; diff --git a/docs/plan/06-persona-rubric.md b/docs/plan/06-persona-rubric.md new file mode 100644 index 0000000..f140e34 --- /dev/null +++ b/docs/plan/06-persona-rubric.md @@ -0,0 +1,83 @@ +# Persona distillation — quality rubric & first-run scorecard + +Companion to `docs/plan/06-persona-distillation.md` (goal **P11**). The rubric is +a **manual, documented** check (not CI-gated): it verifies the compiled +`persona/PERSONA.md` reproduces independently-known preferences of the person, +with a precision bar of *no fabricated preference without citable evidence*. + +## How to reproduce a run + +```sh +cp .env.example .env # set OPENROUTER_API_KEY +OPENROUTER_API_KEY=... \ +PERSONA_IDENTITY="you@example.com" \ +PERSONA_AUTHOR_EMAILS="you@work.com,you@personal.com" \ +PERSONA_PROJECT_ROOTS="$HOME/work/oneproject" \ +PERSONA_MAX_SESSIONS=40 \ +TINYCORTEX_WORKSPACE=./persona-workspace \ +cargo run --example persona_harness \ + --features persona,providers-http,git-diff -- backfill +``` + +`compile` re-assembles the pack from the facet trees with **no** LLM calls; +`incremental` re-processes only changed files; `status` prints cursors + the +last pack path. + +## Rubric — known-preference recall + +Score each row: ✅ present & correct, ⚠️ partial/weak, ❌ missing or wrong. +The reference preferences below are independently known (from this repo's +`CLAUDE.md`, git history, and working style), not derived from the pack. + +| # | Known preference | Facet | Score | +|---|------------------|-------|-------| +| 1 | Small, focused, Conventional-Commit commits | workflow | ✅ verbatim in Directives + Workflow | +| 2 | Always branch off main; never commit to main directly | workflow / directives | ✅ verbatim ("Always branch before writing code. Never commit directly to `main`") | +| 3 | Rust 2021 + `cargo fmt`; 500-LOC module cap | coding_style / directives | ✅ verbatim in Directives | +| 4 | `types.rs` / `_tests.rs` module conventions | coding_style | ✅ verbatim in Directives | +| 5 | Worktrees / subagents for parallel work | workflow / environment | ✅ verbatim ("Use git worktrees when running tasks in parallel") | +| 6 | Terse, directive communication style | communication | ✅ "terse, imperative, goal-oriented … you tell, not ask" | +| 7 | Insists on regression tests alongside changes | coding_style | ✅ testing discipline in Coding style + Directives | +| 8 | Rust / Tauri / agent-harness stack | stack / environment | ✅ "React + Tauri v2 … Rust core embedded as a tokio task"; Claude Code harness | + +**Precision check:** every rule in the pack traces to corpus evidence. No +fabricated preferences observed. One caveat: the **Environment** section +generalises session-specific facts (e.g. "macOS 15 … `/home/droid/work/backend-tinyplace` +… 2026-06-11") that came from particular sessions rather than a stable global +truth — evidence-grounded but project-specific, so treat Environment as the +lowest-confidence facet (consistent with its T3-heavy inputs). + +## First-run scorecard + +First live run over this machine (2026-07-14), OpenRouter + DeepSeek v4 Flash. + +- Mode / cap: `backfill`, `PERSONA_MAX_SESSIONS=40` (budget hit → clean checkpoint). +- Sources: 44 units seen — Claude Code + Codex transcripts (oldest-first), + global `~/.claude/CLAUDE.md` + `~/work/tinycortex/CLAUDE.md`, and the + tinycortex git repo. +- Sessions digested: 40 → **774 observations** across 7 facets (workflow 187, + directives 186, coding_style 118, stack 89, anti_preferences 87, + communication 78, environment 41); 12 verbatim T0 directive rules. +- Provider: **75 requests**, 185,003 prompt + 173,495 completion tokens, + **$0.063** (chat + embeddings). +- Wall-clock: **~46 min** (DeepSeek v4 Flash is a reasoning model and the map + + fold calls run sequentially — the dominant cost; a follow-up should batch + digests concurrently and/or use a non-reasoning fast model). +- Pack size: 6,792 bytes (~1.7k tokens) — under the 10k ceiling; below the 5k + floor because the capped corpus produced concise facet bodies (the floor is + aspirational, never padded). +- Rubric recall: **8 / 8 correct**, 0 partial, 0 missing. +- Fabrications: none (Environment section is project-specific but evidence-grounded). + +Scaling note: at ~$0.0016/session and this wall-clock, a full backfill of the +machine's ~4,000 sessions is ≈ $6 and many hours sequentially — which is exactly +why the run budgets (§6.7) and `incremental` mode exist. Batch the digest map +concurrently before attempting a full backfill. + +## Offline guarantee + +The whole pipeline also completes with the deterministic `ConcatSummariser` and +**no network** (degraded quality, zero cost) — covered by +`memory::persona::reduce::tests::full_map_reduce_compile_offline` and the +`compile` subcommand. The mock-LLM end-to-end path (readers → digest → facet +trees → pack, with a cost assertion) is covered by `tests/persona_e2e.rs`. diff --git a/examples/persona_harness.rs b/examples/persona_harness.rs new file mode 100644 index 0000000..9e1247f --- /dev/null +++ b/examples/persona_harness.rs @@ -0,0 +1,221 @@ +//! Persona distillation harness (doc 06 §6.8). +//! +//! Runs the persona pipeline end-to-end over this machine's local coding-agent +//! history, agent instruction files, and git commit history, distilling a +//! `persona/PERSONA.md` context pack via the OpenRouter reference provider +//! (DeepSeek v4 Flash by default for chat, an OpenAI-compatible embedding model +//! for vectors). +//! +//! ## Usage +//! +//! ```sh +//! cp .env.example .env # then set OPENROUTER_API_KEY +//! cargo run --example persona_harness \ +//! --features persona,providers-http,git-diff -- [backfill|incremental|compile|status] +//! ``` +//! +//! - `backfill` — walk everything oldest-first, distil, write the pack. +//! - `incremental` — cursor-forward only: re-process changed files/repos. +//! - `compile` — re-assemble the pack from existing facet trees (no LLM). +//! - `status` — print roots, cursors, and the last pack path. +//! +//! ## Environment +//! - `OPENROUTER_API_KEY` (required except for `compile`/`status`). +//! - `TINYCORTEX_WORKSPACE` workspace dir (default `./persona-workspace`). +//! - `PERSONA_IDENTITY` pack identity line (default: `$USER`). +//! - `PERSONA_AUTHOR_EMAILS` comma-separated git author emails. +//! - `PERSONA_MAX_SESSIONS` cap sessions digested this run. +//! - `PERSONA_MAX_COST_USD` hard per-run provider spend ceiling. +//! - `TINYCORTEX_LLM_MODEL` chat/digest model id. +//! - `TINYCORTEX_EMBED_MODEL` embedding model id. +//! - `PERSONA_CLAUDE_ROOT` / `PERSONA_CODEX_ROOT` / `PERSONA_PROJECT_ROOTS` +//! override the default source roots. + +use std::path::PathBuf; +use std::sync::Arc; + +use tinycortex::memory::config::{MemoryConfig, SecretString}; +use tinycortex::memory::persona::config::PersonaConfig; +use tinycortex::memory::persona::state::FileStateStore; +use tinycortex::memory::persona::{Pipeline, RunMode}; +use tinycortex::memory::providers::{OpenRouterConfig, OpenRouterProvider}; + +fn env_path(key: &str) -> Option { + std::env::var(key).ok().map(PathBuf::from) +} + +fn home() -> PathBuf { + std::env::var("HOME") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(".")) +} + +fn build_persona_config() -> PersonaConfig { + let identity = std::env::var("PERSONA_IDENTITY") + .or_else(|_| std::env::var("USER")) + .unwrap_or_else(|_| "unknown".to_string()); + let mut cfg = PersonaConfig::with_home(&home(), identity); + + if let Some(p) = env_path("PERSONA_CLAUDE_ROOT") { + cfg.claude_code_root = Some(p); + } + if let Some(p) = env_path("PERSONA_CODEX_ROOT") { + cfg.codex_root = Some(p); + } + if let Ok(roots) = std::env::var("PERSONA_PROJECT_ROOTS") { + cfg.project_roots = roots.split(',').map(|s| PathBuf::from(s.trim())).collect(); + } + if let Ok(emails) = std::env::var("PERSONA_AUTHOR_EMAILS") { + cfg.author_emails = emails + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + if let Ok(model) = std::env::var("TINYCORTEX_LLM_MODEL") { + cfg.chat_model = model; + } + if let Ok(model) = std::env::var("TINYCORTEX_EMBED_MODEL") { + cfg.embed_model = model; + } + if let Ok(n) = std::env::var("PERSONA_MAX_SESSIONS") { + if let Ok(n) = n.parse() { + cfg.run_budget.max_sessions = n; + } + } + if let Ok(c) = std::env::var("PERSONA_MAX_COST_USD") { + if let Ok(c) = c.parse() { + cfg.run_budget.max_cost_usd = c; + } + } + cfg +} + +fn build_provider(persona: &PersonaConfig) -> anyhow::Result> { + let key = std::env::var("OPENROUTER_API_KEY").map_err(|_| { + anyhow::anyhow!("OPENROUTER_API_KEY not set (needed for backfill/incremental)") + })?; + let provider = OpenRouterProvider::new(OpenRouterConfig { + api_key: SecretString::new(key), + chat_model: persona.chat_model.clone(), + embed_model: persona.embed_model.clone(), + run_cost_limit_usd: Some(persona.run_budget.max_cost_usd), + run_call_limit: Some(persona.run_budget.max_llm_calls), + ..Default::default() + })?; + Ok(Arc::new(provider)) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + let _ = dotenvy::dotenv(); + let mode = std::env::args() + .nth(1) + .unwrap_or_else(|| "status".to_string()); + + let workspace = + env_path("TINYCORTEX_WORKSPACE").unwrap_or_else(|| PathBuf::from("./persona-workspace")); + std::fs::create_dir_all(&workspace)?; + let config = MemoryConfig::new(&workspace); + let persona = build_persona_config(); + let store = FileStateStore::open_in_workspace(&workspace)?; + + println!("persona harness: mode={mode}"); + println!(" workspace: {}", workspace.display()); + println!(" identity: {}", persona.identity); + println!( + " chat model: {} embed model: {}", + persona.chat_model, persona.embed_model + ); + println!( + " roots: claude={:?} codex={:?} projects={:?}", + persona.claude_code_root, persona.codex_root, persona.project_roots + ); + + match mode.as_str() { + "backfill" | "incremental" => { + let provider = build_provider(&persona)?; + let run_mode = if mode == "backfill" { + RunMode::Backfill + } else { + RunMode::Incremental + }; + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: provider.as_ref(), + summariser: provider.as_ref(), + store: &store, + }; + let started = std::time::Instant::now(); + let report = pipeline.run(run_mode).await?; + let usage = provider.usage(); + println!("\n── run report ──"); + println!("{}", serde_json::to_string_pretty(&report)?); + println!( + " provider: {} requests, {} prompt + {} completion tokens, ${:.4}", + usage.requests, usage.prompt_tokens, usage.completion_tokens, usage.cost_usd + ); + println!(" wall-clock: {:.1}s", started.elapsed().as_secs_f64()); + if let Some(path) = &report.pack_path { + println!(" pack: {path}"); + } + } + "compile" => { + let provider = build_provider(&persona).ok(); + // compile_only needs a summariser only to (re)open flavoured trees; + // it performs no folds, so any provider — or none — works. Fall back + // to a deterministic summariser when the key is absent. + let concat = tinycortex::memory::tree::summarise::ConcatSummariser::new(); + let summariser: &dyn tinycortex::memory::tree::Summariser = match &provider { + Some(p) => p.as_ref(), + None => &concat, + }; + // A dummy chat provider is never called by compile_only. + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &NullChat, + summariser, + store: &store, + }; + let path = pipeline.compile_only()?; + println!(" recompiled pack: {}", path.display()); + } + "status" => { + let pack = tinycortex::memory::persona::compile::pack_path(&config); + println!("\n── status ──"); + println!(" pack exists: {} ({})", pack.exists(), pack.display()); + let state_file = workspace.join("persona/sync-state.json"); + if state_file.exists() { + let bytes = std::fs::read(&state_file)?; + let map: serde_json::Value = serde_json::from_slice(&bytes)?; + let n = map.as_object().map(|o| o.len()).unwrap_or(0); + println!(" tracked cursors: {n}"); + } else { + println!(" tracked cursors: 0 (no prior run)"); + } + } + other => { + eprintln!("unknown mode '{other}' (expected backfill|incremental|compile|status)"); + std::process::exit(2); + } + } + Ok(()) +} + +/// A chat provider that is never invoked (used by `compile`, which does no map). +struct NullChat; + +#[async_trait::async_trait] +impl tinycortex::memory::score::extract::ChatProvider for NullChat { + fn name(&self) -> &str { + "null" + } + async fn chat_for_json( + &self, + _p: &tinycortex::memory::score::extract::ChatPrompt, + ) -> anyhow::Result { + anyhow::bail!("NullChat must not be called") + } +} diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 2892210..d07330b 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -97,6 +97,14 @@ pub mod providers; #[cfg(feature = "rpc")] pub mod rpc; +/// Persona distillation: turns local coding-agent history, instruction files, +/// and git commits into a durable persona memory layer (doc 06). +/// +/// Gated behind the default-off `persona` feature. Adds no new dependencies; +/// the git-history reader additionally requires `git-diff`. +#[cfg(feature = "persona")] +pub mod persona; + // ── Re-exports ────────────────────────────────────────────────────────────── pub use config::{MemoryConfig, WeightProfile}; pub use error::{MemoryEngineResult, MemoryError as MemoryEngineError}; diff --git a/src/memory/persona/compile.rs b/src/memory/persona/compile.rs new file mode 100644 index 0000000..f806474 --- /dev/null +++ b/src/memory/persona/compile.rs @@ -0,0 +1,224 @@ +//! Persona compiler (doc 06 §6.5 / §6.9): a **deterministic**, non-LLM step that +//! assembles the facet tree roots into the prompt-ready pack `persona/PERSONA.md`. +//! +//! Section order is fixed (§6.9): identity header, `## Directives` (T0, +//! budget-protected first), then Communication, Coding style, Stack, Workflow, +//! Environment, Anti-preferences. Each facet is clamped to a per-facet token +//! budget and the whole pack to `[5_000, 10_000]` tokens using the crate's token +//! estimator. Strength annotations ("distilled from N observations across M +//! projects") let downstream consumers judge confidence. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use anyhow::{Context, Result}; + +use super::types::PersonaFacet; +use crate::memory::config::MemoryConfig; +use crate::memory::tree::summarise::clamp_to_budget; + +/// Default per-facet token budget for a compiled section. +pub const DEFAULT_PER_FACET_BUDGET: u32 = 1_200; +/// Default total pack floor / ceiling (§6.1). +pub const DEFAULT_TOTAL_MIN: u32 = 5_000; +pub const DEFAULT_TOTAL_MAX: u32 = 10_000; + +/// Everything the deterministic compiler needs. Gathered by the pipeline from +/// the sealed facet trees, or reconstructed from disk by the `compile` +/// subcommand. +#[derive(Debug, Clone)] +pub struct PackInputs { + /// Identity line (e.g. the owner's email or name). + pub identity: String, + /// Compiled root body per facet (front-matter already stripped). + pub facet_bodies: BTreeMap, + /// Verbatim T0 directive rules for the Directives section (near-verbatim, + /// not LLM-folded). Rendered ahead of any distilled directives body. + pub directives: Vec, + /// Observation counts per facet (strength annotation). + pub counts: BTreeMap, + /// Distinct scope (project/repo) counts per facet (strength annotation). + pub scopes: BTreeMap, + /// Per-facet token budget. + pub per_facet_budget: u32, + /// Total pack token ceiling. + pub total_budget_max: u32, +} + +impl PackInputs { + /// Build inputs with the default budgets. + pub fn new(identity: impl Into) -> Self { + Self { + identity: identity.into(), + facet_bodies: BTreeMap::new(), + directives: Vec::new(), + counts: BTreeMap::new(), + scopes: BTreeMap::new(), + per_facet_budget: DEFAULT_PER_FACET_BUDGET, + total_budget_max: DEFAULT_TOTAL_MAX, + } + } +} + +/// Compile the pack markdown. Deterministic: identical inputs → identical bytes. +pub fn compile_pack(inputs: &PackInputs) -> String { + let mut out = String::new(); + out.push_str(&format!("# Persona: {}\n\n", inputs.identity.trim())); + out.push_str(&header_summary(inputs)); + out.push('\n'); + + // Budget accounting: directives (ALL[0]) is emitted first and is therefore + // protected — later facets are dropped before it if the ceiling is hit. + let mut spent: u32 = estimate(&out); + for facet in PersonaFacet::ALL { + let body = match facet_body(inputs, facet) { + Some(b) if !b.trim().is_empty() => b, + _ => continue, + }; + let remaining = inputs.total_budget_max.saturating_sub(spent); + if remaining == 0 { + break; + } + let facet_budget = inputs.per_facet_budget.min(remaining); + let (clamped, _) = clamp_to_budget(body.trim(), facet_budget); + if clamped.trim().is_empty() { + continue; + } + + let mut section = String::new(); + section.push_str(&format!("## {}\n", facet.heading())); + if let Some(annotation) = strength_annotation(inputs, facet) { + section.push_str(&format!("_{annotation}_\n\n")); + } else { + section.push('\n'); + } + section.push_str(clamped.trim()); + section.push_str("\n\n"); + + spent += estimate(§ion); + out.push_str(§ion); + } + + out.trim_end().to_string() + "\n" +} + +/// Effective body for a facet. The Directives section is the verbatim T0 rules +/// (near-verbatim) followed by any distilled directives body; every other facet +/// is just its distilled tree body. +fn facet_body(inputs: &PackInputs, facet: PersonaFacet) -> Option { + if facet == PersonaFacet::Directives { + let mut out = String::new(); + for rule in &inputs.directives { + out.push_str("- "); + out.push_str(rule.trim()); + out.push('\n'); + } + if let Some(b) = inputs.facet_bodies.get(&facet) { + if !b.trim().is_empty() { + out.push_str(b.trim()); + out.push('\n'); + } + } + return (!out.trim().is_empty()).then(|| out.trim().to_string()); + } + inputs + .facet_bodies + .get(&facet) + .filter(|b| !b.trim().is_empty()) + .cloned() +} + +/// One-line trait summary naming the facets that carry content. +fn header_summary(inputs: &PackInputs) -> String { + let present: Vec<&str> = PersonaFacet::ALL + .iter() + .filter(|f| facet_body(inputs, **f).is_some()) + .map(|f| f.heading()) + .collect(); + if present.is_empty() { + return "_No persona evidence distilled yet._\n".to_string(); + } + format!( + "> Mimic-grade persona pack distilled from this person's coding-agent history, \ +instruction files, and git commits. Facets covered: {}.\n", + present.join(", ") + ) +} + +/// "distilled from N observations across M projects" for a facet, or `None`. +fn strength_annotation(inputs: &PackInputs, facet: PersonaFacet) -> Option { + let n = inputs.counts.get(&facet).copied().unwrap_or(0); + if n == 0 { + return None; + } + let m = inputs.scopes.get(&facet).copied().unwrap_or(0); + if m > 1 { + Some(format!( + "distilled from {n} observations across {m} projects" + )) + } else { + Some(format!("distilled from {n} observations")) + } +} + +/// Token estimate consistent with [`clamp_to_budget`]. +fn estimate(text: &str) -> u32 { + clamp_to_budget(text, u32::MAX).1 +} + +/// Directory holding persona artifacts (`/persona`). +pub fn persona_dir(config: &MemoryConfig) -> PathBuf { + config.workspace.join("persona") +} + +/// Absolute path of the compiled pack. +pub fn pack_path(config: &MemoryConfig) -> PathBuf { + persona_dir(config).join("PERSONA.md") +} + +/// Path of the persisted verbatim-directives store (§6.9). One rule per line. +pub fn directives_path(config: &MemoryConfig) -> PathBuf { + persona_dir(config).join("directives.md") +} + +/// Persist the verbatim T0 directive rules so a later no-LLM `compile` can +/// reconstruct the Directives section. +pub fn write_directives(config: &MemoryConfig, directives: &[String]) -> Result<()> { + let path = directives_path(config); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create persona dir {}", parent.display()))?; + } + crate::memory::fsutil::atomic_write(&path, directives.join("\n").as_bytes()) + .with_context(|| format!("write directives {}", path.display()))?; + Ok(()) +} + +/// Read the persisted verbatim directives (empty if none). +pub fn read_directives(config: &MemoryConfig) -> Vec { + match std::fs::read_to_string(directives_path(config)) { + Ok(s) => s + .lines() + .filter(|l| !l.trim().is_empty()) + .map(str::to_string) + .collect(), + Err(_) => Vec::new(), + } +} + +/// Compile the pack and write it to [`pack_path`], returning the path. +pub fn write_pack(config: &MemoryConfig, inputs: &PackInputs) -> Result { + let markdown = compile_pack(inputs); + let path = pack_path(config); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create persona dir {}", parent.display()))?; + } + crate::memory::fsutil::atomic_write(&path, markdown.as_bytes()) + .with_context(|| format!("write persona pack {}", path.display()))?; + Ok(path) +} + +#[cfg(test)] +#[path = "compile_tests.rs"] +mod tests; diff --git a/src/memory/persona/compile_tests.rs b/src/memory/persona/compile_tests.rs new file mode 100644 index 0000000..f097f95 --- /dev/null +++ b/src/memory/persona/compile_tests.rs @@ -0,0 +1,97 @@ +//! Tests for the deterministic persona compiler. + +use super::*; + +fn inputs_with(bodies: &[(PersonaFacet, &str)]) -> PackInputs { + let mut inputs = PackInputs::new("test@example.com"); + for (f, b) in bodies { + inputs.facet_bodies.insert(*f, b.to_string()); + inputs.counts.insert(*f, 5); + inputs.scopes.insert(*f, 2); + } + inputs +} + +#[test] +fn emits_header_and_sections_in_fixed_order() { + let inputs = inputs_with(&[ + (PersonaFacet::Stack, "- Uses Rust 2021."), + ( + PersonaFacet::Directives, + "- Always branch before writing code.", + ), + (PersonaFacet::CodingStyle, "- Small focused modules."), + ]); + let pack = compile_pack(&inputs); + + assert!(pack.starts_with("# Persona: test@example.com")); + // Directives must come before Coding style, which comes before Stack. + let d = pack.find("## Directives").unwrap(); + let c = pack.find("## Coding style").unwrap(); + let s = pack.find("## Stack").unwrap(); + assert!(d < c && c < s, "section order wrong:\n{pack}"); + // Strength annotation is present. + assert!(pack.contains("distilled from 5 observations across 2 projects")); + // Verbatim directive survives. + assert!(pack.contains("Always branch before writing code.")); +} + +#[test] +fn empty_inputs_produce_placeholder() { + let pack = compile_pack(&PackInputs::new("nobody")); + assert!(pack.contains("# Persona: nobody")); + assert!(pack.contains("No persona evidence distilled yet")); + assert!(!pack.contains("## ")); +} + +#[test] +fn total_budget_ceiling_protects_directives_first() { + // A huge body per facet; a tiny total budget. Directives (first) must + // survive; later facets get dropped. + let big = "- ".to_string() + &"word ".repeat(4000); + let mut inputs = inputs_with(&[ + (PersonaFacet::Directives, &big), + (PersonaFacet::Stack, &big), + (PersonaFacet::AntiPreferences, &big), + ]); + inputs.per_facet_budget = 2_000; + inputs.total_budget_max = 2_100; // only room for ~one facet + let pack = compile_pack(&inputs); + assert!( + pack.contains("## Directives"), + "directives must be protected" + ); + assert!( + !pack.contains("## Anti-preferences"), + "later facets dropped under the ceiling:\n{}", + &pack[..pack.len().min(200)] + ); +} + +#[test] +fn per_facet_body_is_clamped() { + let big = "- ".to_string() + &"token ".repeat(3000); + let mut inputs = inputs_with(&[(PersonaFacet::Directives, &big)]); + inputs.per_facet_budget = 100; + inputs.total_budget_max = 10_000; + let pack = compile_pack(&inputs); + // The clamped directives section must be far smaller than the raw body. + assert!( + pack.len() < big.len() / 2, + "body was not clamped: {} vs {}", + pack.len(), + big.len() + ); +} + +#[test] +fn deterministic_across_runs() { + let inputs = inputs_with(&[ + (PersonaFacet::Communication, "- Terse and direct."), + ( + PersonaFacet::Workflow, + "- Uses worktrees for parallel work.", + ), + ]); + assert_eq!(compile_pack(&inputs), compile_pack(&inputs)); +} diff --git a/src/memory/persona/config.rs b/src/memory/persona/config.rs new file mode 100644 index 0000000..94dffec --- /dev/null +++ b/src/memory/persona/config.rs @@ -0,0 +1,166 @@ +//! Declarative configuration for the persona pipeline (doc 06 §6.8). +//! +//! Fully serde-declarative like the rest of the crate: source roots (with +//! platform defaults for the launch vendors), export paths, author emails, +//! instruction-file roots, model ids, per-facet asks (defaulted, overridable), +//! per-facet + total token budgets, and run budgets. Kept in its own +//! feature-gated module rather than bolted onto the core `MemoryConfig` so the +//! dependency-light core never carries persona fields. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use super::compile::{DEFAULT_PER_FACET_BUDGET, DEFAULT_TOTAL_MAX}; +use super::reduce::FacetAsks; +use super::types::PersonaFacet; + +/// Run-budget ceilings (§6.7): a run aborts cleanly once any is hit. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersonaRunBudget { + /// Max sessions/batches digested per run. + pub max_sessions: usize, + /// Max LLM chat calls per run (enforced by the provider too). + pub max_llm_calls: u32, + /// Max provider spend per run, USD. + pub max_cost_usd: f64, +} + +impl Default for PersonaRunBudget { + fn default() -> Self { + Self { + max_sessions: 5_000, + max_llm_calls: 5_000, + max_cost_usd: 5.0, + } + } +} + +/// Git reader tunables mirrored into config (serde-friendly). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersonaGitConfig { + /// Commit-message evidence units per digest batch. + pub batch_size: usize, + /// Max author commits scanned per repo. + pub max_commits: usize, + /// Max sampled diffs per repo. + pub diff_sample_cap: usize, + /// Max bytes of any single sampled diff kept. + pub diff_size_cap_bytes: usize, + /// A commit qualifies for diff sampling only if it changed at most this many + /// files. + pub small_commit_max_files: usize, +} + +impl Default for PersonaGitConfig { + fn default() -> Self { + Self { + batch_size: 100, + max_commits: 2_000, + diff_sample_cap: 20, + diff_size_cap_bytes: 4_000, + small_commit_max_files: 3, + } + } +} + +/// Top-level persona configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersonaConfig { + /// Identity line for the pack header (email / name). + pub identity: String, + /// Claude Code transcript root (`~/.claude/projects`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub claude_code_root: Option, + /// Codex rollout root (`~/.codex/sessions`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub codex_root: Option, + /// Roots walked for repo-scoped instruction files + git repos. + #[serde(default)] + pub project_roots: Vec, + /// Explicit global instruction files (`~/.claude/CLAUDE.md`, …). + #[serde(default)] + pub global_instruction_files: Vec, + /// Author emails used to filter git history (case-insensitive). + #[serde(default)] + pub author_emails: Vec, + /// Chat/digest model id. + pub chat_model: String, + /// Embedding model id. + pub embed_model: String, + /// Per-facet ask overrides (facet wire-string → ask). Missing facets use the + /// built-in default ask. + #[serde(default)] + pub facet_asks: BTreeMap, + /// Per-facet compiled-section token budget. + #[serde(default = "default_per_facet_budget")] + pub per_facet_token_budget: u32, + /// Total pack token ceiling. + #[serde(default = "default_total_budget")] + pub total_token_budget: u32, + /// Run budgets. + #[serde(default)] + pub run_budget: PersonaRunBudget, + /// Git reader tunables. + #[serde(default)] + pub git: PersonaGitConfig, + /// Max digest (LLM map) calls in flight at once. The map step is network- + /// bound and dominates wall-clock, so digests run concurrently (folds stay + /// serial). Order is preserved so trees still fold oldest-first. + #[serde(default = "default_digest_concurrency")] + pub digest_concurrency: usize, +} + +fn default_per_facet_budget() -> u32 { + DEFAULT_PER_FACET_BUDGET +} +fn default_total_budget() -> u32 { + DEFAULT_TOTAL_MAX +} +fn default_digest_concurrency() -> usize { + 8 +} + +impl PersonaConfig { + /// Construct with platform defaults derived from `home`. This is where the + /// five-vendor source-root defaults live (config itself stays pure). + pub fn with_home(home: &Path, identity: impl Into) -> Self { + Self { + identity: identity.into(), + claude_code_root: Some(home.join(".claude/projects")), + codex_root: Some(home.join(".codex/sessions")), + project_roots: vec![home.join("work")], + global_instruction_files: vec![ + home.join(".claude/CLAUDE.md"), + home.join(".codex/AGENTS.md"), + ], + author_emails: Vec::new(), + // Model ids are plain strings the host may override — persona code + // must not name any concrete provider, only the id it routes. + chat_model: "deepseek/deepseek-v4-flash".to_string(), + embed_model: "openai/text-embedding-3-small".to_string(), + facet_asks: BTreeMap::new(), + per_facet_token_budget: DEFAULT_PER_FACET_BUDGET, + total_token_budget: DEFAULT_TOTAL_MAX, + run_budget: PersonaRunBudget::default(), + git: PersonaGitConfig::default(), + digest_concurrency: default_digest_concurrency(), + } + } + + /// Resolve per-facet asks (config overrides + built-in defaults). + pub fn asks(&self) -> FacetAsks { + let mut map = BTreeMap::new(); + for facet in PersonaFacet::ALL { + if let Some(ask) = self.facet_asks.get(facet.as_str()) { + map.insert(facet, ask.clone()); + } + } + FacetAsks(map) + } +} + +#[cfg(test)] +#[path = "config_tests.rs"] +mod tests; diff --git a/src/memory/persona/config_tests.rs b/src/memory/persona/config_tests.rs new file mode 100644 index 0000000..6ff875e --- /dev/null +++ b/src/memory/persona/config_tests.rs @@ -0,0 +1,50 @@ +//! Tests for PersonaConfig defaults, serde, and ask resolution. + +use super::*; +use std::path::Path; + +#[test] +fn with_home_sets_platform_defaults() { + let cfg = PersonaConfig::with_home(Path::new("/home/x"), "x@example.com"); + assert_eq!( + cfg.claude_code_root.as_deref(), + Some(Path::new("/home/x/.claude/projects")) + ); + assert_eq!( + cfg.codex_root.as_deref(), + Some(Path::new("/home/x/.codex/sessions")) + ); + assert!(cfg + .global_instruction_files + .iter() + .any(|p| p.ends_with(".claude/CLAUDE.md"))); + assert!(cfg + .global_instruction_files + .iter() + .any(|p| p.ends_with(".codex/AGENTS.md"))); + assert_eq!(cfg.total_token_budget, DEFAULT_TOTAL_MAX); +} + +#[test] +fn asks_fall_back_to_defaults_and_honour_overrides() { + let mut cfg = PersonaConfig::with_home(Path::new("/home/x"), "x"); + cfg.facet_asks + .insert("workflow".to_string(), "custom workflow ask".to_string()); + let asks = cfg.asks(); + assert_eq!(asks.ask(PersonaFacet::Workflow), "custom workflow ask"); + // Un-overridden facet uses the built-in default. + assert_eq!( + asks.ask(PersonaFacet::CodingStyle), + PersonaFacet::CodingStyle.default_ask() + ); +} + +#[test] +fn round_trips_through_json() { + let cfg = PersonaConfig::with_home(Path::new("/home/x"), "x@example.com"); + let json = serde_json::to_string(&cfg).unwrap(); + let back: PersonaConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(back.identity, "x@example.com"); + assert_eq!(back.chat_model, cfg.chat_model); + assert_eq!(back.run_budget.max_sessions, cfg.run_budget.max_sessions); +} diff --git a/src/memory/persona/distill.rs b/src/memory/persona/distill.rs new file mode 100644 index 0000000..8d0e31f --- /dev/null +++ b/src/memory/persona/distill.rs @@ -0,0 +1,213 @@ +//! Distillation map step (doc 06 §6.5): one `chat_for_json` call per extracted +//! session (or commit batch) turning a [`RawSession`] of redacted evidence into +//! a [`SessionDigest`] of per-facet, prescriptive observations. +//! +//! Follows the `LlmEntityExtractor` pattern: a strict-JSON instruction with the +//! schema in the prompt, and a **soft fallback** — any failure (transport, +//! malformed JSON, empty) skips the session by returning an empty digest rather +//! than aborting the run. Oversized sessions are windowed and digested in parts, +//! and the observations are concatenated. + +use anyhow::Result; +use serde::Deserialize; + +use super::readers::RawSession; +use super::types::{DigestObservation, EvidenceTier, PersonaFacet, SessionDigest}; +use crate::memory::score::extract::{ChatPrompt, ChatProvider}; +use crate::memory::store::safety::sanitize_text; + +/// Max characters of evidence sent in a single digest call. Larger sessions are +/// split into windows and digested part-by-part. +const WINDOW_CHARS: usize = 12_000; +/// Output-token cap for a digest response (one small JSON object). +const DIGEST_MAX_OUTPUT_TOKENS: u32 = 4_096; + +/// The strict-JSON system prompt: schema + extraction contract. +fn system_prompt() -> String { + let facets = PersonaFacet::ALL + .iter() + .map(|f| f.as_str()) + .collect::>() + .join(", "); + format!( + "You analyse a person's own messages to their AI coding agents (and their \ +git commit style). Your job is to distil a durable profile of THE PERSON so another \ +agent can mimic them — never describe the agent or the task.\n\n\ +Extract prescriptive observations across these facets: {facets}.\n\ +- communication: tone, verbosity, directness, phrasing quirks, how they give feedback.\n\ +- coding_style: naming, structure, comments, error handling, testing, size discipline.\n\ +- stack: languages, frameworks, libraries, architectural choices they favour.\n\ +- workflow: branching/commit granularity, plan-first vs dive-in, PR/review habits, parallelism.\n\ +- environment: editors, agent harnesses, CLIs, package managers, OS.\n\ +- directives: explicit standing rules they state.\n\ +- anti_preferences: things they dislike, correct, revert, or forbid.\n\n\ +Each observation must be a concrete rule an agent should follow to act like this \ +person, backed by a short supporting quote drawn from the evidence, plus a \ +confidence tier:\n\ +- t0: an explicit written rule.\n\ +- t1: a correction/interrupt (the person stopped the agent and redirected it).\n\ +- t2: prompt phrasing, command habits, commit-message style.\n\ +- t3: inferred from accepted outcomes (weak; corroborates only).\n\n\ +Only emit well-supported observations; omit weak guesses. Respond with a JSON \ +object of exactly this shape and nothing else:\n\ +{{\"observations\":[{{\"facet\":\"coding_style\",\"observation\":\"...\",\"quote\":\"...\",\"tier\":\"t2\"}}]}}" + ) +} + +/// Build the user prompt for one window of evidence. +fn user_prompt(session: &RawSession, window: &str) -> String { + let provenance = format!( + "source={} scope={}", + session.source.kind.as_str(), + session.source.scope.as_deref().unwrap_or("(unknown)") + ); + format!("Provenance: {provenance}\n\nEvidence (each line is one unit):\n{window}\n\nReturn JSON only.") +} + +/// Split a session's evidence into windows of at most [`WINDOW_CHARS`] chars, +/// each a newline-joined block of the evidence excerpts with their tiers. +fn windows(session: &RawSession) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + for ev in &session.evidence { + let line = format!("[{}] {}\n", ev.tier.as_str(), ev.excerpt()); + if !cur.is_empty() && cur.len() + line.len() > WINDOW_CHARS { + out.push(std::mem::take(&mut cur)); + } + // A single oversized unit is truncated to the window. + if line.len() > WINDOW_CHARS { + cur.push_str(&line.chars().take(WINDOW_CHARS).collect::()); + out.push(std::mem::take(&mut cur)); + } else { + cur.push_str(&line); + } + } + if !cur.trim().is_empty() { + out.push(cur); + } + out +} + +/// Digest one session into a [`SessionDigest`] via the chat provider. +/// +/// Distinguishes two failure modes so the pipeline can checkpoint correctly: +/// - **Provider/transport failure** (a `chat_for_json` error — budget exhausted, +/// 401/403, transport) → returns `Err`. The caller must NOT commit the +/// session's cursor, so the evidence is re-attempted on the next run. +/// - **Model produced no usable output** (a valid call whose response wasn't +/// parseable JSON, or yielded zero observations) → returns `Ok` with an empty +/// digest. Re-running would reproduce it, so the cursor IS committed. +pub async fn digest_session( + provider: &dyn ChatProvider, + session: &RawSession, +) -> Result { + if session.is_empty() { + return Ok(SessionDigest::empty(session.source.clone())); + } + let mut observations: Vec = Vec::new(); + for window in windows(session) { + // A hard provider failure bubbles up (the whole session is retried next + // run); a soft parse failure yields an empty window and is tolerated. + let obs = digest_window(provider, session, &window).await?; + observations.extend(obs); + } + Ok(SessionDigest { + source: session.source.clone(), + observations, + }) +} + +/// One window → observations. A `chat_for_json` failure bubbles up as `Err` +/// (hard, non-committable); an unparseable-but-received response degrades to an +/// empty window (`Ok(vec![])`) since retrying reproduces it. +async fn digest_window( + provider: &dyn ChatProvider, + session: &RawSession, + window: &str, +) -> Result> { + let prompt = ChatPrompt { + system: system_prompt(), + user: user_prompt(session, window), + temperature: 0.0, + kind: "persona::digest", + max_tokens: Some(DIGEST_MAX_OUTPUT_TOKENS), + }; + let raw = provider.chat_for_json(&prompt).await?; + let parsed: RawDigest = match parse_digest(&raw) { + Ok(p) => p, + Err(e) => { + log::warn!( + "[persona] digest parse failed for {} ({}): {e:#}", + session.source.kind.as_str(), + session.source.session_id.as_deref().unwrap_or("?") + ); + return Ok(Vec::new()); + } + }; + Ok(parsed + .observations + .into_iter() + .filter_map(RawObservation::into_observation) + .collect()) +} + +/// Parse a digest response, tolerating models that wrap the JSON in prose or +/// code fences by extracting the first `{...}` object. +fn parse_digest(raw: &str) -> Result { + if let Ok(d) = serde_json::from_str::(raw) { + return Ok(d); + } + let start = raw.find('{'); + let end = raw.rfind('}'); + if let (Some(s), Some(e)) = (start, end) { + if e > s { + return Ok(serde_json::from_str::(&raw[s..=e])?); + } + } + Err(anyhow::anyhow!( + "digest response was not JSON: {}", + raw.chars().take(120).collect::() + )) +} + +#[derive(Debug, Deserialize)] +struct RawDigest { + #[serde(default)] + observations: Vec, +} + +#[derive(Debug, Deserialize)] +struct RawObservation { + #[serde(default)] + facet: String, + #[serde(default)] + observation: String, + #[serde(default)] + quote: String, + #[serde(default)] + tier: String, +} + +impl RawObservation { + /// Validate and normalise one raw observation, dropping unusable ones. + fn into_observation(self) -> Option { + let facet = PersonaFacet::parse_loose(&self.facet)?; + let observation = self.observation.trim().to_string(); + if observation.len() < 3 { + return None; + } + let tier = EvidenceTier::parse_loose(&self.tier).unwrap_or(EvidenceTier::T3); + // Defensive: redact the quote in case the model echoed raw evidence. + let quote = sanitize_text(self.quote.trim()).value; + Some(DigestObservation { + facet, + observation: sanitize_text(&observation).value, + quote, + tier, + }) + } +} + +#[cfg(test)] +#[path = "distill_tests.rs"] +mod tests; diff --git a/src/memory/persona/distill_tests.rs b/src/memory/persona/distill_tests.rs new file mode 100644 index 0000000..82541e9 --- /dev/null +++ b/src/memory/persona/distill_tests.rs @@ -0,0 +1,115 @@ +//! Tests for the digest map step (mock chat provider). + +use super::*; +use async_trait::async_trait; +use chrono::Utc; + +use crate::memory::persona::readers::RawSession; +use crate::memory::persona::types::{EvidenceSource, EvidenceTier, PersonaSourceKind}; + +/// A chat provider that returns a canned body (or errors) for every call. +struct MockChat { + body: Result, +} + +#[async_trait] +impl ChatProvider for MockChat { + fn name(&self) -> &str { + "mock" + } + async fn chat_for_json(&self, _prompt: &ChatPrompt) -> anyhow::Result { + self.body.clone().map_err(|e| anyhow::anyhow!(e)) + } +} + +fn session_with(excerpts: &[(&str, EvidenceTier)]) -> RawSession { + let src = EvidenceSource::new(PersonaSourceKind::ClaudeCode).with_scope("demo"); + let mut s = RawSession::new(src.clone()); + for (text, tier) in excerpts { + s.push(crate::memory::persona::types::PersonaEvidence::new( + src.clone(), + Utc::now(), + *tier, + text, + vec![], + )); + } + s +} + +#[tokio::test] +async fn parses_observations_from_json() { + let body = r#"{"observations":[ + {"facet":"workflow","observation":"Commits small and often","quote":"commit small","tier":"t2"}, + {"facet":"coding-style","observation":"Wants regression tests","quote":"add a test","tier":"t1"} + ]}"#; + let provider = MockChat { + body: Ok(body.into()), + }; + let session = session_with(&[("commit small and often", EvidenceTier::T2)]); + let digest = digest_session(&provider, &session).await.unwrap(); + assert_eq!(digest.observations.len(), 2); + assert_eq!(digest.observations[0].facet, PersonaFacet::Workflow); + assert_eq!(digest.observations[1].facet, PersonaFacet::CodingStyle); + assert_eq!(digest.observations[1].tier, EvidenceTier::T1); +} + +#[tokio::test] +async fn tolerates_prose_wrapped_json() { + let body = "Sure! Here is the JSON:\n```json\n{\"observations\":[{\"facet\":\"stack\",\"observation\":\"Uses Rust\",\"quote\":\"cargo\",\"tier\":\"t2\"}]}\n```"; + let provider = MockChat { + body: Ok(body.into()), + }; + let session = session_with(&[("cargo test", EvidenceTier::T2)]); + let digest = digest_session(&provider, &session).await.unwrap(); + assert_eq!(digest.observations.len(), 1); + assert_eq!(digest.observations[0].facet, PersonaFacet::Stack); +} + +#[tokio::test] +async fn soft_falls_back_on_error_and_bad_json() { + let session = session_with(&[("x", EvidenceTier::T2)]); + + // A hard provider failure surfaces as Err (so the caller won't commit the + // cursor and the session is retried next run). + let failing = MockChat { + body: Err("402 requires more credits".into()), + }; + assert!(digest_session(&failing, &session).await.is_err()); + + // A received-but-unparseable response is a soft failure: Ok + empty digest + // (re-running reproduces it, so the cursor may commit). + let garbage = MockChat { + body: Ok("not json at all".into()), + }; + assert!(digest_session(&garbage, &session).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn empty_session_yields_empty_digest() { + let provider = MockChat { + body: Ok("{\"observations\":[]}".into()), + }; + let session = RawSession::new(EvidenceSource::new(PersonaSourceKind::Codex)); + assert!(digest_session(&provider, &session) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn drops_unusable_observations() { + // Unknown facet + too-short observation are both dropped. + let body = r#"{"observations":[ + {"facet":"bogus","observation":"whatever","tier":"t2"}, + {"facet":"stack","observation":"x","tier":"t2"}, + {"facet":"stack","observation":"Prefers Postgres","quote":"","tier":"t3"} + ]}"#; + let provider = MockChat { + body: Ok(body.into()), + }; + let session = session_with(&[("db", EvidenceTier::T2)]); + let digest = digest_session(&provider, &session).await.unwrap(); + assert_eq!(digest.observations.len(), 1); + assert_eq!(digest.observations[0].observation, "Prefers Postgres"); +} diff --git a/src/memory/persona/mod.rs b/src/memory/persona/mod.rs new file mode 100644 index 0000000..0668a2e --- /dev/null +++ b/src/memory/persona/mod.rs @@ -0,0 +1,34 @@ +//! Persona distillation (doc 06): turns a person's local coding-agent history, +//! agent instruction files, and git commit history into a durable **persona +//! memory layer** — personality, communication style, coding style, and +//! tool/stack preferences compiled into a small, prompt-ready context pack. +//! +//! The surface is organised around the canonical evidence model in [`types`]: +//! readers ([`readers`]) emit redacted [`PersonaEvidence`], the map step +//! ([`distill`]) turns batches of evidence into [`SessionDigest`]s via a +//! [`ChatProvider`](crate::memory::score::extract::ChatProvider), the reduce +//! step folds digests into seven facet flavoured trees, and the [`compile`] +//! step assembles `persona/PERSONA.md`. [`state`] makes runs incremental and +//! resumable; [`pipeline`] wires it all together for the CLI harness. +//! +//! Everything is local-first and depends only on the crate's `ChatProvider` / +//! `Summariser` / `EmbeddingBackend` trait seams — nothing here names a +//! concrete provider (the OpenRouter reference provider lives under +//! `memory::providers`). + +pub mod compile; +pub mod config; +pub mod distill; +pub mod pipeline; +pub mod readers; +pub mod reduce; +pub mod state; +pub mod types; + +pub use config::PersonaConfig; +pub use pipeline::{Pipeline, RunMode, RunReport}; + +pub use types::{ + evidence_id, DigestObservation, EvidenceSource, EvidenceTier, PersonaEvidence, PersonaFacet, + PersonaSourceKind, SessionDigest, +}; diff --git a/src/memory/persona/pipeline.rs b/src/memory/persona/pipeline.rs new file mode 100644 index 0000000..b65d29b --- /dev/null +++ b/src/memory/persona/pipeline.rs @@ -0,0 +1,457 @@ +//! Persona pipeline orchestration (doc 06 §6.5–§6.8): drives readers → digest → +//! facet-tree reduce → compile, with incremental cursors and run budgets. +//! +//! Two modes (§6.7): `Backfill` walks everything oldest-first so trees fold +//! chronologically; `Incremental` skips files/repos whose cursor is unchanged. +//! Both honour the run budget (max sessions / LLM calls); provider cost is +//! bounded by the provider's own per-run ceiling. Because evidence ids are +//! content-addressed, re-runs dedupe naturally — cursors are a fast-skip, not a +//! correctness gate. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use serde::Serialize; + +use super::compile::{write_pack, PackInputs}; +use super::config::PersonaConfig; +use super::distill::digest_session; +use super::readers::{claude_code, codex, instruction, RawSession}; +use super::reduce::{fold_digest, fold_directives, seal_and_collect, FacetAsks, ReduceState}; +use super::state::PersonaStateStore; +use super::state::{self, file_key, file_unchanged, record_file}; +use super::types::{PersonaFacet, SessionDigest}; +use crate::memory::config::MemoryConfig; +use crate::memory::score::extract::ChatProvider; +use crate::memory::tree::Summariser; +use futures::stream::{self, StreamExt}; + +/// Run mode (§6.7). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunMode { + /// Walk everything, oldest-first. + Backfill, + /// Cursor-forward only: skip unchanged files/repos. + Incremental, +} + +impl RunMode { + fn as_str(self) -> &'static str { + match self { + RunMode::Backfill => "backfill", + RunMode::Incremental => "incremental", + } + } +} + +/// What a run did — printed by the harness `status`/run output. +#[derive(Debug, Clone, Default, Serialize)] +pub struct RunReport { + /// The mode that ran. + pub mode: String, + /// Transcript/instruction files discovered. + pub files_seen: usize, + /// Sessions/batches actually digested. + pub sessions_processed: usize, + /// Files skipped because their cursor was unchanged. + pub sessions_skipped: usize, + /// Instruction-file rules folded (verbatim T0). + pub directives_folded: usize, + /// Total evidence units extracted this run. + pub evidence_units: usize, + /// Digest calls that produced at least one observation. + pub digests: usize, + /// Sessions whose digest hit a hard provider failure (cursor NOT committed; + /// retried next run). + pub sessions_failed: usize, + /// Observations distilled. + pub observations: usize, + /// Per-facet observation counts (facet wire-string → count). + pub facet_counts: BTreeMap, + /// True when a run budget stopped the run early (checkpointed). + pub budget_hit: bool, + /// Path of the compiled pack, if written. + pub pack_path: Option, +} + +/// A run's budget accounting. +struct Budget { + max_sessions: usize, + max_calls: usize, + sessions: usize, + calls: usize, +} + +impl Budget { + fn from(cfg: &PersonaConfig) -> Self { + Self { + max_sessions: cfg.run_budget.max_sessions, + max_calls: cfg.run_budget.max_llm_calls as usize, + sessions: 0, + calls: 0, + } + } + /// True if another digest would exceed the budget (stop cleanly). + fn exhausted(&self) -> bool { + self.sessions >= self.max_sessions || self.calls >= self.max_calls + } + fn charge(&mut self) { + self.sessions += 1; + self.calls += 1; + } +} + +/// The pipeline binds the workspace, config, provider, summariser, and state +/// store; `run` executes one pass. +pub struct Pipeline<'a> { + /// Memory workspace config (tree store, content root). + pub config: &'a MemoryConfig, + /// Persona config (roots, models, budgets, asks). + pub persona: &'a PersonaConfig, + /// Chat provider for the digest map step. + pub provider: &'a dyn ChatProvider, + /// Summariser for the facet-tree folds. + pub summariser: &'a dyn Summariser, + /// Incremental-run state store. + pub store: &'a dyn PersonaStateStore, +} + +impl Pipeline<'_> { + /// Execute one pass in `mode`, writing `persona/PERSONA.md`. + pub async fn run(&self, mode: RunMode) -> Result { + let asks = self.persona.asks(); + let mut state = ReduceState::default(); + let mut budget = Budget::from(self.persona); + let mut report = RunReport { + mode: mode.as_str().to_string(), + ..Default::default() + }; + + // 1. Instruction files (no LLM) — highest-confidence T0 directives. + // Always re-read the full set every run (cheap, no LLM) so edits and + // removals are reflected — directives are rebuilt fresh, never seeded + // from a stale persisted copy. + self.ingest_instructions(&mut state, &mut report).await?; + + // 2. Transcripts (Claude Code + Codex) — the digest map step. + self.ingest_transcripts(mode, &asks, &mut state, &mut budget, &mut report) + .await?; + + // 3. Git history (feature-gated). + #[cfg(feature = "git-diff")] + self.ingest_git(mode, &asks, &mut state, &mut budget, &mut report) + .await?; + + report.budget_hit = budget.exhausted(); + + // 4. Seal facet trees + compile the pack. + let bodies = seal_and_collect(self.config, &asks, self.summariser).await?; + let pack_path = self.compile_and_write(bodies, &state)?; + report.pack_path = Some(pack_path.display().to_string()); + for (facet, n) in &state.counts { + report.facet_counts.insert(facet.as_str().to_string(), *n); + } + Ok(report) + } + + /// Re-assemble the pack from the current facet-tree roots without any LLM + /// calls (the `compile` subcommand). + pub fn compile_only(&self) -> Result { + use super::reduce::strip_frontmatter; + use crate::memory::tree::flavoured::compile_flavoured_root; + use crate::memory::tree::TreeFactory; + + let asks = self.persona.asks(); + let mut bodies = BTreeMap::new(); + let mut state = ReduceState::default(); + // Reconstruct verbatim directives from the persisted store. + state.directives = super::compile::read_directives(self.config); + for facet in PersonaFacet::ALL { + let factory = TreeFactory::flavoured(facet.tree_scope(), asks.ask(facet)); + let tree = factory.get_or_create(self.config)?; + let markdown = compile_flavoured_root(self.config, &tree.id)?; + let body = strip_frontmatter(&markdown); + if !body.trim().is_empty() { + bodies.insert(facet, body); + // Use leaves-folded as a rough observation count proxy. + *state.counts.entry(facet).or_default() += tree.root_id.is_some() as usize; + } + } + self.compile_and_write(bodies, &state) + } + + /// Build [`PackInputs`] and write the pack. + fn compile_and_write( + &self, + bodies: BTreeMap, + state: &ReduceState, + ) -> Result { + // Persist the verbatim directives so a later `compile` can rebuild the + // Directives section without re-reading instruction files. + if !state.directives.is_empty() { + super::compile::write_directives(self.config, &state.directives)?; + } + let mut inputs = PackInputs::new(self.persona.identity.clone()); + inputs.facet_bodies = bodies; + inputs.directives = state.directives.clone(); + inputs.counts = state.counts.clone(); + inputs.scopes = state.scopes.iter().map(|(k, v)| (*k, v.len())).collect(); + inputs.per_facet_budget = self.persona.per_facet_token_budget; + inputs.total_budget_max = self.persona.total_token_budget; + write_pack(self.config, &inputs) + } + + async fn ingest_instructions( + &self, + state: &mut ReduceState, + report: &mut RunReport, + ) -> Result<()> { + let files = instruction::discover( + &self.persona.project_roots, + &self.persona.global_instruction_files, + ); + for file in files { + report.files_seen += 1; + let key = file_key("instruction_file", &file.path); + let bytes = match std::fs::read(&file.path) { + Ok(b) => b, + Err(_) => continue, + }; + let sha = instruction::content_sha(&bytes); + // Re-read every run so removed/edited rules drop out of the freshly + // rebuilt directive set (dedup keeps repeats collapsed). No LLM cost. + let session = match instruction::read_file(&file) { + Ok(s) => s, + Err(_) => continue, + }; + report.evidence_units += session.evidence.len(); + report.directives_folded += session.evidence.len(); + fold_directives(&session.evidence, state); + state::record_watermark(self.store, &key, &sha).await?; + } + Ok(()) + } + + async fn ingest_transcripts( + &self, + mode: RunMode, + asks: &FacetAsks, + state: &mut ReduceState, + budget: &mut Budget, + report: &mut RunReport, + ) -> Result<()> { + let mut files: Vec<(PathBuf, &'static str)> = Vec::new(); + if let Some(root) = &self.persona.claude_code_root { + for p in claude_code::discover(root) { + files.push((p, "claude_code")); + } + } + if let Some(root) = &self.persona.codex_root { + for p in codex::discover(root) { + files.push((p, "codex")); + } + } + // Oldest-first for chronological folding. + files.sort_by_key(|(p, _)| file_mtime_ms(p)); + + // Read (cheap I/O, serial) into a pending list, then digest concurrently + // + fold serially below. Cursors are committed only AFTER a session is + // digested, so a budget-truncated file is re-processed on resume. + let mut pending: Vec = Vec::new(); + for (path, kind) in files { + report.files_seen += 1; + let key = file_key(kind, &path); + if mode == RunMode::Incremental && file_unchanged(self.store, &key, &path).await? { + report.sessions_skipped += 1; + continue; + } + let session: RawSession = match read_transcript(kind, &path) { + Ok(s) => s, + Err(_) => continue, + }; + if session.is_empty() { + // Nothing to digest — record the cursor now so we don't re-read. + record_file(self.store, &key, &path).await?; + continue; + } + report.evidence_units += session.evidence.len(); + let commit = state::FileCursor::of(&path) + .and_then(|c| serde_json::to_value(c).ok()) + .map(|v| (key, v)); + pending.push(Pending { session, commit }); + } + self.digest_and_fold(pending, asks, state, budget, report) + .await + } + + /// Digest the `pending` sessions concurrently (the network-bound map step) + /// and fold the results serially into the facet trees (SQLite writes must + /// stay serial). Order is preserved (`buffered`, not `buffer_unordered`) so + /// trees fold oldest-first. Selection honours the shared run budget; each + /// selected session's cursor is committed only after it is folded, so a + /// budget-truncated tail is re-processed on the next run. + async fn digest_and_fold( + &self, + pending: Vec, + asks: &FacetAsks, + state: &mut ReduceState, + budget: &mut Budget, + report: &mut RunReport, + ) -> Result<()> { + // Select within budget up front. + let mut selected: Vec = Vec::new(); + for p in pending { + if budget.exhausted() { + report.budget_hit = true; + break; + } + budget.charge(); + selected.push(p); + } + if selected.is_empty() { + return Ok(()); + } + let concurrency = self.persona.digest_concurrency.max(1); + let results: Vec> = stream::iter(selected.iter()) + .map(|p| digest_session(self.provider, &p.session)) + .buffered(concurrency) + .collect() + .await; + for (p, result) in selected.iter().zip(results) { + let digest = match result { + Ok(d) => d, + Err(e) => { + // Hard provider failure: do NOT commit the cursor, so this + // session is re-attempted on the next run. + log::warn!("[persona] digest failed, cursor not committed: {e:#}"); + report.sessions_failed += 1; + continue; + } + }; + report.sessions_processed += 1; + if !digest.is_empty() { + report.digests += 1; + report.observations += digest.observations.len(); + fold_digest(self.config, &digest, asks, self.summariser, state).await?; + } + // Commit the cursor/watermark now that the session is folded (a valid + // empty digest still commits — retrying would reproduce it). + if let Some((key, value)) = &p.commit { + self.store.set(state::NAMESPACE, key, value).await?; + } + } + Ok(()) + } + + #[cfg(feature = "git-diff")] + async fn ingest_git( + &self, + mode: RunMode, + asks: &FacetAsks, + state: &mut ReduceState, + budget: &mut Budget, + report: &mut RunReport, + ) -> Result<()> { + use super::readers::git_history::{self, GitReadConfig}; + + let git_cfg = GitReadConfig { + author_emails: self.persona.author_emails.clone(), + batch_size: self.persona.git.batch_size, + max_commits: self.persona.git.max_commits, + diff_sample_cap: self.persona.git.diff_sample_cap, + diff_size_cap_bytes: self.persona.git.diff_size_cap_bytes, + small_commit_max_files: self.persona.git.small_commit_max_files, + }; + let author_hash = author_set_hash(&self.persona.author_emails); + + // Read all qualifying repos into a pending list (serial, cheap), then + // digest concurrently + fold serially. A repo's HEAD watermark is + // attached to the LAST of its sessions, so it is only committed once the + // whole repo has been folded (a budget-truncated repo re-scans next run). + let mut pending: Vec = Vec::new(); + for repo in git_history::discover(&self.persona.project_roots) { + report.files_seen += 1; + let head = match git_head_sha(&repo) { + Some(h) => format!("{h}:{author_hash}"), + None => continue, + }; + let key = state::git_key(&repo); + if mode == RunMode::Incremental + && state::watermark_unchanged(self.store, &key, &head).await? + { + report.sessions_skipped += 1; + continue; + } + let sessions = match git_history::read_repo(&repo, &git_cfg) { + Ok(s) => s, + Err(_) => continue, + }; + for session in &sessions { + report.evidence_units += session.evidence.len(); + } + if sessions.is_empty() { + // No author commits — watermark immediately (nothing to digest). + state::record_watermark(self.store, &key, &head).await?; + continue; + } + let head_value = serde_json::Value::String(head); + let last = sessions.len() - 1; + for (i, session) in sessions.into_iter().enumerate() { + let commit = (i == last).then(|| (key.clone(), head_value.clone())); + pending.push(Pending { session, commit }); + } + } + self.digest_and_fold(pending, asks, state, budget, report) + .await + } +} + +/// A read-but-not-yet-digested session plus an optional state commit (cursor or +/// watermark) applied only after the session is folded — so a budget-truncated +/// session is re-processed on the next run. +struct Pending { + session: RawSession, + commit: Option<(String, serde_json::Value)>, +} + +/// Dispatch to the right transcript reader by source-kind tag. +fn read_transcript(kind: &str, path: &Path) -> Result { + match kind { + "claude_code" => claude_code::read_session(path), + "codex" => codex::read_session(path), + other => anyhow::bail!("unknown transcript kind {other}"), + } +} + +/// File mtime in millis for oldest-first ordering (0 when unknown). +fn file_mtime_ms(path: &Path) -> i64 { + state::FileCursor::of(path).map(|c| c.mtime_ms).unwrap_or(0) +} + +/// Stable short hash of the author-email set, so changing it forces a re-scan. +#[cfg(feature = "git-diff")] +fn author_set_hash(emails: &[String]) -> String { + use sha2::{Digest, Sha256}; + let mut sorted: Vec = emails.iter().map(|e| e.to_lowercase()).collect(); + sorted.sort(); + let mut h = Sha256::new(); + h.update(sorted.join(",").as_bytes()); + h.finalize() + .iter() + .take(4) + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Current HEAD sha of a repo, or `None` for an empty/broken repo. +#[cfg(feature = "git-diff")] +fn git_head_sha(repo: &Path) -> Option { + let repo = git2::Repository::open(repo).ok()?; + let head = repo.head().ok()?; + head.target().map(|oid| oid.to_string()) +} + +#[cfg(test)] +#[path = "pipeline_tests.rs"] +mod tests; diff --git a/src/memory/persona/pipeline_tests.rs b/src/memory/persona/pipeline_tests.rs new file mode 100644 index 0000000..f7c6a28 --- /dev/null +++ b/src/memory/persona/pipeline_tests.rs @@ -0,0 +1,256 @@ +//! End-to-end pipeline tests: backfill → compile, incremental skip/resume, and +//! run-budget cutoff — all on the offline mock-provider + ConcatSummariser path. + +use super::*; +use async_trait::async_trait; +use std::io::Write; +use tempfile::TempDir; + +use crate::memory::config::MemoryConfig; +use crate::memory::persona::config::PersonaConfig; +use crate::memory::persona::state::FileStateStore; +use crate::memory::score::extract::{ChatPrompt, ChatProvider}; +use crate::memory::tree::summarise::ConcatSummariser; + +struct MockChat; +#[async_trait] +impl ChatProvider for MockChat { + fn name(&self) -> &str { + "mock" + } + async fn chat_for_json(&self, _p: &ChatPrompt) -> anyhow::Result { + Ok(r#"{"observations":[ + {"facet":"workflow","observation":"Commits small and often","quote":"commit","tier":"t2"}, + {"facet":"communication","observation":"Terse and direct","quote":"do X","tier":"t2"} + ]}"# + .into()) + } +} + +/// A provider that always hard-fails (e.g. budget exhausted / auth error). +struct FailChat; +#[async_trait] +impl ChatProvider for FailChat { + fn name(&self) -> &str { + "fail" + } + async fn chat_for_json(&self, _p: &ChatPrompt) -> anyhow::Result { + anyhow::bail!("503 upstream unavailable") + } +} + +fn user_turn(session: &str, ts: &str, text: &str) -> String { + format!( + r#"{{"type":"user","isSidechain":false,"cwd":"/work/demo","sessionId":"{session}","timestamp":"{ts}","message":{{"role":"user","content":"{text}"}}}}"# + ) +} + +/// Build a workspace + persona config with two transcripts and one instruction +/// file. Returns (workspace tempdir, sources tempdir, MemoryConfig, PersonaConfig). +fn setup() -> (TempDir, TempDir, MemoryConfig, PersonaConfig) { + let ws = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + + // Two Claude Code transcripts. + let cc_root = src.path().join("claude/projects/-work-demo"); + std::fs::create_dir_all(&cc_root).unwrap(); + for (i, name) in ["a.jsonl", "b.jsonl"].iter().enumerate() { + let mut f = std::fs::File::create(cc_root.join(name)).unwrap(); + writeln!( + f, + "{}", + user_turn( + "s1", + &format!("2026-07-0{}T10:00:00.000Z", i + 1), + "implement the thing and commit small" + ) + ) + .unwrap(); + } + + // One instruction file under project_roots. + let proj = src.path().join("proj"); + std::fs::create_dir_all(&proj).unwrap(); + std::fs::write( + proj.join("CLAUDE.md"), + "- Always branch before writing code.\n- Commit regularly.\n", + ) + .unwrap(); + + let cfg = MemoryConfig::new(ws.path()); + let mut persona = PersonaConfig::with_home(src.path(), "me@example.com"); + persona.claude_code_root = Some(src.path().join("claude/projects")); + persona.codex_root = None; + persona.project_roots = vec![proj]; + persona.global_instruction_files = vec![]; + (ws, src, cfg, persona) +} + +#[tokio::test] +async fn backfill_then_incremental_resume() { + let (ws, _src, cfg, persona) = setup(); + let provider = MockChat; + let summariser = ConcatSummariser::new(); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + let pipeline = Pipeline { + config: &cfg, + persona: &persona, + provider: &provider, + summariser: &summariser, + store: &store, + }; + + // Backfill: both transcripts digested, instruction rules folded, pack written. + let report = pipeline.run(RunMode::Backfill).await.unwrap(); + assert_eq!(report.sessions_processed, 2, "both transcripts digested"); + assert_eq!(report.directives_folded, 2, "two instruction rules"); + assert!(report.observations >= 2); + let pack = std::fs::read_to_string(report.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack.contains("# Persona: me@example.com")); + assert!(pack.contains("## Workflow")); + assert!(pack.contains("## Directives")); + assert!(pack.contains("Always branch before writing code.")); + + // Incremental: transcripts are cursor-skipped (no new digests). Instruction + // files are always re-read (cheap, no LLM) so edits/removals are reflected. + let report2 = pipeline.run(RunMode::Incremental).await.unwrap(); + assert_eq!(report2.sessions_processed, 0, "unchanged → no re-digest"); + assert!( + report2.sessions_skipped >= 2, + "the 2 transcripts are skipped" + ); + // Directives are rebuilt from the re-read instruction file and still appear. + let pack2 = std::fs::read_to_string(report2.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack2.contains("Always branch before writing code.")); +} + +#[tokio::test] +async fn budget_cutoff_checkpoints() { + let (ws, _src, cfg, mut persona) = setup(); + persona.run_budget.max_sessions = 1; // only one transcript may digest + let provider = MockChat; + let summariser = ConcatSummariser::new(); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + let pipeline = Pipeline { + config: &cfg, + persona: &persona, + provider: &provider, + summariser: &summariser, + store: &store, + }; + let report = pipeline.run(RunMode::Backfill).await.unwrap(); + assert_eq!(report.sessions_processed, 1); + assert!(report.budget_hit, "budget should have stopped the run"); + // The pack is still compiled from what was processed (clean checkpoint). + assert!(report.pack_path.is_some()); +} + +#[tokio::test] +async fn compile_only_reassembles_without_llm() { + let (ws, _src, cfg, persona) = setup(); + let provider = MockChat; + let summariser = ConcatSummariser::new(); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + let pipeline = Pipeline { + config: &cfg, + persona: &persona, + provider: &provider, + summariser: &summariser, + store: &store, + }; + pipeline.run(RunMode::Backfill).await.unwrap(); + + // compile_only reads the existing facet-tree roots and rewrites the pack. + let path = pipeline.compile_only().unwrap(); + let pack = std::fs::read_to_string(path).unwrap(); + assert!(pack.contains("# Persona: me@example.com")); + assert!(pack.contains("## Directives")); +} + +#[tokio::test] +async fn hard_provider_failure_does_not_commit_cursor() { + // A hard provider failure must NOT checkpoint the transcript, so a later run + // with a working provider re-processes it (evidence isn't lost). + let (ws, _src, cfg, persona) = setup(); + let summariser = ConcatSummariser::new(); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + let failing = FailChat; + let first = Pipeline { + config: &cfg, + persona: &persona, + provider: &failing, + summariser: &summariser, + store: &store, + } + .run(RunMode::Backfill) + .await + .unwrap(); + assert_eq!(first.sessions_processed, 0, "all digests failed"); + assert_eq!(first.sessions_failed, 2, "both transcripts failed"); + assert_eq!(first.observations, 0); + // Instruction directives still land (no LLM needed). + let pack = std::fs::read_to_string(first.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack.contains("Always branch before writing code.")); + + // A working provider re-processes the un-committed transcripts. + let good = MockChat; + let second = Pipeline { + config: &cfg, + persona: &persona, + provider: &good, + summariser: &summariser, + store: &store, + } + .run(RunMode::Incremental) + .await + .unwrap(); + assert_eq!(second.sessions_processed, 2, "failed sessions were retried"); + assert!(second.observations >= 2); +} + +#[tokio::test] +async fn removed_directive_drops_out_on_rerun() { + // Editing an instruction file (removing a rule) must drop the stale rule + // from the pack — directives are rebuilt fresh, not appended. + let (ws, src, cfg, persona) = setup(); + let provider = MockChat; + let summariser = ConcatSummariser::new(); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + let claude_md = src.path().join("proj/CLAUDE.md"); + + let first = Pipeline { + config: &cfg, + persona: &persona, + provider: &provider, + summariser: &summariser, + store: &store, + } + .run(RunMode::Backfill) + .await + .unwrap(); + let pack1 = std::fs::read_to_string(first.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack1.contains("Always branch before writing code.")); + assert!(pack1.contains("Commit regularly.")); + + // Remove the "Commit regularly." rule and re-run incrementally. + std::fs::write(&claude_md, "- Always branch before writing code.\n").unwrap(); + let second = Pipeline { + config: &cfg, + persona: &persona, + provider: &provider, + summariser: &summariser, + store: &store, + } + .run(RunMode::Incremental) + .await + .unwrap(); + let pack2 = std::fs::read_to_string(second.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack2.contains("Always branch before writing code.")); + assert!( + !pack2.contains("Commit regularly."), + "removed directive must not persist:\n{pack2}" + ); +} diff --git a/src/memory/persona/readers/claude_code.rs b/src/memory/persona/readers/claude_code.rs new file mode 100644 index 0000000..648a29d --- /dev/null +++ b/src/memory/persona/readers/claude_code.rs @@ -0,0 +1,246 @@ +//! Claude Code transcript reader (doc 06 §6.4, Family A). +//! +//! Discovers `~/.claude/projects//.jsonl` event streams and +//! extracts **user-authored turns only**: genuine prompts (T2), corrections / +//! interrupts (T1), and slash-command habits (T2). It drops the machine — +//! assistant prose, tool results, reasoning, sidechains (`isSidechain`), meta +//! turns (`isMeta`), and system-reminder / local-command scaffolding — which is +//! the ≥95% byte reduction the persona pipeline relies on. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use walkdir::WalkDir; + +use super::super::types::{EvidenceSource, EvidenceTier, PersonaEvidence, PersonaSourceKind}; +use super::{jsonl, looks_like_correction, RawSession}; + +/// Max chars of preceding assistant text carried into a correction excerpt so +/// the digest model can see what was being corrected (§6.4). +const ASSISTANT_CONTEXT_CHARS: usize = 200; + +/// Discover every Claude Code transcript file under `root` (typically +/// `~/.claude/projects`). Returns paths sorted for deterministic ordering. +pub fn discover(root: &Path) -> Vec { + let mut files: Vec = WalkDir::new(root) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .map(|e| e.into_path()) + .filter(|p| p.extension().is_some_and(|x| x == "jsonl")) + .collect(); + files.sort(); + files +} + +/// Parse one Claude Code transcript file into a [`RawSession`]. +pub fn read_session(path: &Path) -> Result { + let mut source = EvidenceSource::new(PersonaSourceKind::ClaudeCode) + .with_path(path.to_string_lossy().to_string()); + // Provisional scope from the project-slug directory; refined by `cwd` below. + if let Some(slug) = path.parent().and_then(|p| p.file_name()) { + source = source.with_scope(slug.to_string_lossy().to_string()); + } + if let Some(stem) = path.file_stem() { + source = source.with_session(stem.to_string_lossy().to_string()); + } + + let mut session = RawSession::new(source); + let mut last_assistant: Option = None; + let mut cwd_scope: Option = None; + let mut session_id: Option = None; + let mut pending: Vec<(DateTime, EvidenceTier, String)> = Vec::new(); + + let raw_bytes = jsonl::for_each_json_line(path, |_len, v| { + let ty = v.get("type").and_then(|t| t.as_str()).unwrap_or(""); + // Learn provenance from any event that carries it. + if cwd_scope.is_none() { + if let Some(cwd) = v.get("cwd").and_then(|c| c.as_str()) { + cwd_scope = Some(cwd.to_string()); + } + } + if session_id.is_none() { + if let Some(sid) = v.get("sessionId").and_then(|c| c.as_str()) { + session_id = Some(sid.to_string()); + } + } + + match ty { + "assistant" => { + if let Some(text) = assistant_text(v) { + last_assistant = Some(text); + } + } + "user" => { + // Drop sidechains (subagent traffic) and meta turns outright. + if v.get("isSidechain") + .and_then(|b| b.as_bool()) + .unwrap_or(false) + { + return; + } + if v.get("isMeta").and_then(|b| b.as_bool()).unwrap_or(false) { + return; + } + let content = match v.get("message").and_then(|m| m.get("content")) { + Some(c) => c, + None => return, + }; + let text = match user_text(content) { + Some(t) => t, + None => return, // synthetic tool-result turn or empty + }; + let ts = event_timestamp(v).unwrap_or_else(Utc::now); + let (tier, excerpt) = if looks_like_correction(&text) { + ( + EvidenceTier::T1, + with_assistant_context(&last_assistant, &text), + ) + } else { + (EvidenceTier::T2, format!("user: {text}")) + }; + pending.push((ts, tier, excerpt)); + } + _ => {} + } + })?; + + // Prefer the real cwd as scope; keep the session id. + if let Some(cwd) = cwd_scope { + session.source = session.source.with_scope(cwd); + } + if let Some(sid) = session_id { + session.source = session.source.with_session(sid); + } + let src = session.source.clone(); + for (ts, tier, excerpt) in pending { + session.push(PersonaEvidence::new( + src.clone(), + ts, + tier, + &excerpt, + vec![], + )); + } + session.raw_bytes = raw_bytes; + Ok(session) +} + +/// Parse an event's RFC3339 `timestamp` field into UTC. +fn event_timestamp(v: &serde_json::Value) -> Option> { + let s = v.get("timestamp").and_then(|t| t.as_str())?; + DateTime::parse_from_rfc3339(s) + .ok() + .map(|t| t.with_timezone(&Utc)) +} + +/// Extract the leading assistant text (first [`ASSISTANT_CONTEXT_CHARS`] chars), +/// ignoring thinking / tool-use blocks. `None` if the turn has no text. +fn assistant_text(v: &serde_json::Value) -> Option { + let content = v.get("message").and_then(|m| m.get("content"))?; + let text = collect_text_blocks(content); + let text = text.trim(); + if text.is_empty() { + return None; + } + Some(text.chars().take(ASSISTANT_CONTEXT_CHARS).collect()) +} + +/// Prefix a correction excerpt with the preceding assistant snippet so the +/// digest can see what was being corrected. +fn with_assistant_context(last_assistant: &Option, user_text: &str) -> String { + match last_assistant { + Some(ctx) if !ctx.trim().is_empty() => { + format!("user (interrupting agent that said \"{ctx}\"): {user_text}") + } + _ => format!("user (interrupt): {user_text}"), + } +} + +/// Extract genuine user text from a `message.content`, or `None` if the turn is +/// synthetic (tool-result) or empty after cleaning. +fn user_text(content: &serde_json::Value) -> Option { + let raw = match content { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Array(_) => { + // A list with any tool_result block is a synthetic user turn. + if has_block_type(content, "tool_result") { + return None; + } + collect_text_blocks(content) + } + _ => return None, + }; + let cleaned = clean_user_text(&raw); + if cleaned.is_empty() { + None + } else { + Some(cleaned) + } +} + +/// True if a content block array contains a block of `kind`. +fn has_block_type(content: &serde_json::Value, kind: &str) -> bool { + content + .as_array() + .map(|arr| { + arr.iter() + .any(|b| b.get("type").and_then(|t| t.as_str()) == Some(kind)) + }) + .unwrap_or(false) +} + +/// Concatenate the `text` fields of any `text`-typed blocks in a content array. +fn collect_text_blocks(content: &serde_json::Value) -> String { + match content { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Array(arr) => arr + .iter() + .filter(|b| b.get("type").and_then(|t| t.as_str()) == Some("text")) + .filter_map(|b| b.get("text").and_then(|t| t.as_str())) + .collect::>() + .join("\n"), + _ => String::new(), + } +} + +/// Strip harness scaffolding (system reminders, local-command wrappers) and +/// return the human-authored remainder. Empty string means "not user text". +fn clean_user_text(raw: &str) -> String { + let s = raw.trim(); + // Local-command / caveat wrappers are execution scaffolding, not prompts. + if s.starts_with("… spans. + let without_reminders = strip_tag_spans(s, "system-reminder"); + without_reminders.trim().to_string() +} + +/// Remove every `…` span (case-sensitive) from `text`. +fn strip_tag_spans(text: &str, tag: &str) -> String { + let open = format!("<{tag}>"); + let close = format!(""); + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find(&open) { + out.push_str(&rest[..start]); + if let Some(end) = rest[start..].find(&close) { + rest = &rest[start + end + close.len()..]; + } else { + rest = &rest[start + open.len()..]; + } + } + out.push_str(rest); + out +} + +#[cfg(test)] +#[path = "claude_code_tests.rs"] +mod tests; diff --git a/src/memory/persona/readers/claude_code_tests.rs b/src/memory/persona/readers/claude_code_tests.rs new file mode 100644 index 0000000..c97564f --- /dev/null +++ b/src/memory/persona/readers/claude_code_tests.rs @@ -0,0 +1,115 @@ +//! Tests for the Claude Code transcript reader. + +use super::*; +use crate::memory::persona::types::EvidenceTier; +use std::io::Write; +use tempfile::TempDir; + +/// Write a `.jsonl` fixture and return its path (kept alive by `dir`). +fn write_fixture(dir: &TempDir, name: &str, lines: &[&str]) -> std::path::PathBuf { + let proj = dir.path().join("-home-droid-work-demo"); + std::fs::create_dir_all(&proj).unwrap(); + let path = proj.join(name); + let mut f = std::fs::File::create(&path).unwrap(); + for l in lines { + writeln!(f, "{l}").unwrap(); + } + path +} + +const REAL_PROMPT: &str = r#"{"type":"user","isSidechain":false,"cwd":"/home/droid/work/demo","sessionId":"sess-1","timestamp":"2026-07-01T10:00:00.000Z","message":{"role":"user","content":"implement the parser and add a regression test"}}"#; +const CORRECTION: &str = r#"{"type":"user","isSidechain":false,"sessionId":"sess-1","timestamp":"2026-07-01T10:05:00.000Z","message":{"role":"user","content":"no, use a streaming approach instead"}}"#; +const ASSISTANT: &str = r#"{"type":"assistant","timestamp":"2026-07-01T10:04:00.000Z","message":{"role":"assistant","content":[{"type":"thinking","thinking":"hmm"},{"type":"text","text":"I loaded the whole file into memory."}]}}"#; +const TOOL_RESULT: &str = r#"{"type":"user","isSidechain":false,"message":{"role":"user","content":[{"type":"tool_result","content":"stdout blob"}]}}"#; +const SIDECHAIN: &str = + r#"{"type":"user","isSidechain":true,"message":{"role":"user","content":"subagent prompt"}}"#; +const META: &str = + r#"{"type":"user","isMeta":true,"message":{"role":"user","content":"meta note"}}"#; +const CAVEAT: &str = r#"{"type":"user","isSidechain":false,"message":{"role":"user","content":"Caveat..."}}"#; +const REMINDER: &str = r#"{"type":"user","isSidechain":false,"timestamp":"2026-07-01T10:06:00.000Z","message":{"role":"user","content":"ship it background noise"}}"#; + +#[test] +fn extracts_only_user_authored_turns() { + let dir = TempDir::new().unwrap(); + let path = write_fixture( + &dir, + "abc.jsonl", + &[ + REAL_PROMPT, + ASSISTANT, + CORRECTION, + TOOL_RESULT, + SIDECHAIN, + META, + CAVEAT, + REMINDER, + ], + ); + let session = read_session(&path).unwrap(); + + // Four genuine user turns survive: prompt, correction, and the reminder- + // wrapped "ship it". (tool_result, sidechain, meta, caveat all dropped.) + let excerpts: Vec<&str> = session.evidence.iter().map(|e| e.excerpt()).collect(); + assert_eq!(session.evidence.len(), 3, "got: {excerpts:?}"); + + // Correction is tiered T1 and carries the preceding assistant context. + let corr = session + .evidence + .iter() + .find(|e| e.tier == EvidenceTier::T1) + .expect("a T1 correction"); + assert!(corr.excerpt().contains("streaming approach")); + assert!( + corr.excerpt().contains("whole file into memory"), + "correction should carry assistant context: {}", + corr.excerpt() + ); + + // system-reminder scaffolding is stripped from the kept prompt. + assert!(session + .evidence + .iter() + .any(|e| e.excerpt().contains("ship it") && !e.excerpt().contains("background noise"))); +} + +#[test] +fn provenance_prefers_cwd_and_session_id() { + let dir = TempDir::new().unwrap(); + let path = write_fixture(&dir, "abc.jsonl", &[REAL_PROMPT]); + let session = read_session(&path).unwrap(); + assert_eq!( + session.source.scope.as_deref(), + Some("/home/droid/work/demo") + ); + assert_eq!(session.source.session_id.as_deref(), Some("sess-1")); +} + +#[test] +fn tool_result_heavy_session_reduces_over_95_percent() { + let dir = TempDir::new().unwrap(); + // One short prompt amidst a pile of big tool-result blobs. + let big = format!( + r#"{{"type":"user","isSidechain":false,"message":{{"role":"user","content":[{{"type":"tool_result","content":"{}"}}]}}}}"#, + "x".repeat(5000) + ); + let mut lines: Vec<&str> = vec![REAL_PROMPT]; + for _ in 0..20 { + lines.push(&big); + } + let path = write_fixture(&dir, "abc.jsonl", &lines); + let session = read_session(&path).unwrap(); + assert!( + session.reduction_ratio() > 0.95, + "reduction was only {:.3}", + session.reduction_ratio() + ); +} + +#[test] +fn discover_finds_jsonl_recursively() { + let dir = TempDir::new().unwrap(); + write_fixture(&dir, "one.jsonl", &[REAL_PROMPT]); + write_fixture(&dir, "two.jsonl", &[REAL_PROMPT]); + let found = discover(dir.path()); + assert_eq!(found.len(), 2); +} diff --git a/src/memory/persona/readers/codex.rs b/src/memory/persona/readers/codex.rs new file mode 100644 index 0000000..4a4752a --- /dev/null +++ b/src/memory/persona/readers/codex.rs @@ -0,0 +1,179 @@ +//! Codex rollout transcript reader (doc 06 §6.4, Family A). +//! +//! Discovers `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and extracts +//! user-authored turns from `response_item` `message` events with +//! `role == "user"`, dropping the `developer` role (permissions / +//! `base_instructions` — vendor prompts) and synthetic wrappers +//! (``, ``, ``, +//! ``). Session provenance (cwd, id) comes from `session_meta`. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use walkdir::WalkDir; + +use super::super::types::{EvidenceSource, EvidenceTier, PersonaEvidence, PersonaSourceKind}; +use super::{jsonl, looks_like_correction, RawSession}; + +const ASSISTANT_CONTEXT_CHARS: usize = 200; + +/// Discover every Codex rollout file under `root` (typically +/// `~/.codex/sessions`). Sorted for deterministic ordering. +pub fn discover(root: &Path) -> Vec { + let mut files: Vec = WalkDir::new(root) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .map(|e| e.into_path()) + .filter(|p| { + p.extension().is_some_and(|x| x == "jsonl") + && p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("rollout-")) + }) + .collect(); + files.sort(); + files +} + +/// Parse one Codex rollout file into a [`RawSession`]. +pub fn read_session(path: &Path) -> Result { + let mut source = + EvidenceSource::new(PersonaSourceKind::Codex).with_path(path.to_string_lossy().to_string()); + if let Some(stem) = path.file_stem() { + source = source.with_session(stem.to_string_lossy().to_string()); + } + + let mut session = RawSession::new(source); + let mut last_assistant: Option = None; + let mut cwd_scope: Option = None; + let mut session_id: Option = None; + let mut pending: Vec<(DateTime, EvidenceTier, String)> = Vec::new(); + + let raw_bytes = jsonl::for_each_json_line(path, |_len, v| { + let ty = v.get("type").and_then(|t| t.as_str()).unwrap_or(""); + let payload = v.get("payload"); + + if ty == "session_meta" { + if let Some(p) = payload { + if let Some(cwd) = p.get("cwd").and_then(|c| c.as_str()) { + cwd_scope = Some(cwd.to_string()); + } + if let Some(id) = p.get("id").and_then(|c| c.as_str()) { + session_id = Some(id.to_string()); + } + } + return; + } + + if ty != "response_item" { + return; + } + let p = match payload { + Some(p) if p.get("type").and_then(|t| t.as_str()) == Some("message") => p, + _ => return, + }; + let role = p.get("role").and_then(|r| r.as_str()).unwrap_or(""); + match role { + "assistant" => { + let text = collect_message_text(p); + let text = text.trim(); + if !text.is_empty() { + last_assistant = Some(text.chars().take(ASSISTANT_CONTEXT_CHARS).collect()); + } + } + "user" => { + let raw = collect_message_text(p); + let text = clean_user_text(&raw); + if text.is_empty() { + return; + } + let ts = event_timestamp(v).unwrap_or_else(Utc::now); + let (tier, excerpt) = if looks_like_correction(&text) { + ( + EvidenceTier::T1, + with_assistant_context(&last_assistant, &text), + ) + } else { + (EvidenceTier::T2, format!("user: {text}")) + }; + pending.push((ts, tier, excerpt)); + } + _ => {} // developer / tool / system — vendor scaffolding + } + })?; + + if let Some(cwd) = cwd_scope { + session.source = session.source.with_scope(cwd); + } + if let Some(sid) = session_id { + session.source = session.source.with_session(sid); + } + let src = session.source.clone(); + for (ts, tier, excerpt) in pending { + session.push(PersonaEvidence::new( + src.clone(), + ts, + tier, + &excerpt, + vec![], + )); + } + session.raw_bytes = raw_bytes; + Ok(session) +} + +fn event_timestamp(v: &serde_json::Value) -> Option> { + let s = v.get("timestamp").and_then(|t| t.as_str())?; + DateTime::parse_from_rfc3339(s) + .ok() + .map(|t| t.with_timezone(&Utc)) +} + +fn with_assistant_context(last_assistant: &Option, user_text: &str) -> String { + match last_assistant { + Some(ctx) if !ctx.trim().is_empty() => { + format!("user (interrupting agent that said \"{ctx}\"): {user_text}") + } + _ => format!("user (interrupt): {user_text}"), + } +} + +/// Concatenate `input_text` / `output_text` / `text` blocks of a Codex message. +fn collect_message_text(msg: &serde_json::Value) -> String { + let content = match msg.get("content").and_then(|c| c.as_array()) { + Some(a) => a, + None => return String::new(), + }; + content + .iter() + .filter_map(|b| { + let t = b.get("type").and_then(|t| t.as_str()).unwrap_or(""); + if matches!(t, "input_text" | "output_text" | "text") { + b.get("text").and_then(|t| t.as_str()) + } else { + None + } + }) + .collect::>() + .join("\n") +} + +/// Strip Codex synthetic wrappers; empty string means "not user text". +fn clean_user_text(raw: &str) -> String { + let s = raw.trim(); + if s.starts_with("") + || s.starts_with("") + || s.starts_with("") + || s.starts_with(" std::path::PathBuf { + let path = dir.path().join("rollout-2026-07-01T10-00-00-abc.jsonl"); + let mut f = std::fs::File::create(&path).unwrap(); + for l in lines { + writeln!(f, "{l}").unwrap(); + } + path +} + +const META: &str = r#"{"timestamp":"2026-07-01T10:00:00.000Z","type":"session_meta","payload":{"id":"sess-9","cwd":"/home/droid/work/oh","originator":"codex_exec"}}"#; +const DEVELOPER: &str = r#"{"timestamp":"2026-07-01T10:00:01.000Z","type":"response_item","payload":{"type":"message","role":"developer","content":[{"type":"input_text","text":" you are Codex ..."}]}}"#; +const USER_PROMPT: &str = r#"{"timestamp":"2026-07-01T10:00:02.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"resolve this issue and open a PR"}]}}"#; +const ENV_CTX: &str = r#"{"timestamp":"2026-07-01T10:00:03.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"cwd=/x"}]}}"#; +const SUBAGENT: &str = r#"{"timestamp":"2026-07-01T10:00:04.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"{\"status\":\"shutdown\"}"}]}}"#; +const ASSISTANT: &str = r#"{"timestamp":"2026-07-01T10:00:05.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"I will squash all commits into one."}]}}"#; +const CORRECTION: &str = r#"{"timestamp":"2026-07-01T10:00:06.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"stop, keep the commits separate"}]}}"#; +const TOKEN_EVT: &str = r#"{"timestamp":"2026-07-01T10:00:07.000Z","type":"event_msg","payload":{"type":"token_count","total":123}}"#; + +#[test] +fn extracts_user_turns_excluding_vendor_and_synthetic() { + let dir = TempDir::new().unwrap(); + let path = write_fixture( + &dir, + &[ + META, + DEVELOPER, + USER_PROMPT, + ENV_CTX, + SUBAGENT, + ASSISTANT, + CORRECTION, + TOKEN_EVT, + ], + ); + let session = read_session(&path).unwrap(); + + // Two genuine user turns: the prompt (T2) and the correction (T1). + assert_eq!(session.evidence.len(), 2); + assert!(session + .evidence + .iter() + .any(|e| e.excerpt().contains("resolve this issue"))); + + let corr = session + .evidence + .iter() + .find(|e| e.tier == EvidenceTier::T1) + .expect("T1 correction"); + assert!(corr.excerpt().contains("keep the commits separate")); + assert!( + corr.excerpt().contains("squash all commits"), + "carries assistant context" + ); + + // base_instructions (developer), environment_context, and subagent + // notifications never become evidence. + assert!(!session + .evidence + .iter() + .any(|e| e.excerpt().contains("Codex"))); + assert!(!session + .evidence + .iter() + .any(|e| e.excerpt().contains("environment_context"))); + assert!(!session + .evidence + .iter() + .any(|e| e.excerpt().contains("subagent_notification"))); +} + +#[test] +fn provenance_from_session_meta() { + let dir = TempDir::new().unwrap(); + let path = write_fixture(&dir, &[META, USER_PROMPT]); + let session = read_session(&path).unwrap(); + assert_eq!(session.source.scope.as_deref(), Some("/home/droid/work/oh")); + assert_eq!(session.source.session_id.as_deref(), Some("sess-9")); + assert_eq!(session.source.kind, PersonaSourceKind::Codex); +} + +#[test] +fn discover_matches_rollout_prefix_only() { + let dir = TempDir::new().unwrap(); + write_fixture(&dir, &[META, USER_PROMPT]); + std::fs::File::create(dir.path().join("notes.jsonl")).unwrap(); + let found = discover(dir.path()); + assert_eq!(found.len(), 1); +} diff --git a/src/memory/persona/readers/git_history.rs b/src/memory/persona/readers/git_history.rs new file mode 100644 index 0000000..002c198 --- /dev/null +++ b/src/memory/persona/readers/git_history.rs @@ -0,0 +1,298 @@ +//! Git commit-history reader (doc 06 §6.4, Family C — requires `git-diff`). +//! +//! Over configured repo roots, author-filtered by a configured email set, this +//! reader emits two evidence streams: +//! +//! - **Message style (T2)** — subject/body/stats, batched (~100 commits per +//! unit) so the digest infers conventions (Conventional Commits, tense, +//! subject length, body habits), cadence, and granularity. +//! - **Code style (T3)** — a bounded sample of small commit diffs (small +//! commits first, hard size/count caps) for naming/comment/test signals. +//! Explicitly T3: merged code includes agent-written code, so it can only +//! corroborate. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use git2::{DiffOptions, Repository, Sort}; +use walkdir::WalkDir; + +use super::super::types::{EvidenceSource, EvidenceTier, PersonaEvidence, PersonaSourceKind}; +use super::{keep_walk_entry, RawSession}; + +/// Tunables for the git reader. +#[derive(Debug, Clone)] +pub struct GitReadConfig { + /// Author emails to include (case-insensitive). Empty = accept all authors. + pub author_emails: Vec, + /// Commit-message evidence units per digest batch (~100). + pub batch_size: usize, + /// Max author commits scanned per repo (newest bounded). + pub max_commits: usize, + /// Max sampled diffs per repo (T3). + pub diff_sample_cap: usize, + /// Max bytes of any single sampled diff kept. + pub diff_size_cap_bytes: usize, + /// A commit qualifies for diff sampling only if it changed at most this many + /// files (small commits first). + pub small_commit_max_files: usize, +} + +impl Default for GitReadConfig { + fn default() -> Self { + Self { + author_emails: Vec::new(), + batch_size: 100, + max_commits: 2000, + diff_sample_cap: 20, + diff_size_cap_bytes: 4000, + small_commit_max_files: 3, + } + } +} + +/// Discover git repositories under `roots` (directories containing a `.git` +/// entry), bounded in depth. Nested repos are each returned once. +pub fn discover(roots: &[PathBuf]) -> Vec { + let mut repos: Vec = Vec::new(); + for root in roots { + for entry in WalkDir::new(root) + .max_depth(4) + .into_iter() + .filter_entry(keep_walk_entry) + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_dir()) + { + if entry.path().join(".git").exists() { + repos.push(entry.path().to_path_buf()); + } + } + } + repos.sort(); + repos.dedup(); + repos +} + +/// One author commit's extracted facts. +struct CommitFacts { + id: String, + when: DateTime, + subject: String, + body: String, + files: usize, + insertions: usize, + deletions: usize, +} + +/// Read a repo into digest-ready sessions: message-style batches (T2) followed +/// by a bounded sample of small-commit diffs (T3). Empty repos yield nothing. +pub fn read_repo(repo_path: &Path, cfg: &GitReadConfig) -> Result> { + let repo = Repository::open(repo_path) + .with_context(|| format!("open git repo {}", repo_path.display()))?; + let repo_name = repo_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| "repo".to_string()); + + let mut walk = repo.revwalk()?; + walk.push_head().ok(); // empty repo / detached HEAD → no commits + walk.set_sorting(Sort::TIME)?; + + let emails: Vec = cfg.author_emails.iter().map(|e| e.to_lowercase()).collect(); + let mut commits: Vec = Vec::new(); + + for oid in walk.flatten() { + if commits.len() >= cfg.max_commits { + break; + } + let commit = match repo.find_commit(oid) { + Ok(c) => c, + Err(_) => continue, + }; + let author_email = commit.author().email().unwrap_or("").to_lowercase(); + if !emails.is_empty() && !emails.contains(&author_email) { + continue; + } + let (files, insertions, deletions) = commit_stats(&repo, &commit); + let full = commit.message().unwrap_or(""); + let subject = full.lines().next().unwrap_or("").trim().to_string(); + let body = full + .lines() + .skip(1) + .collect::>() + .join("\n") + .trim() + .to_string(); + let when = DateTime::from_timestamp(commit.time().seconds(), 0).unwrap_or_else(Utc::now); + commits.push(CommitFacts { + id: oid.to_string(), + when, + subject, + body, + files, + insertions, + deletions, + }); + } + + if commits.is_empty() { + return Ok(Vec::new()); + } + // Oldest-first so trees fold chronologically at backfill. + commits.reverse(); + + let mut sessions: Vec = Vec::new(); + build_message_batches(&repo_name, repo_path, &commits, cfg, &mut sessions); + build_diff_sample(&repo, &repo_name, repo_path, &commits, cfg, &mut sessions); + Ok(sessions) +} + +/// Files-changed / insertions / deletions for a commit vs. its first parent. +fn commit_stats(repo: &Repository, commit: &git2::Commit) -> (usize, usize, usize) { + let to_tree = match commit.tree() { + Ok(t) => t, + Err(_) => return (0, 0, 0), + }; + let from_tree = commit.parent(0).ok().and_then(|p| p.tree().ok()); + let mut opts = DiffOptions::new(); + let diff = match repo.diff_tree_to_tree(from_tree.as_ref(), Some(&to_tree), Some(&mut opts)) { + Ok(d) => d, + Err(_) => return (0, 0, 0), + }; + match diff.stats() { + Ok(s) => (s.files_changed(), s.insertions(), s.deletions()), + Err(_) => (0, 0, 0), + } +} + +/// Emit T2 message-style evidence, batched. +fn build_message_batches( + repo_name: &str, + repo_path: &Path, + commits: &[CommitFacts], + cfg: &GitReadConfig, + sessions: &mut Vec, +) { + for (batch_idx, chunk) in commits.chunks(cfg.batch_size.max(1)).enumerate() { + let source = EvidenceSource::new(PersonaSourceKind::GitHistory) + .with_scope(repo_name.to_string()) + .with_session(format!("messages-{batch_idx}")) + .with_path(repo_path.to_string_lossy().to_string()); + let mut session = RawSession::new(source.clone()); + for c in chunk { + let mut excerpt = format!( + "commit {}: {} [{} files, +{} -{}]", + &c.id[..c.id.len().min(8)], + c.subject, + c.files, + c.insertions, + c.deletions + ); + if !c.body.is_empty() { + let body: String = c.body.chars().take(400).collect(); + excerpt.push_str("\n body: "); + excerpt.push_str(&body); + } + session.push(PersonaEvidence::new( + source.clone(), + c.when, + EvidenceTier::T2, + &excerpt, + vec![], + )); + } + session.raw_bytes = session.kept_bytes; + if !session.is_empty() { + sessions.push(session); + } + } +} + +/// Emit a bounded T3 sample of small-commit diffs. +fn build_diff_sample( + repo: &Repository, + repo_name: &str, + repo_path: &Path, + commits: &[CommitFacts], + cfg: &GitReadConfig, + sessions: &mut Vec, +) { + if cfg.diff_sample_cap == 0 { + return; + } + // Small commits first (fewest files, then fewest line changes). + let mut small: Vec<&CommitFacts> = commits + .iter() + .filter(|c| c.files > 0 && c.files <= cfg.small_commit_max_files) + .collect(); + small.sort_by_key(|c| (c.files, c.insertions + c.deletions)); + small.truncate(cfg.diff_sample_cap); + + if small.is_empty() { + return; + } + let source = EvidenceSource::new(PersonaSourceKind::GitHistory) + .with_scope(repo_name.to_string()) + .with_session("diffs") + .with_path(repo_path.to_string_lossy().to_string()); + let mut session = RawSession::new(source.clone()); + for c in small { + if let Some(patch) = sampled_patch(repo, &c.id, cfg.diff_size_cap_bytes) { + let excerpt = format!("diff for \"{}\":\n{patch}", c.subject); + session.push(PersonaEvidence::new( + source.clone(), + c.when, + EvidenceTier::T3, + &excerpt, + vec![], + )); + } + } + session.raw_bytes = session.kept_bytes; + if !session.is_empty() { + sessions.push(session); + } +} + +/// Build a truncated unified diff for a commit vs. its first parent, capped at +/// `size_cap` bytes. +fn sampled_patch(repo: &Repository, oid_hex: &str, size_cap: usize) -> Option { + let oid = git2::Oid::from_str(oid_hex).ok()?; + let commit = repo.find_commit(oid).ok()?; + let to_tree = commit.tree().ok()?; + let from_tree = commit.parent(0).ok().and_then(|p| p.tree().ok()); + let mut opts = DiffOptions::new(); + opts.context_lines(1); + let diff = repo + .diff_tree_to_tree(from_tree.as_ref(), Some(&to_tree), Some(&mut opts)) + .ok()?; + + let mut out = String::new(); + let mut truncated = false; + diff.print(git2::DiffFormat::Patch, |_delta, _hunk, line| { + if out.len() >= size_cap { + truncated = true; + return true; // keep iterating cheaply; we just stop appending + } + let origin = line.origin(); + if matches!(origin, '+' | '-' | ' ') { + out.push(origin); + } + out.push_str(&String::from_utf8_lossy(line.content())); + true + }) + .ok()?; + if out.trim().is_empty() { + return None; + } + if truncated { + out.truncate(size_cap); + out.push_str("\n… [diff truncated]"); + } + Some(out) +} + +#[cfg(test)] +#[path = "git_history_tests.rs"] +mod tests; diff --git a/src/memory/persona/readers/git_history_tests.rs b/src/memory/persona/readers/git_history_tests.rs new file mode 100644 index 0000000..15f815e --- /dev/null +++ b/src/memory/persona/readers/git_history_tests.rs @@ -0,0 +1,164 @@ +//! Tests for the git-history reader (synthetic repo built with git2). + +use super::*; +use crate::memory::persona::types::EvidenceTier; +use git2::{Repository, Signature, Time}; +use std::cell::Cell; +use tempfile::TempDir; + +thread_local! { + /// Monotonic commit clock so TIME-sort ordering is unambiguous in tests. + static CLOCK: Cell = const { Cell::new(1_700_000_000) }; +} + +/// Commit `files` (path, contents) as `email` with `message` at a strictly +/// increasing timestamp. +fn commit(repo: &Repository, email: &str, message: &str, files: &[(&str, &str)]) { + let workdir = repo.workdir().unwrap().to_path_buf(); + for (name, contents) in files { + std::fs::write(workdir.join(name), contents).unwrap(); + } + let mut index = repo.index().unwrap(); + for (name, _) in files { + index.add_path(std::path::Path::new(name)).unwrap(); + } + index.write().unwrap(); + let tree_id = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_id).unwrap(); + let secs = CLOCK.with(|c| { + let v = c.get(); + c.set(v + 60); + v + }); + let sig = Signature::new("Test Author", email, &Time::new(secs, 0)).unwrap(); + let parent = repo + .head() + .ok() + .and_then(|h| h.target()) + .and_then(|o| repo.find_commit(o).ok()); + let parents: Vec<&git2::Commit> = parent.iter().collect(); + repo.commit(Some("HEAD"), &sig, &sig, message, &tree, &parents) + .unwrap(); +} + +fn build_repo(dir: &TempDir) -> Repository { + let repo = Repository::init(dir.path()).unwrap(); + commit( + &repo, + "me@work.com", + "feat: add parser\n\nImplements the streaming parser.", + &[("parser.rs", "fn parse() {}\n")], + ); + commit( + &repo, + "me@personal.com", + "fix: handle empty input", + &[("parser.rs", "fn parse() { /* guard */ }\n")], + ); + commit( + &repo, + "someone-else@example.com", + "chore: unrelated", + &[("other.rs", "// not mine\n")], + ); + commit( + &repo, + "me@work.com", + "test: add regression test", + &[("parser_test.rs", "#[test] fn t() {}\n")], + ); + repo +} + +#[test] +fn reads_author_filtered_message_and_diff_evidence() { + let dir = TempDir::new().unwrap(); + let _repo = build_repo(&dir); + + let cfg = GitReadConfig { + author_emails: vec!["me@work.com".into(), "me@personal.com".into()], + batch_size: 100, + diff_sample_cap: 10, + ..Default::default() + }; + let sessions = read_repo(dir.path(), &cfg).unwrap(); + + // Message batch (T2) + diff sample (T3). + let msg: Vec<_> = sessions + .iter() + .flat_map(|s| &s.evidence) + .filter(|e| e.tier == EvidenceTier::T2) + .collect(); + // Three of the four commits are by the configured authors. + assert_eq!( + msg.len(), + 3, + "message evidence: {:?}", + msg.iter().map(|e| e.excerpt()).collect::>() + ); + assert!(msg.iter().any(|e| e.excerpt().contains("feat: add parser"))); + // The unrelated author's commit is excluded. + assert!(!msg.iter().any(|e| e.excerpt().contains("unrelated"))); + + let diffs: Vec<_> = sessions + .iter() + .flat_map(|s| &s.evidence) + .filter(|e| e.tier == EvidenceTier::T3) + .collect(); + assert!(!diffs.is_empty(), "expected sampled diff evidence"); + assert!(diffs.iter().any(|e| e.excerpt().contains("fn parse"))); + + // Message batch is oldest-first (chronological folding). + let batch = sessions + .iter() + .find(|s| s.source.session_id.as_deref() == Some("messages-0")) + .unwrap(); + assert!(batch.evidence[0].excerpt().contains("feat: add parser")); +} + +#[test] +fn empty_author_set_accepts_all() { + let dir = TempDir::new().unwrap(); + let _repo = build_repo(&dir); + let cfg = GitReadConfig::default(); // empty author_emails + let sessions = read_repo(dir.path(), &cfg).unwrap(); + let msg_count = sessions + .iter() + .flat_map(|s| &s.evidence) + .filter(|e| e.tier == EvidenceTier::T2) + .count(); + assert_eq!(msg_count, 4, "all four commits accepted"); +} + +#[test] +fn discover_finds_repo() { + let dir = TempDir::new().unwrap(); + Repository::init(dir.path()).unwrap(); + let found = discover(&[dir.path().to_path_buf()]); + assert!(found.iter().any(|p| p == dir.path())); +} + +#[test] +fn diff_size_cap_truncates() { + let dir = TempDir::new().unwrap(); + let repo = Repository::init(dir.path()).unwrap(); + let big = "x = 1;\n".repeat(2000); + commit(&repo, "me@work.com", "feat: big file", &[("big.rs", &big)]); + let cfg = GitReadConfig { + author_emails: vec!["me@work.com".into()], + diff_size_cap_bytes: 500, + small_commit_max_files: 5, + ..Default::default() + }; + let sessions = read_repo(dir.path(), &cfg).unwrap(); + let diff = sessions + .iter() + .flat_map(|s| &s.evidence) + .find(|e| e.tier == EvidenceTier::T3); + if let Some(d) = diff { + assert!( + d.excerpt().contains("truncated"), + "large diff should be truncated" + ); + } +} diff --git a/src/memory/persona/readers/instruction.rs b/src/memory/persona/readers/instruction.rs new file mode 100644 index 0000000..ae23cfc --- /dev/null +++ b/src/memory/persona/readers/instruction.rs @@ -0,0 +1,283 @@ +//! Agent instruction-file reader (doc 06 §6.4, Family B — T0 evidence). +//! +//! Explicitly authored rules are the highest-confidence persona evidence there +//! is. This reader discovers `CLAUDE.md` / `AGENTS.md` / `.cursorrules` / +//! `.github/copilot-instructions.md` across configured roots (plus explicit +//! global files), splits each into rule-granular chunks (headings / bullets / +//! paragraphs), and emits **verbatim** T0 `directives` evidence tagged with +//! global-vs-repo scope. No LLM is involved — these flow into the pack with +//! minimal rewriting. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use sha2::{Digest, Sha256}; +use walkdir::WalkDir; + +use super::super::types::{ + EvidenceSource, EvidenceTier, PersonaEvidence, PersonaFacet, PersonaSourceKind, +}; +use super::{keep_walk_entry, RawSession}; + +/// Filenames recognised as agent instruction files. +const INSTRUCTION_FILENAMES: [&str; 4] = [ + "CLAUDE.md", + "AGENTS.md", + ".cursorrules", + "copilot-instructions.md", +]; + +/// Default depth walked under each configured root. +const DEFAULT_MAX_DEPTH: usize = 6; + +/// Whether a discovered rule applies everywhere or is project-local. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuleScope { + /// A global instruction file (`~/.claude/CLAUDE.md`, `~/.codex/AGENTS.md`). + Global, + /// A repo-scoped instruction file; carries the repo directory name. + Repo(String), +} + +impl RuleScope { + /// Scope string used on evidence provenance. + fn as_scope(&self) -> String { + match self { + RuleScope::Global => "global".to_string(), + RuleScope::Repo(name) => format!("repo({name})"), + } + } +} + +/// A discovered instruction file plus its resolved scope. +#[derive(Debug, Clone)] +pub struct InstructionFile { + /// Absolute path of the file. + pub path: PathBuf, + /// Global vs. repo-scoped. + pub scope: RuleScope, +} + +/// Discover instruction files under `roots` (repo-scoped) plus explicit +/// `globals` (global-scoped). Repo scope is the nearest ancestor directory that +/// contains a `.git` entry; failing that, the file's parent directory name. +pub fn discover(roots: &[PathBuf], globals: &[PathBuf]) -> Vec { + let mut out: Vec = Vec::new(); + + for g in globals { + if g.is_file() { + out.push(InstructionFile { + path: g.clone(), + scope: RuleScope::Global, + }); + } + } + + for root in roots { + for entry in WalkDir::new(root) + .max_depth(DEFAULT_MAX_DEPTH) + .into_iter() + .filter_entry(keep_walk_entry) + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + { + let path = entry.path(); + let name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n, + None => continue, + }; + if !INSTRUCTION_FILENAMES.contains(&name) { + continue; + } + // copilot-instructions.md only counts under a `.github` directory. + if name == "copilot-instructions.md" + && path + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + != Some(".github") + { + continue; + } + let scope = repo_scope_for(path).unwrap_or_else(|| { + RuleScope::Repo( + path.parent() + .and_then(|p| p.file_name()) + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| "unknown".to_string()), + ) + }); + out.push(InstructionFile { + path: path.to_path_buf(), + scope, + }); + } + } + + out.sort_by(|a, b| a.path.cmp(&b.path)); + out.dedup_by(|a, b| a.path == b.path); + out +} + +/// Find the nearest ancestor of `path` that is a git repo root and return it as +/// a [`RuleScope::Repo`] named after that directory. +fn repo_scope_for(path: &Path) -> Option { + let mut dir = path.parent(); + while let Some(d) = dir { + if d.join(".git").exists() { + return Some(RuleScope::Repo( + d.file_name()?.to_string_lossy().to_string(), + )); + } + dir = d.parent(); + } + None +} + +/// Content sha of a file's bytes, for change detection (feeds P9). Hex sha256. +pub fn content_sha(bytes: &[u8]) -> String { + let mut h = Sha256::new(); + h.update(bytes); + h.finalize().iter().map(|b| format!("{b:02x}")).collect() +} + +/// Read one instruction file into a [`RawSession`] of T0 `directives` evidence, +/// one unit per rule-granular chunk (verbatim). +pub fn read_file(file: &InstructionFile) -> Result { + let bytes = std::fs::read(&file.path) + .with_context(|| format!("read instruction file {}", file.path.display()))?; + let text = String::from_utf8_lossy(&bytes); + let mtime = file_mtime(&file.path).unwrap_or_else(Utc::now); + + let source = EvidenceSource::new(PersonaSourceKind::InstructionFile) + .with_scope(file.scope.as_scope()) + .with_path(file.path.to_string_lossy().to_string()); + let mut session = RawSession::new(source.clone()); + session.raw_bytes = bytes.len() as u64; + + for rule in split_rules(&text) { + session.push(PersonaEvidence::new( + source.clone(), + mtime, + EvidenceTier::T0, + &rule, + vec![PersonaFacet::Directives], + )); + } + Ok(session) +} + +/// File modification time as UTC. +fn file_mtime(path: &Path) -> Option> { + let meta = std::fs::metadata(path).ok()?; + let modified = meta.modified().ok()?; + Some(DateTime::::from(modified)) +} + +/// Split markdown into rule-granular, verbatim chunks: each top-level bullet is +/// one rule; consecutive non-bullet lines form a paragraph rule. Headings and +/// fenced code blocks are dropped (headings are structure, not rules). Bullets +/// carry their nested continuation lines so multi-line rules stay intact. +pub fn split_rules(md: &str) -> Vec { + let mut rules: Vec = Vec::new(); + let mut para: Vec = Vec::new(); + let mut bullet: Vec = Vec::new(); + let mut in_fence = false; + + let flush_para = |para: &mut Vec, rules: &mut Vec| { + if !para.is_empty() { + let joined = para.join("\n").trim().to_string(); + if joined.len() >= 3 { + rules.push(joined); + } + para.clear(); + } + }; + let flush_bullet = |bullet: &mut Vec, rules: &mut Vec| { + if !bullet.is_empty() { + let joined = bullet.join("\n").trim().to_string(); + if joined.len() >= 3 { + rules.push(joined); + } + bullet.clear(); + } + }; + + for raw_line in md.lines() { + let line = raw_line.trim_end(); + let trimmed = line.trim_start(); + + if trimmed.starts_with("```") || trimmed.starts_with("~~~") { + in_fence = !in_fence; + flush_para(&mut para, &mut rules); + flush_bullet(&mut bullet, &mut rules); + continue; + } + if in_fence { + continue; + } + if trimmed.is_empty() { + flush_para(&mut para, &mut rules); + flush_bullet(&mut bullet, &mut rules); + continue; + } + if trimmed.starts_with('#') { + flush_para(&mut para, &mut rules); + flush_bullet(&mut bullet, &mut rules); + continue; // heading = structure + } + + let indent = line.len() - trimmed.len(); + if is_bullet(trimmed) { + if indent == 0 { + // New top-level bullet — close the previous one. + flush_para(&mut para, &mut rules); + flush_bullet(&mut bullet, &mut rules); + bullet.push(strip_bullet(trimmed).to_string()); + } else if !bullet.is_empty() { + // Nested bullet — part of the current rule. + bullet.push(trimmed.to_string()); + } else { + bullet.push(strip_bullet(trimmed).to_string()); + } + } else if !bullet.is_empty() { + // Continuation line of the current bullet rule. + bullet.push(trimmed.to_string()); + } else { + para.push(line.to_string()); + } + } + flush_para(&mut para, &mut rules); + flush_bullet(&mut bullet, &mut rules); + rules +} + +fn is_bullet(t: &str) -> bool { + t.starts_with("- ") + || t.starts_with("* ") + || t.starts_with("+ ") + || t.chars().next().is_some_and(|c| c.is_ascii_digit()) + && (t.contains(". ") || t.contains(") ")) +} + +fn strip_bullet(t: &str) -> &str { + if let Some(rest) = t + .strip_prefix("- ") + .or_else(|| t.strip_prefix("* ")) + .or_else(|| t.strip_prefix("+ ")) + { + return rest.trim_start(); + } + // Numbered list: strip the leading "N. " / "N) ". + if let Some(pos) = t.find(['.', ')']) { + if t[..pos].chars().all(|c| c.is_ascii_digit()) { + return t[pos + 1..].trim_start(); + } + } + t +} + +#[cfg(test)] +#[path = "instruction_tests.rs"] +mod tests; diff --git a/src/memory/persona/readers/instruction_tests.rs b/src/memory/persona/readers/instruction_tests.rs new file mode 100644 index 0000000..8da68ae --- /dev/null +++ b/src/memory/persona/readers/instruction_tests.rs @@ -0,0 +1,109 @@ +//! Tests for the instruction-file reader / rule normaliser. + +use super::*; +use crate::memory::persona::types::{EvidenceTier, PersonaFacet}; +use std::io::Write; +use tempfile::TempDir; + +const SAMPLE: &str = r#"# Global rules + +## Version control + +- Always branch before writing code. +- Commit regularly with clear messages. + +## Style + +Use Rust 2021 and standard cargo fmt style. + +```sh +cargo fmt --all +``` +"#; + +#[test] +fn splits_into_rule_granular_verbatim_chunks() { + let rules = split_rules(SAMPLE); + // Two bullets + one paragraph = three rules; headings and the code fence + // are dropped. + assert_eq!(rules.len(), 3, "rules: {rules:?}"); + assert!(rules + .iter() + .any(|r| r == "Always branch before writing code.")); + assert!(rules.iter().any(|r| r.starts_with("Use Rust 2021"))); + assert!(!rules.iter().any(|r| r.contains("cargo fmt --all"))); // fenced code dropped + assert!(!rules.iter().any(|r| r.contains('#'))); // headings dropped +} + +#[test] +fn nested_bullets_stay_with_their_rule() { + let md = "- Parent rule\n - detail a\n - detail b\n- Second rule"; + let rules = split_rules(md); + assert_eq!(rules.len(), 2); + assert!(rules[0].contains("Parent rule")); + assert!(rules[0].contains("detail a")); + assert!(rules[0].contains("detail b")); + assert_eq!(rules[1], "Second rule"); +} + +#[test] +fn read_file_emits_t0_directives() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("CLAUDE.md"); + write!(std::fs::File::create(&path).unwrap(), "{SAMPLE}").unwrap(); + let file = InstructionFile { + path: path.clone(), + scope: RuleScope::Global, + }; + let session = read_file(&file).unwrap(); + assert_eq!(session.evidence.len(), 3); + for ev in &session.evidence { + assert_eq!(ev.tier, EvidenceTier::T0); + assert_eq!(ev.facets, vec![PersonaFacet::Directives]); + assert_eq!(ev.source.scope.as_deref(), Some("global")); + } +} + +#[test] +fn discover_matches_names_and_repo_scope() { + let dir = TempDir::new().unwrap(); + // A fake repo: root/.git + root/CLAUDE.md + root/.github/copilot-instructions.md + let repo = dir.path().join("myrepo"); + std::fs::create_dir_all(repo.join(".git")).unwrap(); + std::fs::create_dir_all(repo.join(".github")).unwrap(); + std::fs::write(repo.join("CLAUDE.md"), "- rule one").unwrap(); + std::fs::write(repo.join("AGENTS.md"), "- rule two").unwrap(); + std::fs::write(repo.join(".github/copilot-instructions.md"), "- rule three").unwrap(); + // A copilot file NOT under .github must be ignored. + std::fs::write(repo.join("copilot-instructions.md"), "- ignored").unwrap(); + // An unrelated markdown file must be ignored. + std::fs::write(repo.join("README.md"), "- nope").unwrap(); + + let global = dir.path().join("global-CLAUDE.md"); + std::fs::write(&global, "- global rule").unwrap(); + + let found = discover(&[dir.path().to_path_buf()], &[global.clone()]); + let names: Vec = found + .iter() + .map(|f| f.path.file_name().unwrap().to_string_lossy().to_string()) + .collect(); + assert!(names.contains(&"CLAUDE.md".to_string())); + assert!(names.contains(&"AGENTS.md".to_string())); + assert!(names.contains(&"copilot-instructions.md".to_string())); + assert!(!found + .iter() + .any(|f| f.path.ends_with("myrepo/copilot-instructions.md"))); + + // Global file is Global-scoped; repo files are Repo("myrepo"). + assert!(found.iter().any(|f| f.scope == RuleScope::Global)); + assert!(found + .iter() + .any(|f| f.scope == RuleScope::Repo("myrepo".to_string()))); +} + +#[test] +fn content_sha_is_stable_and_sensitive() { + assert_eq!(content_sha(b"abc"), content_sha(b"abc")); + assert_ne!(content_sha(b"abc"), content_sha(b"abd")); + assert_eq!(content_sha(b"abc").len(), 64); +} diff --git a/src/memory/persona/readers/jsonl.rs b/src/memory/persona/readers/jsonl.rs new file mode 100644 index 0000000..6c8183b --- /dev/null +++ b/src/memory/persona/readers/jsonl.rs @@ -0,0 +1,44 @@ +//! Shared line-streaming JSONL helper for the transcript readers. +//! +//! Reads a `.jsonl` file one line at a time (never materialising the whole +//! file), yielding `(byte_len, Value)` for each line that parses as JSON. +//! Lines that fail to parse — truncated writes, non-JSON scaffolding — are +//! skipped, and their bytes are still reported so byte-reduction accounting +//! stays honest. Unknown event shapes are the caller's concern. + +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Stream `path` line-by-line, invoking `on_line(byte_len, value)` for every +/// line that parses as JSON. `byte_len` counts the raw line bytes (including the +/// newline) so the caller can total the file's read size incrementally. +/// +/// Returns the total number of raw bytes walked (including unparseable lines). +pub fn for_each_json_line(path: &Path, mut on_line: F) -> Result +where + F: FnMut(u64, &serde_json::Value), +{ + let file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let reader = BufReader::new(file); + let mut total: u64 = 0; + for line in reader.lines() { + // A single unreadable line (invalid UTF-8) shouldn't abort the file. + let line = match line { + Ok(l) => l, + Err(_) => continue, + }; + let byte_len = line.len() as u64 + 1; // + newline + total += byte_len; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if let Ok(value) = serde_json::from_str::(trimmed) { + on_line(byte_len, &value); + } + } + Ok(total) +} diff --git a/src/memory/persona/readers/mod.rs b/src/memory/persona/readers/mod.rs new file mode 100644 index 0000000..9669486 --- /dev/null +++ b/src/memory/persona/readers/mod.rs @@ -0,0 +1,143 @@ +//! Source readers (doc 06 §6.4): stream a source's on-disk history and emit +//! redacted [`PersonaEvidence`], one [`RawSession`] per logical session / file / +//! commit batch. +//! +//! The shared principle is **extract the person, discard the machine**: only +//! user decisions, corrections, and prompts are kept; tool output, reasoning, +//! file dumps, and vendor scaffolding are dropped. Each reader records the raw +//! bytes it walked and the bytes it kept so the ≥95% reduction contract is +//! measurable (see [`RawSession::reduction_ratio`]). +//! +//! Readers stream line-at-a-time and never materialise a whole session file. + +use chrono::{DateTime, Utc}; + +use super::types::{EvidenceSource, PersonaEvidence}; + +pub mod claude_code; +pub mod codex; +pub mod instruction; + +/// Git commit-history reader — requires the heavy `git2` dependency, so it is +/// gated behind the `git-diff` feature (doc 06 §6.3). +#[cfg(feature = "git-diff")] +pub mod git_history; + +mod jsonl; + +/// A logical session's worth of extracted, redacted evidence plus the byte +/// accounting that proves the extraction discarded the machine. +#[derive(Debug, Clone)] +pub struct RawSession { + /// Provenance shared by every unit in this session. + pub source: EvidenceSource, + /// The extracted, redacted evidence (user turns, corrections, habits). + pub evidence: Vec, + /// Earliest event timestamp seen, if any. + pub started_at: Option>, + /// Latest event timestamp seen, if any. + pub ended_at: Option>, + /// Total bytes read from the source (the whole file / rows scanned). + pub raw_bytes: u64, + /// Bytes of extracted excerpt text kept (post-redaction). + pub kept_bytes: u64, +} + +impl RawSession { + /// Start an empty session for `source`. + pub fn new(source: EvidenceSource) -> Self { + Self { + source, + evidence: Vec::new(), + started_at: None, + ended_at: None, + raw_bytes: 0, + kept_bytes: 0, + } + } + + /// Record a kept unit of evidence and fold its timestamp into the window. + pub fn push(&mut self, ev: PersonaEvidence) { + let ts = ev.timestamp; + self.started_at = Some(self.started_at.map_or(ts, |cur| cur.min(ts))); + self.ended_at = Some(self.ended_at.map_or(ts, |cur| cur.max(ts))); + self.kept_bytes += ev.excerpt().len() as u64; + self.evidence.push(ev); + } + + /// Fraction of raw bytes discarded before the LLM sees anything. `0.0` when + /// nothing was read. + pub fn reduction_ratio(&self) -> f64 { + if self.raw_bytes == 0 { + return 0.0; + } + 1.0 - (self.kept_bytes as f64 / self.raw_bytes as f64) + } + + /// True when the session yielded no evidence (skip it in the pipeline). + pub fn is_empty(&self) -> bool { + self.evidence.is_empty() + } +} + +/// Heuristic correction/interrupt detector for tier assignment (T1 — the single +/// highest inference signal short of an explicit rule). A user turn that opens +/// with a redirection ("no, do X", "actually", "stop", "don't", "revert") is +/// the person stopping an agent and steering it. +pub fn looks_like_correction(text: &str) -> bool { + let t = text.trim_start().to_lowercase(); + const OPENERS: [&str; 14] = [ + "no,", + "no ", + "nope", + "actually", + "wait", + "stop", + "don't", + "dont", + "instead", + "revert", + "undo", + "that's wrong", + "thats wrong", + "not like that", + ]; + OPENERS.iter().any(|p| t.starts_with(p)) + || t.contains("not what i") + || t.contains("do it again") + || t.contains("that's not") +} + +/// Directory names pruned from `walkdir` traversals: build output, dependency +/// caches, vendored submodules, and nested agent worktrees (which would +/// double-count the same repo/instruction files). Keeping these out of the walk +/// is both a large speedup and a correctness win. +pub(crate) fn keep_walk_entry(entry: &walkdir::DirEntry) -> bool { + if !entry.file_type().is_dir() { + return true; + } + let name = entry.file_name().to_string_lossy(); + !matches!( + name.as_ref(), + "target" + | "node_modules" + | ".git" + | ".venv" + | "venv" + | "dist" + | "build" + | ".cache" + | ".claude" + | "vendor" + ) +} + +/// True when a user turn is a slash-command / custom-command invocation (a +/// habit trace, T2) — leading `/` or `$` token, e.g. `/code-review`, +/// `$ship-and-babysit`. +pub fn looks_like_command(text: &str) -> bool { + let t = text.trim_start(); + (t.starts_with('/') || t.starts_with('$')) + && t.len() > 1 + && t.chars().nth(1).is_some_and(|c| c.is_ascii_alphanumeric()) +} diff --git a/src/memory/persona/reduce.rs b/src/memory/persona/reduce.rs new file mode 100644 index 0000000..6f97aae --- /dev/null +++ b/src/memory/persona/reduce.rs @@ -0,0 +1,261 @@ +//! Distillation reduce step (doc 06 §6.5): fold [`SessionDigest`] observations +//! (and verbatim T0 directive evidence) into the seven facet flavoured trees. +//! +//! Each facet maps to one `TreeFactory::flavoured(scope, ask)` tree. A digest +//! contributes one leaf per facet it has observations for; the existing +//! seal/fold mechanic re-summarises through the facet's `ask` and recompiles the +//! root, so incremental re-distillation comes for free. The reduce step is +//! summariser-agnostic — the deterministic `ConcatSummariser` gives the offline +//! path, a real provider gives distilled prose. + +use std::collections::BTreeMap; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use sha2::{Digest, Sha256}; + +use super::types::{DigestObservation, EvidenceTier, PersonaEvidence, PersonaFacet, SessionDigest}; +use crate::memory::chunks::{chunk_id, upsert_chunks, Chunk, Metadata, SourceKind}; +use crate::memory::config::MemoryConfig; +use crate::memory::tree::bucket_seal::LeafRef; +use crate::memory::tree::flavoured::compile_flavoured_root; +use crate::memory::tree::{Summariser, TreeFactory}; + +/// Per-facet natural-language asks, defaulting to [`PersonaFacet::default_ask`] +/// and overridable via config. +#[derive(Debug, Clone, Default)] +pub struct FacetAsks(pub BTreeMap); + +impl FacetAsks { + /// The ask for `facet`, falling back to the built-in default. + pub fn ask(&self, facet: PersonaFacet) -> String { + self.0 + .get(&facet) + .cloned() + .unwrap_or_else(|| facet.default_ask().to_string()) + } +} + +/// Running reduce state threaded across sessions: per-facet observation counts +/// (for the pack's strength annotations) and the verbatim directive set. +#[derive(Debug, Default)] +pub struct ReduceState { + /// Observation counts per facet, for "seen in N observations" annotations. + pub counts: BTreeMap, + /// Distinct scopes (repos/projects) each facet was observed in. + pub scopes: BTreeMap>, + /// Verbatim T0 directive rules (from instruction files), deduped in order. + /// Kept out of the LLM fold so the compiled Directives section stays + /// near-verbatim regardless of the summariser (§6.5). + pub directives: Vec, +} + +impl ReduceState { + fn record(&mut self, facet: PersonaFacet, n: usize, scope: Option<&str>) { + *self.counts.entry(facet).or_default() += n; + if let Some(sc) = scope { + self.scopes.entry(facet).or_default().insert(sc.to_string()); + } + } +} + +/// Weight a leaf by its confidence tier so higher-tier evidence folds first and +/// survives budget pressure. +pub fn tier_score(tier: EvidenceTier) -> f32 { + match tier { + EvidenceTier::T0 => 1.0, + EvidenceTier::T1 => 0.9, + EvidenceTier::T2 => 0.7, + EvidenceTier::T3 => 0.4, + } +} + +/// Fold one digest into the facet trees. Groups observations by facet and writes +/// one leaf per facet. +pub async fn fold_digest( + config: &MemoryConfig, + digest: &SessionDigest, + asks: &FacetAsks, + summariser: &dyn Summariser, + state: &mut ReduceState, +) -> Result<()> { + if digest.is_empty() { + return Ok(()); + } + let scope = digest.source.scope.clone(); + // Group by facet, preserving order. + let mut by_facet: BTreeMap> = BTreeMap::new(); + for obs in &digest.observations { + by_facet.entry(obs.facet).or_default().push(obs); + } + for (facet, obs) in by_facet { + let leaf_text = render_observations(&obs); + // Highest tier present in this leaf drives its fold weight. + let top_tier = obs.iter().map(|o| o.tier).max().unwrap_or(EvidenceTier::T3); + fold_leaf( + config, + facet, + &asks.ask(facet), + &leaf_text, + digest_timestamp(digest), + tier_score(top_tier), + summariser, + ) + .await?; + state.record(facet, obs.len(), scope.as_deref()); + } + Ok(()) +} + +/// Collect verbatim T0 directive evidence (from instruction files). These are +/// *not* folded through the LLM — they flow into the pack near-verbatim so the +/// person's explicit rules survive exactly (§6.4/§6.5). Deduped in first-seen +/// order. +pub fn fold_directives(evidence: &[PersonaEvidence], state: &mut ReduceState) { + for ev in evidence { + let scope = ev.source.scope.clone(); + let scope_label = scope.as_deref().unwrap_or("global"); + let rule = format!("[{scope_label}] {}", ev.excerpt()); + if !state.directives.contains(&rule) { + state.directives.push(rule); + } + state.record(PersonaFacet::Directives, 1, scope.as_deref()); + } +} + +/// Render a facet's observations as a leaf body. +fn render_observations(obs: &[&DigestObservation]) -> String { + obs.iter() + .map(|o| { + if o.quote.trim().is_empty() { + format!("- {} [{}]", o.observation, o.tier.as_str()) + } else { + format!( + "- {} (\"{}\") [{}]", + o.observation, + o.quote, + o.tier.as_str() + ) + } + }) + .collect::>() + .join("\n") +} + +fn digest_timestamp(_digest: &SessionDigest) -> DateTime { + // Digests don't carry their own time; use now (folds are order-driven by + // the pipeline's oldest-first backfill, not by this stamp). + Utc::now() +} + +/// Persist `leaf_text` as a chunk and append it to `facet`'s flavoured tree. +async fn fold_leaf( + config: &MemoryConfig, + facet: PersonaFacet, + ask: &str, + leaf_text: &str, + timestamp: DateTime, + score: f32, + summariser: &dyn Summariser, +) -> Result<()> { + let scope = facet.tree_scope(); + let chunk = persona_chunk(&scope, leaf_text, timestamp); + upsert_chunks(config, std::slice::from_ref(&chunk))?; + + let leaf = LeafRef { + chunk_id: chunk.id.clone(), + token_count: estimate_tokens(leaf_text), + timestamp, + content: leaf_text.to_string(), + entities: Vec::new(), + topics: Vec::new(), + score, + }; + let factory = TreeFactory::flavoured(scope, ask.to_string()); + factory.insert_leaf(config, &leaf, summariser).await?; + Ok(()) +} + +/// Build a persona evidence chunk (stored as a `Document` source). +/// +/// The chunk id is **content-stable**: the `seq` fed into [`chunk_id`] is +/// derived from `(scope, content)`, not a per-run counter, so the same +/// observation text yields the same id on every run. Tree insertion dedupes by +/// chunk id, so re-reading the same evidence (e.g. a repo re-scanned after a +/// budget-truncated pass) folds it at most once. +fn persona_chunk(scope: &str, content: &str, timestamp: DateTime) -> Chunk { + let seq = stable_seq(scope, content); + Chunk { + id: chunk_id(SourceKind::Document, scope, seq, content), + content: content.to_string(), + metadata: Metadata { + source_kind: SourceKind::Document, + source_id: scope.to_string(), + owner: "persona".to_string(), + timestamp, + time_range: (timestamp, timestamp), + tags: vec!["persona".to_string()], + source_ref: None, + path_scope: None, + }, + token_count: estimate_tokens(content), + seq_in_source: seq, + created_at: timestamp, + partial_message: false, + } +} + +/// Deterministic pseudo-sequence derived from the leaf's scope + content, so a +/// given observation maps to a stable chunk id across runs (dedup-safe). +fn stable_seq(scope: &str, content: &str) -> u32 { + let mut h = Sha256::new(); + h.update(scope.as_bytes()); + h.update(b"\x1f"); + h.update(content.as_bytes()); + let d = h.finalize(); + u32::from_le_bytes([d[0], d[1], d[2], d[3]]) +} + +fn estimate_tokens(text: &str) -> u32 { + ((text.len() / 4) as u32).max(1) +} + +/// Seal every facet tree (force-flush the L0 buffer) and recompile its root. +/// Returns the compiled root body per facet (front-matter stripped). +pub async fn seal_and_collect( + config: &MemoryConfig, + asks: &FacetAsks, + summariser: &dyn Summariser, +) -> Result> { + let mut bodies = BTreeMap::new(); + for facet in PersonaFacet::ALL { + let factory = TreeFactory::flavoured(facet.tree_scope(), asks.ask(facet)); + let tree = factory.get_or_create(config)?; + // Only trees that received leaves have a non-empty buffer or root. + factory.seal_now(config, summariser).await?; + let markdown = compile_flavoured_root(config, &tree.id)?; + let body = strip_frontmatter(&markdown); + if !body.trim().is_empty() { + bodies.insert(facet, body); + } + } + Ok(bodies) +} + +/// Strip a leading YAML front-matter block (`---\n … \n---\n`) from `md`. +pub fn strip_frontmatter(md: &str) -> String { + let trimmed = md.trim_start(); + if let Some(rest) = trimmed.strip_prefix("---\n") { + if let Some(end) = rest.find("\n---\n") { + return rest[end + 5..].trim().to_string(); + } + if let Some(end) = rest.find("\n---") { + return rest[end + 4..].trim().to_string(); + } + } + md.trim().to_string() +} + +#[cfg(test)] +#[path = "reduce_tests.rs"] +mod tests; diff --git a/src/memory/persona/reduce_tests.rs b/src/memory/persona/reduce_tests.rs new file mode 100644 index 0000000..00b1857 --- /dev/null +++ b/src/memory/persona/reduce_tests.rs @@ -0,0 +1,145 @@ +//! End-to-end map→reduce→compile tests using a mock chat provider and the +//! deterministic `ConcatSummariser` (the offline, zero-cost path §6.10). + +use super::*; +use async_trait::async_trait; +use chrono::Utc; +use tempfile::TempDir; + +use crate::memory::config::MemoryConfig; +use crate::memory::persona::compile::{compile_pack, PackInputs}; +use crate::memory::persona::distill::digest_session; +use crate::memory::persona::readers::RawSession; +use crate::memory::persona::types::{ + EvidenceSource, EvidenceTier, PersonaEvidence, PersonaSourceKind, +}; +use crate::memory::score::extract::{ChatPrompt, ChatProvider}; +use crate::memory::tree::summarise::ConcatSummariser; + +struct MockChat(String); + +#[async_trait] +impl ChatProvider for MockChat { + fn name(&self) -> &str { + "mock" + } + async fn chat_for_json(&self, _p: &ChatPrompt) -> anyhow::Result { + Ok(self.0.clone()) + } +} + +fn cfg() -> (TempDir, MemoryConfig) { + let tmp = TempDir::new().unwrap(); + let cfg = MemoryConfig::new(tmp.path()); + (tmp, cfg) +} + +fn session(scope: &str) -> RawSession { + let src = EvidenceSource::new(PersonaSourceKind::ClaudeCode).with_scope(scope); + let mut s = RawSession::new(src.clone()); + s.push(PersonaEvidence::new( + src, + Utc::now(), + EvidenceTier::T2, + "commit small and often", + vec![], + )); + s +} + +#[tokio::test] +async fn full_map_reduce_compile_offline() { + let (_tmp, config) = cfg(); + let summariser = ConcatSummariser::new(); + let asks = FacetAsks::default(); + let mut state = ReduceState::default(); + + // Mock digest: two facets of observations. + let provider = MockChat( + r#"{"observations":[ + {"facet":"workflow","observation":"Commits small and often","quote":"commit small and often","tier":"t2"}, + {"facet":"coding_style","observation":"Insists on regression tests","quote":"add a test","tier":"t1"} + ]}"# + .into(), + ); + + // Two sessions from two different scopes → cross-project strength. + for scope in ["projA", "projB"] { + let digest = digest_session(&provider, &session(scope)).await.unwrap(); + fold_digest(&config, &digest, &asks, &summariser, &mut state) + .await + .unwrap(); + } + + // Fold a verbatim T0 directive too. + let src = EvidenceSource::new(PersonaSourceKind::InstructionFile).with_scope("global"); + let directive = PersonaEvidence::new( + src, + Utc::now(), + EvidenceTier::T0, + "Always branch before writing code.", + vec![PersonaFacet::Directives], + ); + fold_directives(std::slice::from_ref(&directive), &mut state); + + // Seal + compile the facet trees. + let bodies = seal_and_collect(&config, &asks, &summariser).await.unwrap(); + assert!(bodies.contains_key(&PersonaFacet::Workflow)); + assert!(bodies.contains_key(&PersonaFacet::CodingStyle)); + // Directives are collected verbatim (not folded into a tree). + assert!(state.directives.iter().any(|d| d.contains("Always branch"))); + + // Compile the pack. + let mut inputs = PackInputs::new("me@example.com"); + inputs.facet_bodies = bodies; + inputs.directives = state.directives.clone(); + inputs.counts = state.counts.clone(); + inputs.scopes = state.scopes.iter().map(|(k, v)| (*k, v.len())).collect(); + let pack = compile_pack(&inputs); + + assert!(pack.contains("# Persona: me@example.com")); + assert!(pack.contains("## Workflow")); + assert!(pack.contains("## Directives")); + assert!(pack.contains("commit small and often") || pack.contains("Commits small")); + // Workflow was seen in two projects. + assert_eq!(state.scopes[&PersonaFacet::Workflow].len(), 2); + assert_eq!(state.counts[&PersonaFacet::Workflow], 2); + // The verbatim directive survives into the pack. + assert!(pack.contains("Always branch before writing code.")); +} + +#[tokio::test] +async fn empty_digests_produce_empty_bodies() { + let (_tmp, config) = cfg(); + let summariser = ConcatSummariser::new(); + let asks = FacetAsks::default(); + let bodies = seal_and_collect(&config, &asks, &summariser).await.unwrap(); + assert!(bodies.is_empty(), "no leaves folded → no bodies"); +} + +#[test] +fn strip_frontmatter_removes_yaml_block() { + let md = "---\nkind: flavoured_root\nask: \"x\"\n---\nThe body here.\n"; + assert_eq!(strip_frontmatter(md), "The body here."); + assert_eq!(strip_frontmatter("no frontmatter"), "no frontmatter"); +} + +#[test] +fn persona_chunk_id_is_content_stable_across_runs() { + // Same (scope, content) → same chunk id regardless of any per-run counter, + // so re-reading the same evidence dedupes on tree insertion. + let t = Utc::now(); + let a = persona_chunk("persona/workflow", "- Commits small [t2]", t); + let b = persona_chunk( + "persona/workflow", + "- Commits small [t2]", + t + chrono::Duration::hours(3), + ); + assert_eq!(a.id, b.id, "id must not depend on run order or timestamp"); + // Different content → different id. + let c = persona_chunk("persona/workflow", "- Different [t2]", t); + assert_ne!(a.id, c.id); + // Different facet scope → different id. + let d = persona_chunk("persona/stack", "- Commits small [t2]", t); + assert_ne!(a.id, d.id); +} diff --git a/src/memory/persona/state.rs b/src/memory/persona/state.rs new file mode 100644 index 0000000..1ce4397 --- /dev/null +++ b/src/memory/persona/state.rs @@ -0,0 +1,189 @@ +//! Incremental-run state for the persona pipeline (doc 06 §6.7). +//! +//! Mirrors the `sync::state::SyncStateStore` pattern with a persona-local +//! [`PersonaStateStore`] trait (the `sync` module is gated behind the heavy +//! `sync`/reqwest feature, and persona stays dependency-light). Cursors are +//! deliberately coarse and cheap: +//! +//! - JSONL / transcript sources: `(mtime_ms, len)` per file — append-only in +//! practice, so an unchanged `(mtime, len)` skips the file. +//! - Git: last-distilled HEAD sha per repo (+ the author-set hash, so changing +//! the email list forces a re-scan). +//! - Instruction files: content sha per path — re-digest only on change. +//! +//! A concrete [`FileStateStore`] persists the map as one JSON file under the +//! workspace so cursors survive across process runs; tests use an in-memory +//! store. Evidence ids are content-addressed (§6.3), so overlapping cursors are +//! harmless — this state is a *fast-skip* optimisation, not a correctness gate. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; + +/// Persona-scoped state namespace. +pub const NAMESPACE: &str = "persona-sync-state"; + +/// A minimal key/value state store, scoped by namespace. Mirrors +/// `sync::state::SyncStateStore` so the same persistence pattern is reused +/// without depending on the feature-gated `sync` module. +#[async_trait] +pub trait PersonaStateStore: Send + Sync { + /// Fetch a stored value, if any. + async fn get(&self, namespace: &str, key: &str) -> Result>; + /// Store a value, overwriting any prior value. + async fn set(&self, namespace: &str, key: &str, value: &serde_json::Value) -> Result<()>; +} + +/// A file's fast-skip cursor. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileCursor { + /// File modification time in milliseconds since the epoch. + pub mtime_ms: i64, + /// File length in bytes. + pub len: u64, +} + +impl FileCursor { + /// Read the current cursor for `path`. + pub fn of(path: &Path) -> Option { + let meta = std::fs::metadata(path).ok()?; + let mtime_ms = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as i64)?; + Some(Self { + mtime_ms, + len: meta.len(), + }) + } +} + +/// Cursor key for a transcript/instruction file of a given source kind. +pub fn file_key(source_kind: &str, path: &Path) -> String { + format!("{source_kind}:{}", path.display()) +} + +/// Cursor key for a git repo. +pub fn git_key(repo: &Path) -> String { + format!("git_history:{}", repo.display()) +} + +/// True when `path` is unchanged since its recorded cursor (skip it). +pub async fn file_unchanged(store: &dyn PersonaStateStore, key: &str, path: &Path) -> Result { + let current = match FileCursor::of(path) { + Some(c) => c, + None => return Ok(false), + }; + match store.get(NAMESPACE, key).await? { + Some(v) => { + let stored: FileCursor = match serde_json::from_value(v) { + Ok(c) => c, + Err(_) => return Ok(false), + }; + Ok(stored == current) + } + None => Ok(false), + } +} + +/// Record `path`'s current cursor under `key`. +pub async fn record_file(store: &dyn PersonaStateStore, key: &str, path: &Path) -> Result<()> { + if let Some(cursor) = FileCursor::of(path) { + store + .set(NAMESPACE, key, &serde_json::to_value(cursor)?) + .await?; + } + Ok(()) +} + +/// True when the recorded string watermark under `key` equals `value` (skip). +pub async fn watermark_unchanged( + store: &dyn PersonaStateStore, + key: &str, + value: &str, +) -> Result { + match store.get(NAMESPACE, key).await? { + Some(serde_json::Value::String(s)) => Ok(s == value), + _ => Ok(false), + } +} + +/// Record a string watermark (git sha, content sha) under `key`. +pub async fn record_watermark(store: &dyn PersonaStateStore, key: &str, value: &str) -> Result<()> { + store + .set( + NAMESPACE, + key, + &serde_json::Value::String(value.to_string()), + ) + .await +} + +/// JSON-file-backed [`PersonaStateStore`]. Persists the whole map atomically on +/// each `set` so cursors survive across runs. +pub struct FileStateStore { + path: PathBuf, + data: Mutex>, +} + +impl FileStateStore { + /// Open (or create) the state file under a workspace's persona directory. + pub fn open_in_workspace(workspace: &Path) -> Result { + let path = workspace.join("persona").join("sync-state.json"); + let data = if path.exists() { + let bytes = std::fs::read(&path) + .with_context(|| format!("read persona state {}", path.display()))?; + serde_json::from_slice(&bytes).unwrap_or_default() + } else { + HashMap::new() + }; + Ok(Self { + path, + data: Mutex::new(data), + }) + } + + fn composite(namespace: &str, key: &str) -> String { + format!("{namespace}:{key}") + } + + fn persist(&self, snapshot: &HashMap) -> Result<()> { + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create persona dir {}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(snapshot)?; + crate::memory::fsutil::atomic_write(&self.path, &bytes) + .with_context(|| format!("write persona state {}", self.path.display()))?; + Ok(()) + } +} + +#[async_trait] +impl PersonaStateStore for FileStateStore { + async fn get(&self, namespace: &str, key: &str) -> Result> { + Ok(self + .data + .lock() + .get(&Self::composite(namespace, key)) + .cloned()) + } + + async fn set(&self, namespace: &str, key: &str, value: &serde_json::Value) -> Result<()> { + let snapshot = { + let mut guard = self.data.lock(); + guard.insert(Self::composite(namespace, key), value.clone()); + guard.clone() + }; + self.persist(&snapshot) + } +} + +#[cfg(test)] +#[path = "state_tests.rs"] +mod tests; diff --git a/src/memory/persona/state_tests.rs b/src/memory/persona/state_tests.rs new file mode 100644 index 0000000..9ee153f --- /dev/null +++ b/src/memory/persona/state_tests.rs @@ -0,0 +1,52 @@ +//! Tests for persona incremental-run state. + +use super::*; +use tempfile::TempDir; + +#[tokio::test] +async fn file_cursor_detects_change() { + let dir = TempDir::new().unwrap(); + let f = dir.path().join("a.jsonl"); + std::fs::write(&f, "one").unwrap(); + let store = FileStateStore::open_in_workspace(dir.path()).unwrap(); + let key = file_key("claude_code", &f); + + // Unrecorded → not unchanged. + assert!(!file_unchanged(&store, &key, &f).await.unwrap()); + record_file(&store, &key, &f).await.unwrap(); + assert!(file_unchanged(&store, &key, &f).await.unwrap()); + + // Append changes len → cursor differs. + std::fs::write(&f, "one-two-three").unwrap(); + assert!(!file_unchanged(&store, &key, &f).await.unwrap()); +} + +#[tokio::test] +async fn watermark_roundtrips() { + let dir = TempDir::new().unwrap(); + let store = FileStateStore::open_in_workspace(dir.path()).unwrap(); + assert!(!watermark_unchanged(&store, "git_history:/r", "sha1") + .await + .unwrap()); + record_watermark(&store, "git_history:/r", "sha1") + .await + .unwrap(); + assert!(watermark_unchanged(&store, "git_history:/r", "sha1") + .await + .unwrap()); + assert!(!watermark_unchanged(&store, "git_history:/r", "sha2") + .await + .unwrap()); +} + +#[tokio::test] +async fn state_persists_across_reopen() { + let dir = TempDir::new().unwrap(); + { + let store = FileStateStore::open_in_workspace(dir.path()).unwrap(); + record_watermark(&store, "k", "v").await.unwrap(); + } + // Reopen from disk. + let store = FileStateStore::open_in_workspace(dir.path()).unwrap(); + assert!(watermark_unchanged(&store, "k", "v").await.unwrap()); +} diff --git a/src/memory/persona/types.rs b/src/memory/persona/types.rs new file mode 100644 index 0000000..9bdfeae --- /dev/null +++ b/src/memory/persona/types.rs @@ -0,0 +1,422 @@ +//! Canonical evidence model for persona distillation (doc 06 §6.3). +//! +//! Everything the readers emit and the pipeline consumes is expressed in terms +//! of the contracts here: the [`PersonaSourceKind`] a unit of evidence came +//! from, its [`EvidenceTier`] (confidence ladder), the [`PersonaFacet`]s it +//! informs, and the redacted [`PersonaEvidence`] excerpt itself. The LLM map +//! step (§6.5) turns a batch of evidence into a [`SessionDigest`]. +//! +//! ## Redaction boundary (fail-closed) +//! +//! [`PersonaEvidence`] can only be constructed through [`PersonaEvidence::new`], +//! which runs the raw excerpt through +//! [`sanitize_text`](crate::memory::store::safety::sanitize_text) *before* the +//! value is stored on the struct. `sanitize_text` is the composite redactor: it +//! scrubs secrets/tokens/keys (OpenAI `sk-…`, GitHub `gh*_…`, OAuth/bearer +//! credentials) *and* runs the formatted-PII pass +//! ([`redact_pii`](crate::memory::store::safety::pii::redact_pii)) the plan +//! names in §6.3 — the stronger of the two, chosen because transcripts are full +//! of tokens and keys, not just national-id-shaped PII. There is no way to build +//! evidence from unredacted text, so nothing unredacted can leave the reader +//! layer or reach an LLM. The struct field is private to enforce this. +//! +//! ## Deterministic ids +//! +//! Evidence ids are content-addressed — `sha256(source_id ‖ excerpt)[..32]` — +//! mirroring the archivist leaf-id convention so re-runs dedupe naturally. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::memory::store::safety::sanitize_text; + +/// The on-disk source a unit of persona evidence came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PersonaSourceKind { + /// Claude Code transcripts (`~/.claude/projects/**/*.jsonl`). + ClaudeCode, + /// Codex rollout transcripts (`~/.codex/sessions/**/rollout-*.jsonl`). + Codex, + /// opencode SQLite store (`opencode.db`). + Opencode, + /// Cursor workspace/global KV stores (`state.vscdb`). + Cursor, + /// User-supplied ChatGPT export (`conversations.json`). + ChatgptExport, + /// User-supplied Claude.ai / Cowork export JSON. + ClaudeExport, + /// Explicitly authored agent instruction files (CLAUDE.md / AGENTS.md / …). + InstructionFile, + /// Git commit history. + GitHistory, +} + +impl PersonaSourceKind { + /// Stable wire/string form (matches the serde `snake_case` encoding). + pub fn as_str(self) -> &'static str { + match self { + PersonaSourceKind::ClaudeCode => "claude_code", + PersonaSourceKind::Codex => "codex", + PersonaSourceKind::Opencode => "opencode", + PersonaSourceKind::Cursor => "cursor", + PersonaSourceKind::ChatgptExport => "chatgpt_export", + PersonaSourceKind::ClaudeExport => "claude_export", + PersonaSourceKind::InstructionFile => "instruction_file", + PersonaSourceKind::GitHistory => "git_history", + } + } +} + +/// Confidence ladder used for weighting and conflict resolution (§6.3). +/// +/// Higher tiers win conflicts; within a tier, newer evidence wins. `T3` may +/// only corroborate — it can never establish or override a preference on its +/// own. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidenceTier { + /// Inferred from accepted outcomes (merged diffs, un-corrected output). + /// Lowest confidence; corroborates only. + T3, + /// User prompt phrasing, slash-command habits, commit-message style. + T2, + /// In-transcript corrections / interrupts — the single highest inference + /// signal short of an explicit rule. + T1, + /// Explicit instruction files — the person wrote the rule down. + T0, +} + +impl EvidenceTier { + /// Stable string form (matches the serde encoding). + pub fn as_str(self) -> &'static str { + match self { + EvidenceTier::T0 => "t0", + EvidenceTier::T1 => "t1", + EvidenceTier::T2 => "t2", + EvidenceTier::T3 => "t3", + } + } + + /// Numeric rank (0 = weakest `T3`, 3 = strongest `T0`) for ordering and + /// weighting. Deliberately aligned with the [`Ord`] derive above. + pub fn rank(self) -> u8 { + match self { + EvidenceTier::T3 => 0, + EvidenceTier::T2 => 1, + EvidenceTier::T1 => 2, + EvidenceTier::T0 => 3, + } + } + + /// Parse the loose forms an LLM might emit (`"T1"`, `"t1"`, `"tier1"`, `1`). + pub fn parse_loose(s: &str) -> Option { + match s.trim().to_lowercase().trim_start_matches("tier").trim() { + "t0" | "0" => Some(EvidenceTier::T0), + "t1" | "1" => Some(EvidenceTier::T1), + "t2" | "2" => Some(EvidenceTier::T2), + "t3" | "3" => Some(EvidenceTier::T3), + _ => None, + } + } +} + +/// The seven distillation lenses (§6.3). Each maps to one flavoured tree with a +/// purpose-written `ask` and one section of the compiled pack. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PersonaFacet { + /// Tone, verbosity, directness, phrasing quirks, how they give feedback. + Communication, + /// Naming, structure, comments, error handling, testing habits, size + /// discipline. + CodingStyle, + /// Languages, frameworks, libraries, recurring architectural choices. + Stack, + /// Branching/commit granularity, plan-first vs. dive-in, PR habits, review + /// strictness, parallelism. + Workflow, + /// Editors/harnesses, CLIs, package managers, OS. + Environment, + /// Explicit standing rules (mostly T0, near-verbatim). + Directives, + /// Pet peeves: things they correct agents for, revert, or forbid. + AntiPreferences, +} + +impl PersonaFacet { + /// Every facet, in the fixed compile order used by the pack. + pub const ALL: [PersonaFacet; 7] = [ + PersonaFacet::Directives, + PersonaFacet::Communication, + PersonaFacet::CodingStyle, + PersonaFacet::Stack, + PersonaFacet::Workflow, + PersonaFacet::Environment, + PersonaFacet::AntiPreferences, + ]; + + /// Stable string form (matches the serde encoding). + pub fn as_str(self) -> &'static str { + match self { + PersonaFacet::Communication => "communication", + PersonaFacet::CodingStyle => "coding_style", + PersonaFacet::Stack => "stack", + PersonaFacet::Workflow => "workflow", + PersonaFacet::Environment => "environment", + PersonaFacet::Directives => "directives", + PersonaFacet::AntiPreferences => "anti_preferences", + } + } + + /// Human-facing section heading used in the compiled pack (§6.9). + pub fn heading(self) -> &'static str { + match self { + PersonaFacet::Communication => "Communication style", + PersonaFacet::CodingStyle => "Coding style", + PersonaFacet::Stack => "Stack", + PersonaFacet::Workflow => "Workflow", + PersonaFacet::Environment => "Environment", + PersonaFacet::Directives => "Directives", + PersonaFacet::AntiPreferences => "Anti-preferences", + } + } + + /// Flavoured-tree scope for this facet (`persona/`). + pub fn tree_scope(self) -> String { + format!("persona/{}", self.as_str()) + } + + /// Parse the loose forms an LLM might emit. + pub fn parse_loose(s: &str) -> Option { + match s.trim().to_lowercase().replace([' ', '-'], "_").as_str() { + "communication" | "comms" | "tone" => Some(PersonaFacet::Communication), + "coding_style" | "code_style" | "coding" | "style" => Some(PersonaFacet::CodingStyle), + "stack" | "tech_stack" | "technology" => Some(PersonaFacet::Stack), + "workflow" | "process" => Some(PersonaFacet::Workflow), + "environment" | "env" | "tooling" => Some(PersonaFacet::Environment), + "directives" | "rules" | "directive" => Some(PersonaFacet::Directives), + "anti_preferences" | "anti_preference" | "antipreferences" | "dislikes" + | "pet_peeves" => Some(PersonaFacet::AntiPreferences), + _ => None, + } + } + + /// Default natural-language `ask` steering this facet's flavoured tree + /// (§6.5). Overridable via `PersonaConfig`. + pub fn default_ask(self) -> &'static str { + match self { + PersonaFacet::Communication => { + "Distill how this person communicates with their coding agents: tone, \ + verbosity, directness, recurring phrasing and quirks, and how they give \ + feedback. Describe the patterns as prescriptive guidance an agent should \ + mirror, not a list of individual messages." + } + PersonaFacet::CodingStyle => { + "Distill this person's coding style: naming, code structure, comment and \ + documentation habits, error handling, testing discipline, and module/size \ + discipline. Phrase as concrete rules an agent writing code for them should \ + follow." + } + PersonaFacet::Stack => { + "Distill this person's technology stack and architectural preferences: the \ + languages, frameworks, libraries, and recurring architectural choices they \ + favour. Phrase as defaults an agent should reach for." + } + PersonaFacet::Workflow => { + "Distill this person's development workflow: branching and commit \ + granularity, plan-first vs. dive-in, PR and review habits, and how they use \ + parallelism (worktrees, subagents). Phrase as process rules an agent should \ + follow." + } + PersonaFacet::Environment => { + "Distill this person's working environment: the editors and agent harnesses, \ + CLIs, package managers, and operating system they use. Phrase as facts an \ + agent should assume." + } + PersonaFacet::Directives => { + "Collect this person's explicit standing rules for their agents, kept as \ + close to verbatim as possible. These are commands they have written down; \ + preserve their exact intent and wording." + } + PersonaFacet::AntiPreferences => { + "Distill the things this person dislikes, corrects agents for, reverts, or \ + explicitly forbids — phrased as rules an agent must not break." + } + } + } +} + +/// Provenance of a unit of evidence: where in the source corpus it came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EvidenceSource { + /// Which on-disk source family produced it. + pub kind: PersonaSourceKind, + /// Project / repo / channel the evidence is scoped to, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, + /// Session id or commit sha the evidence belongs to, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Absolute or repo-relative path the evidence was read from, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +impl EvidenceSource { + /// Build a source with only its kind set; use the `with_*` setters to add + /// provenance. + pub fn new(kind: PersonaSourceKind) -> Self { + Self { + kind, + scope: None, + session_id: None, + path: None, + } + } + + /// Attach the project/repo/channel scope. + pub fn with_scope(mut self, scope: impl Into) -> Self { + self.scope = Some(scope.into()); + self + } + + /// Attach the session id or commit sha. + pub fn with_session(mut self, session_id: impl Into) -> Self { + self.session_id = Some(session_id.into()); + self + } + + /// Attach the source file path. + pub fn with_path(mut self, path: impl Into) -> Self { + self.path = Some(path.into()); + self + } + + /// Stable string used as the first half of a content-addressed evidence id. + /// Deterministic in the fields that identify the *origin* (not the excerpt). + pub fn source_id(&self) -> String { + format!( + "{}|{}|{}|{}", + self.kind.as_str(), + self.scope.as_deref().unwrap_or(""), + self.session_id.as_deref().unwrap_or(""), + self.path.as_deref().unwrap_or(""), + ) + } +} + +/// Content-address an evidence id: `sha256(source_id ‖ excerpt)[..32]` (hex). +pub fn evidence_id(source_id: &str, excerpt: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(source_id.as_bytes()); + hasher.update(b"\x1f"); // unit separator, keeps the two halves unambiguous + hasher.update(excerpt.as_bytes()); + let digest = hasher.finalize(); + let hex = digest + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + hex[..32].to_string() +} + +/// One unit of redacted persona evidence. +/// +/// Construct only via [`PersonaEvidence::new`]; the `excerpt` field is private +/// so evidence cannot be built from unredacted text. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersonaEvidence { + /// Content-addressed id (`sha256(source_id ‖ excerpt)[..32]`). + pub id: String, + /// Where the evidence came from. + pub source: EvidenceSource, + /// When the underlying event happened. + pub timestamp: DateTime, + /// Confidence tier. + pub tier: EvidenceTier, + /// Facets this evidence is a candidate for. The digest step may refine this; + /// readers set a coarse default. + #[serde(default)] + pub facets: Vec, + /// Redacted excerpt (private: only [`PersonaEvidence::new`] may set it, + /// after running the raw text through `redact_pii`). + excerpt: String, +} + +impl PersonaEvidence { + /// Build a unit of evidence, redacting `raw_excerpt` before it is stored. + /// + /// This is the *only* constructor. The id is derived from the redacted + /// excerpt so it is stable across re-runs and independent of any secret or + /// PII that was stripped. + pub fn new( + source: EvidenceSource, + timestamp: DateTime, + tier: EvidenceTier, + raw_excerpt: &str, + facets: Vec, + ) -> Self { + let excerpt = sanitize_text(raw_excerpt).value; + let id = evidence_id(&source.source_id(), &excerpt); + Self { + id, + source, + timestamp, + tier, + facets, + excerpt, + } + } + + /// The redacted excerpt. There is no way to read unredacted text back. + pub fn excerpt(&self) -> &str { + &self.excerpt + } +} + +/// One observation emitted by the LLM map step for a single facet. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DigestObservation { + /// Which facet this observation informs. + pub facet: PersonaFacet, + /// Prescriptive statement of the pattern (an agent-facing rule). + pub observation: String, + /// Short supporting quote drawn from the (already redacted) evidence. + #[serde(default)] + pub quote: String, + /// Confidence tier the model assigned this observation. + pub tier: EvidenceTier, +} + +/// Output of the LLM map step (§6.5): structured, per-facet observations for one +/// digested session or commit batch. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SessionDigest { + /// Provenance of the digested unit. + pub source: EvidenceSource, + /// Per-facet observations. + #[serde(default)] + pub observations: Vec, +} + +impl SessionDigest { + /// A digest with no observations (the soft-fallback result for a session + /// the model failed to process — see §6.5). + pub fn empty(source: EvidenceSource) -> Self { + Self { + source, + observations: Vec::new(), + } + } + + /// True when the model produced no usable observations. + pub fn is_empty(&self) -> bool { + self.observations.is_empty() + } +} + +#[cfg(test)] +#[path = "types_tests.rs"] +mod tests; diff --git a/src/memory/persona/types_tests.rs b/src/memory/persona/types_tests.rs new file mode 100644 index 0000000..2e076cf --- /dev/null +++ b/src/memory/persona/types_tests.rs @@ -0,0 +1,120 @@ +//! Tests for the persona evidence model (§6.3 contracts). + +use super::*; +use chrono::TimeZone; + +fn ts() -> DateTime { + Utc.timestamp_opt(1_700_000_000, 0).unwrap() +} + +#[test] +fn evidence_ids_are_content_addressed_and_deterministic() { + let src = EvidenceSource::new(PersonaSourceKind::ClaudeCode) + .with_scope("proj") + .with_session("sess-1"); + let a = PersonaEvidence::new( + src.clone(), + ts(), + EvidenceTier::T2, + "commit small and often", + vec![], + ); + let b = PersonaEvidence::new( + src.clone(), + ts(), + EvidenceTier::T2, + "commit small and often", + vec![], + ); + // Same source + same excerpt → same id (re-runs dedupe naturally). + assert_eq!(a.id, b.id); + assert_eq!(a.id.len(), 32); + // Different excerpt → different id. + let c = PersonaEvidence::new(src, ts(), EvidenceTier::T2, "different", vec![]); + assert_ne!(a.id, c.id); +} + +#[test] +fn evidence_is_redacted_at_construction() { + let src = EvidenceSource::new(PersonaSourceKind::Codex); + // A realistic transcript leak: an OpenAI-style API key. The composite + // redactor scrubs secrets/tokens/keys before the excerpt is ever stored. + let secret = "sk-abcdefghijklmnopqrstuvwxyz012345"; + let raw = format!("use this key {secret} to call the model"); + let ev = PersonaEvidence::new(src, ts(), EvidenceTier::T2, &raw, vec![]); + // The secret must not survive into the stored excerpt. + assert!( + !ev.excerpt().contains(secret), + "secret leaked into evidence excerpt: {}", + ev.excerpt() + ); + // The id is computed from the redacted excerpt, so it is independent of the + // stripped PII: re-running with the already-redacted text yields the same id. + let ev2 = PersonaEvidence::new( + EvidenceSource::new(PersonaSourceKind::Codex), + ts(), + EvidenceTier::T2, + ev.excerpt(), + vec![], + ); + assert_eq!(ev.id, ev2.id); +} + +#[test] +fn tier_ordering_puts_t0_highest() { + assert!(EvidenceTier::T0 > EvidenceTier::T1); + assert!(EvidenceTier::T1 > EvidenceTier::T2); + assert!(EvidenceTier::T2 > EvidenceTier::T3); + assert_eq!(EvidenceTier::T0.rank(), 3); + assert_eq!(EvidenceTier::T3.rank(), 0); +} + +#[test] +fn tier_parses_loose_forms() { + assert_eq!(EvidenceTier::parse_loose("T1"), Some(EvidenceTier::T1)); + assert_eq!(EvidenceTier::parse_loose(" tier2 "), Some(EvidenceTier::T2)); + assert_eq!(EvidenceTier::parse_loose("0"), Some(EvidenceTier::T0)); + assert_eq!(EvidenceTier::parse_loose("nonsense"), None); +} + +#[test] +fn facet_parses_loose_forms_and_has_seven() { + assert_eq!(PersonaFacet::ALL.len(), 7); + assert_eq!( + PersonaFacet::parse_loose("coding-style"), + Some(PersonaFacet::CodingStyle) + ); + assert_eq!( + PersonaFacet::parse_loose("Anti Preferences"), + Some(PersonaFacet::AntiPreferences) + ); + assert_eq!(PersonaFacet::parse_loose("bogus"), None); + // Directives leads the compile order (T0 budget-protected first). + assert_eq!(PersonaFacet::ALL[0], PersonaFacet::Directives); +} + +#[test] +fn facet_tree_scopes_are_namespaced_and_default_asks_present() { + for f in PersonaFacet::ALL { + assert_eq!(f.tree_scope(), format!("persona/{}", f.as_str())); + assert!(!f.default_ask().trim().is_empty()); + assert!(!f.heading().trim().is_empty()); + } +} + +#[test] +fn wire_strings_are_snake_case() { + assert_eq!(PersonaSourceKind::ChatgptExport.as_str(), "chatgpt_export"); + assert_eq!(PersonaFacet::AntiPreferences.as_str(), "anti_preferences"); + // serde round-trips through the same encoding. + let j = serde_json::to_string(&PersonaSourceKind::GitHistory).unwrap(); + assert_eq!(j, "\"git_history\""); + let f: PersonaFacet = serde_json::from_str("\"coding_style\"").unwrap(); + assert_eq!(f, PersonaFacet::CodingStyle); +} + +#[test] +fn session_digest_empty_helper() { + let d = SessionDigest::empty(EvidenceSource::new(PersonaSourceKind::ClaudeCode)); + assert!(d.is_empty()); +} diff --git a/src/memory/providers/mod.rs b/src/memory/providers/mod.rs index 6bf4f6a..4753662 100644 --- a/src/memory/providers/mod.rs +++ b/src/memory/providers/mod.rs @@ -22,3 +22,9 @@ /// module; the alias exists so future providers (goals **C3**/**M3**) share /// one client type rather than each pulling `reqwest` in independently. pub type HttpClient = reqwest::Client; + +/// OpenRouter reference provider — the crate's first concrete `ChatProvider` / +/// `Summariser` / `EmbeddingBackend` (doc 06 §6.6). A reference implementation +/// only; the persona pipeline depends on the traits, not on this type. +pub mod openrouter; +pub use openrouter::{OpenRouterConfig, OpenRouterProvider, RunUsage}; diff --git a/src/memory/providers/openrouter.rs b/src/memory/providers/openrouter.rs new file mode 100644 index 0000000..d793601 --- /dev/null +++ b/src/memory/providers/openrouter.rs @@ -0,0 +1,398 @@ +//! OpenRouter reference provider (doc 06 §6.6, closes the C3/M3 seam). +//! +//! The crate's **first** concrete LLM provider — a *reference implementation*, +//! not a hard dependency. [`OpenRouterProvider`] implements all three provider +//! seams against OpenRouter's OpenAI-compatible endpoints: +//! +//! - [`ChatProvider`] — `POST /chat/completions` in JSON mode (persona digests). +//! - [`Summariser`] — the same endpoint in plain-text mode (flavoured-tree folds). +//! - [`EmbeddingBackend`] — `POST /embeddings` (vector retrieval). +//! +//! Nothing under `memory::persona` names this type: the pipeline depends only on +//! the traits, and OpenHuman injects its own routes. Secrets are held as +//! [`SecretString`]; token usage is accumulated per-run and the provider aborts +//! cleanly (returns a non-retryable error) once a configured cost/call budget is +//! hit, mirroring the [`DailyBudget`](crate::memory::sync::state::DailyBudget) +//! pattern. Transport and `429`/`5xx` failures retry with backoff; `4xx` client +//! errors (`401`/`402`/`403`) fail fast so the pipeline's +//! `is_non_retryable` classifier short-circuits them. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::json; + +use crate::memory::config::SecretString; +use crate::memory::score::extract::{ChatPrompt, ChatProvider}; +use crate::memory::store::vectors::{format_embedding_signature, EmbeddingBackend}; +use crate::memory::tree::{ + finish_provider_summary, prepare_summary_prompt, Summariser, SummaryCall, SummaryContext, + SummaryInput, SummaryOutput, +}; + +/// Default OpenRouter API base. +pub const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1"; +/// Default chat/digest model (small, fast, cheap). +pub const DEFAULT_CHAT_MODEL: &str = "deepseek/deepseek-v4-flash"; +/// Default embedding model (OpenAI-compatible, 1536 dims). +pub const DEFAULT_EMBED_MODEL: &str = "openai/text-embedding-3-small"; +/// Default embedding dimensionality for [`DEFAULT_EMBED_MODEL`]. +pub const DEFAULT_EMBED_DIMS: usize = 1536; + +/// Declarative configuration for [`OpenRouterProvider`]. +#[derive(Clone, Debug)] +pub struct OpenRouterConfig { + /// API base, e.g. `https://openrouter.ai/api/v1`. + pub base_url: String, + /// Bearer key. Never serialized or printed. + pub api_key: SecretString, + /// Chat/digest model id. + pub chat_model: String, + /// Embedding model id. + pub embed_model: String, + /// Embedding dimensionality (declared, must match the model). + pub embed_dims: usize, + /// Per-request timeout. + pub request_timeout: Duration, + /// Max attempts per request (1 = no retry). + pub max_attempts: u32, + /// Optional hard per-run USD ceiling; the provider aborts once exceeded. + pub run_cost_limit_usd: Option, + /// Optional hard per-run request-count ceiling. + pub run_call_limit: Option, +} + +impl Default for OpenRouterConfig { + fn default() -> Self { + Self { + base_url: DEFAULT_BASE_URL.to_string(), + api_key: SecretString::default(), + chat_model: DEFAULT_CHAT_MODEL.to_string(), + embed_model: DEFAULT_EMBED_MODEL.to_string(), + embed_dims: DEFAULT_EMBED_DIMS, + request_timeout: Duration::from_secs(120), + max_attempts: 4, + run_cost_limit_usd: None, + run_call_limit: None, + } + } +} + +/// Per-run usage accumulator (chat + embeddings), mirroring the `DailyBudget` +/// pattern but denominated in USD and request count. +#[derive(Clone, Debug, Default)] +pub struct RunUsage { + /// Total requests that reached the provider (each retry counts once). + pub requests: u32, + /// Total prompt tokens reported by `usage`. + pub prompt_tokens: u64, + /// Total completion tokens reported by `usage`. + pub completion_tokens: u64, + /// Total USD cost reported by `usage.cost`. + pub cost_usd: f64, +} + +/// OpenRouter-backed provider implementing all three provider seams. +pub struct OpenRouterProvider { + http: reqwest::Client, + cfg: OpenRouterConfig, + usage: Arc>, +} + +impl OpenRouterProvider { + /// Build a provider from config. Fails only if the underlying HTTP client + /// cannot be constructed. + pub fn new(cfg: OpenRouterConfig) -> Result { + let http = reqwest::Client::builder() + .timeout(cfg.request_timeout) + .build() + .context("build OpenRouter HTTP client")?; + Ok(Self { + http, + cfg, + usage: Arc::new(Mutex::new(RunUsage::default())), + }) + } + + /// Snapshot of accumulated per-run usage. + pub fn usage(&self) -> RunUsage { + self.usage.lock().clone() + } + + /// The configured chat model id. + pub fn chat_model(&self) -> &str { + &self.cfg.chat_model + } + + /// Fail-closed budget check run *before* every request. Returns a + /// non-retryable error once a configured ceiling is hit so the pipeline + /// checkpoints and stops cleanly. + fn check_budget(&self) -> Result<()> { + let u = self.usage.lock(); + if let Some(limit) = self.cfg.run_call_limit { + if u.requests >= limit { + return Err(anyhow!( + "run budget exhausted: request limit {limit} reached (non-retryable)" + )); + } + } + if let Some(limit) = self.cfg.run_cost_limit_usd { + if u.cost_usd >= limit { + return Err(anyhow!( + "run budget exhausted: cost limit ${limit:.4} reached (non-retryable)" + )); + } + } + Ok(()) + } + + /// Record the `usage` block of a successful response. + fn record_usage(&self, usage: &serde_json::Value) { + let mut u = self.usage.lock(); + u.requests += 1; + u.prompt_tokens += usage + .get("prompt_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + u.completion_tokens += usage + .get("completion_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + if let Some(cost) = usage.get("cost").and_then(|v| v.as_f64()) { + if cost.is_finite() && cost > 0.0 { + u.cost_usd += cost; + } + } + } + + /// POST `body` to `{base}/{path}` with retry/backoff and return the parsed + /// JSON response. Retries transport failures and `429`/`5xx`; fails fast on + /// `4xx` client errors with the status embedded in the error string. + async fn post_json(&self, path: &str, body: serde_json::Value) -> Result { + self.check_budget()?; + let url = format!("{}/{path}", self.cfg.base_url.trim_end_matches('/')); + let mut last_err: Option = None; + + for attempt in 0..self.cfg.max_attempts.max(1) { + if attempt > 0 { + // Deterministic exponential backoff (no RNG): 400ms, 800ms, … + let backoff = 400u64.saturating_mul(1 << (attempt.min(5) - 1)); + tokio::time::sleep(Duration::from_millis(backoff)).await; + } + + let resp = self + .http + .post(&url) + .bearer_auth(self.cfg.api_key.expose()) + .header("HTTP-Referer", "https://github.com/tinyhumansai/tinycortex") + .header("X-Title", "TinyCortex Persona") + .json(&body) + .send() + .await; + + match resp { + Ok(r) => { + let status = r.status(); + let text = r.text().await.unwrap_or_default(); + if status.is_success() { + return serde_json::from_str(&text) + .with_context(|| format!("decode OpenRouter {path} response")); + } + let code = status.as_u16(); + let snippet: String = text.chars().take(300).collect(); + // 4xx (except 408/429) are permanent client errors — fail fast. + let retryable = code == 408 || code == 429 || (500..600).contains(&code); + let err = anyhow!("OpenRouter {path} HTTP {code}: {snippet}"); + if !retryable { + return Err(err); + } + last_err = Some(err); + } + Err(e) => { + // Transport failure — always retryable. + last_err = Some(anyhow!("OpenRouter {path} transport error: {e}")); + } + } + } + + Err(last_err.unwrap_or_else(|| anyhow!("OpenRouter {path} failed with no error"))) + } + + /// Run one chat completion. When `json_mode` is set the request asks for a + /// JSON object response. Returns the assistant message content. + async fn chat( + &self, + system: &str, + user: &str, + temperature: f32, + max_tokens: Option, + json_mode: bool, + ) -> Result { + let mut body = json!({ + "model": self.cfg.chat_model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "temperature": temperature, + }); + if let Some(mt) = max_tokens { + body["max_tokens"] = json!(mt); + } + if json_mode { + body["response_format"] = json!({"type": "json_object"}); + } + + let resp = self.post_json("chat/completions", body).await?; + if let Some(usage) = resp.get("usage") { + self.record_usage(usage); + } + let content = resp + .get("choices") + .and_then(|c| c.get(0)) + .and_then(|c| c.get("message")) + .and_then(|m| m.get("content")) + .and_then(|c| c.as_str()) + .ok_or_else(|| { + anyhow!("OpenRouter chat response missing choices[0].message.content") + })?; + Ok(content.to_string()) + } +} + +#[async_trait] +impl ChatProvider for OpenRouterProvider { + fn name(&self) -> &str { + "openrouter" + } + + async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result { + self.chat( + &prompt.system, + &prompt.user, + prompt.temperature, + prompt.max_tokens, + true, + ) + .await + } +} + +#[async_trait] +impl Summariser for OpenRouterProvider { + fn name(&self) -> &str { + "openrouter" + } + + async fn summarise( + &self, + inputs: &[SummaryInput], + ctx: &SummaryContext<'_>, + ) -> Result { + Ok(self.summarise_with_usage(inputs, ctx).await?.output) + } + + async fn summarise_with_usage( + &self, + inputs: &[SummaryInput], + ctx: &SummaryContext<'_>, + ) -> Result { + // Reuse the canonical fold prompt (honours the flavoured-tree `ask`). + let prepared = match prepare_summary_prompt(inputs, ctx, None) { + Some(p) => p, + None => return Ok(SummaryCall::default()), + }; + let before = self.usage(); + let text = self + .chat( + &prepared.system, + &prepared.user, + 0.2, + Some(prepared.effective_budget), + false, + ) + .await?; + let after = self.usage(); + let output = finish_provider_summary(&text, ctx.token_budget); + Ok(SummaryCall { + output, + input_tokens: after.prompt_tokens.saturating_sub(before.prompt_tokens), + output_tokens: after + .completion_tokens + .saturating_sub(before.completion_tokens), + charged_amount_usd: Some(after.cost_usd - before.cost_usd), + }) + } +} + +#[async_trait] +impl EmbeddingBackend for OpenRouterProvider { + fn name(&self) -> &str { + "openrouter" + } + + fn model_id(&self) -> &str { + &self.cfg.embed_model + } + + fn dimensions(&self) -> usize { + self.cfg.embed_dims + } + + fn signature(&self) -> String { + format_embedding_signature( + EmbeddingBackend::name(self), + &self.cfg.embed_model, + self.cfg.embed_dims, + ) + } + + async fn embed(&self, texts: &[&str]) -> Result>> { + if texts.is_empty() { + return Ok(Vec::new()); + } + let body = json!({ + "model": self.cfg.embed_model, + "input": texts, + }); + let resp = self.post_json("embeddings", body).await?; + if let Some(usage) = resp.get("usage") { + self.record_usage(usage); + } + let data = resp + .get("data") + .and_then(|d| d.as_array()) + .ok_or_else(|| anyhow!("OpenRouter embeddings response missing data[]"))?; + // The API preserves input order, but honour `index` defensively. + let mut out: Vec> = vec![Vec::new(); data.len()]; + for (i, item) in data.iter().enumerate() { + let idx = item + .get("index") + .and_then(|v| v.as_u64()) + .unwrap_or(i as u64) as usize; + let vec: Vec = item + .get("embedding") + .and_then(|e| e.as_array()) + .ok_or_else(|| anyhow!("OpenRouter embedding item missing embedding[]"))? + .iter() + .map(|v| v.as_f64().unwrap_or(0.0) as f32) + .collect(); + if idx < out.len() { + out[idx] = vec; + } + } + if out.iter().any(|v| v.is_empty()) { + return Err(anyhow!( + "OpenRouter embeddings returned fewer vectors than inputs" + )); + } + Ok(out) + } +} + +#[cfg(test)] +#[path = "openrouter_tests.rs"] +mod tests; diff --git a/src/memory/providers/openrouter_tests.rs b/src/memory/providers/openrouter_tests.rs new file mode 100644 index 0000000..fc7b45a --- /dev/null +++ b/src/memory/providers/openrouter_tests.rs @@ -0,0 +1,194 @@ +//! Wiremock suite for the OpenRouter reference provider (§6.6): happy path, +//! JSON-mode digest, 429 retry, 402 fail-fast, budget cutoff, embeddings. + +use super::*; +use std::time::Duration; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +use crate::memory::score::extract::ChatPrompt; + +fn cfg(base_url: String) -> OpenRouterConfig { + OpenRouterConfig { + base_url, + api_key: SecretString::new("sk-test"), + request_timeout: Duration::from_secs(5), + max_attempts: 4, + ..Default::default() + } +} + +fn chat_completion_body(content: &str) -> serde_json::Value { + json!({ + "id": "gen-1", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15, "cost": 0.0001} + }) +} + +fn prompt(user: &str) -> ChatPrompt { + ChatPrompt { + system: "Return JSON only.".into(), + user: user.into(), + temperature: 0.0, + kind: "persona::digest", + max_tokens: Some(256), + } +} + +#[tokio::test] +async fn chat_for_json_happy_path_returns_content_and_records_usage() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with( + ResponseTemplate::new(200).set_body_json(chat_completion_body("{\"ok\":true}")), + ) + .mount(&server) + .await; + + let provider = OpenRouterProvider::new(cfg(format!("{}/", server.uri()))).unwrap(); + let out = provider.chat_for_json(&prompt("hi")).await.unwrap(); + assert_eq!(out, "{\"ok\":true}"); + + let usage = provider.usage(); + assert_eq!(usage.requests, 1); + assert_eq!(usage.prompt_tokens, 10); + assert!((usage.cost_usd - 0.0001).abs() < 1e-9); +} + +#[tokio::test] +async fn retries_429_then_succeeds() { + let server = MockServer::start().await; + // First response 429 (higher priority, single use), then the 200 fallback. + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(429).set_body_string("rate limited")) + .up_to_n_times(1) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(chat_completion_body("{\"ok\":1}"))) + .with_priority(2) + .mount(&server) + .await; + + let provider = OpenRouterProvider::new(cfg(server.uri())).unwrap(); + let out = provider.chat_for_json(&prompt("hi")).await.unwrap(); + assert_eq!(out, "{\"ok\":1}"); + assert_eq!( + provider.usage().requests, + 1, + "only the successful call counts" + ); +} + +#[tokio::test] +async fn fails_fast_on_402_without_retry() { + let server = MockServer::start().await; + let mock = Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string("requires more credits")) + .expect(1) // must NOT retry a 4xx + .mount_as_scoped(&server) + .await; + + let provider = OpenRouterProvider::new(cfg(server.uri())).unwrap(); + let err = provider.chat_for_json(&prompt("hi")).await.unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("402"), "error should carry the status: {msg}"); + drop(mock); // verifies expect(1) +} + +#[tokio::test] +async fn run_cost_budget_cuts_off_further_calls() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(chat_completion_body("{\"a\":1}"))) + .mount(&server) + .await; + + let mut c = cfg(server.uri()); + c.run_cost_limit_usd = Some(0.00005); // one call (0.0001) exceeds this + let provider = OpenRouterProvider::new(c).unwrap(); + + // First call succeeds and pushes cumulative cost over the ceiling. + provider.chat_for_json(&prompt("one")).await.unwrap(); + // Second call is refused before it hits the network. + let err = provider.chat_for_json(&prompt("two")).await.unwrap_err(); + assert!(format!("{err:#}").contains("budget exhausted")); + assert_eq!(provider.usage().requests, 1); +} + +#[tokio::test] +async fn embeddings_return_vectors_in_order() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "data": [ + {"index": 1, "embedding": [0.1, 0.2, 0.3]}, + {"index": 0, "embedding": [0.4, 0.5, 0.6]} + ], + "usage": {"prompt_tokens": 4, "total_tokens": 4, "cost": 0.00002} + }))) + .mount(&server) + .await; + + let mut c = cfg(server.uri()); + c.embed_dims = 3; + let provider = OpenRouterProvider::new(c).unwrap(); + let vecs = EmbeddingBackend::embed(&provider, &["a", "b"]) + .await + .unwrap(); + assert_eq!(vecs.len(), 2); + // index-0 vector must land first even though it arrived second. + assert_eq!(vecs[0], vec![0.4, 0.5, 0.6]); + assert_eq!(vecs[1], vec![0.1, 0.2, 0.3]); + assert!(EmbeddingBackend::signature(&provider).contains("provider=openrouter")); +} + +#[tokio::test] +async fn summariser_folds_via_chat_and_reports_usage() { + use crate::memory::tree::store::TreeKind; + use chrono::Utc; + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(chat_completion_body("Prefers small commits.")), + ) + .mount(&server) + .await; + + let provider = OpenRouterProvider::new(cfg(server.uri())).unwrap(); + let now = Utc::now(); + let inputs = vec![SummaryInput { + id: "e1".into(), + content: "user: commit small and often".into(), + token_count: 8, + entities: vec![], + topics: vec![], + time_range_start: now, + time_range_end: now, + score: 0.9, + }]; + let ctx = SummaryContext { + tree_id: "t1", + tree_kind: TreeKind::Flavoured, + target_level: 1, + token_budget: 200, + ask: Some("Distill workflow habits."), + }; + let call = Summariser::summarise_with_usage(&provider, &inputs, &ctx) + .await + .unwrap(); + assert!(call.output.content.contains("small commits")); + assert_eq!(call.input_tokens, 10); + assert_eq!(call.output_tokens, 5); +} diff --git a/tests/persona_e2e.rs b/tests/persona_e2e.rs new file mode 100644 index 0000000..1f2af90 --- /dev/null +++ b/tests/persona_e2e.rs @@ -0,0 +1,266 @@ +//! Persona pipeline end-to-end (doc 06 §6.10): readers → digest → facet trees → +//! compiled pack, driven by the OpenRouter reference provider pointed at a +//! wiremock server. Exercises pack structure, tier ordering, incremental +//! cursor resume after a partial run, budget cutoff, and the cost assertion. +//! +//! Requires features: `persona`, `providers-http`, `git-diff`. + +use std::io::Write; +use std::path::Path; +use std::process::Command; + +use serde_json::json; +use tempfile::TempDir; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +use tinycortex::memory::config::{MemoryConfig, SecretString}; +use tinycortex::memory::persona::config::PersonaConfig; +use tinycortex::memory::persona::state::FileStateStore; +use tinycortex::memory::persona::{Pipeline, RunMode}; +use tinycortex::memory::providers::{OpenRouterConfig, OpenRouterProvider}; + +/// A digest response the mock returns for every chat call (JSON-mode digests and +/// plain-text folds alike accept it). +const DIGEST_JSON: &str = r#"{"observations":[ + {"facet":"workflow","observation":"Commits small and often on feature branches","quote":"commit small","tier":"t2"}, + {"facet":"communication","observation":"Terse and directive","quote":"just do it","tier":"t2"}, + {"facet":"coding_style","observation":"Insists on regression tests","quote":"add a test","tier":"t1"} +]}"#; + +fn chat_body() -> serde_json::Value { + json!({ + "id": "gen-1", + "choices": [{"index": 0, "message": {"role": "assistant", "content": DIGEST_JSON}}], + "usage": {"prompt_tokens": 50, "completion_tokens": 30, "total_tokens": 80, "cost": 0.0001} + }) +} + +async fn mock_openrouter() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(chat_body())) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "data": [{"index": 0, "embedding": [0.1, 0.2, 0.3]}], + "usage": {"prompt_tokens": 3, "total_tokens": 3, "cost": 0.0} + }))) + .mount(&server) + .await; + server +} + +fn write(path: &Path, contents: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(contents.as_bytes()).unwrap(); +} + +/// Build fixtures: two Claude Code transcripts, one Codex rollout, an +/// instruction file, and a small git repo. Returns the sources tempdir. +fn build_fixtures() -> TempDir { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + + // Claude Code transcripts. + for (i, name) in ["a.jsonl", "b.jsonl"].iter().enumerate() { + let ts = format!("2026-07-0{}T10:00:00.000Z", i + 1); + write( + &root.join("claude/projects/-work-demo").join(name), + &format!( + r#"{{"type":"user","isSidechain":false,"cwd":"/work/demo","sessionId":"s{i}","timestamp":"{ts}","message":{{"role":"user","content":"implement the parser and commit small"}}}} +{{"type":"assistant","timestamp":"{ts}","message":{{"role":"assistant","content":[{{"type":"text","text":"I loaded the whole file."}}]}}}} +{{"type":"user","isSidechain":false,"sessionId":"s{i}","timestamp":"{ts}","message":{{"role":"user","content":"no, stream it instead"}}}} +"# + ), + ); + } + + // Codex rollout. + write( + &root.join("codex/sessions/2026/07/01/rollout-2026-07-01T10-00-00-x.jsonl"), + r#"{"timestamp":"2026-07-01T10:00:00.000Z","type":"session_meta","payload":{"id":"cx1","cwd":"/work/demo"}} +{"timestamp":"2026-07-01T10:00:02.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"resolve this issue and open a PR"}]}} +"#, + ); + + // Instruction file + git repo under a project root. + let repo = root.join("proj"); + write( + &repo.join("CLAUDE.md"), + "- Always branch before writing code.\n- Commit regularly with clear messages.\n", + ); + init_git_repo(&repo); + + dir +} + +/// Initialise a git repo with one author commit via the `git` CLI. +fn init_git_repo(dir: &Path) { + let run = |args: &[&str]| { + Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "Me") + .env("GIT_AUTHOR_EMAIL", "me@work.com") + .env("GIT_COMMITTER_NAME", "Me") + .env("GIT_COMMITTER_EMAIL", "me@work.com") + .output() + .unwrap(); + }; + run(&["init", "-q"]); + std::fs::write(dir.join("lib.rs"), "fn parse() {}\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-q", "-m", "feat: add streaming parser"]); +} + +fn persona_config(sources: &Path) -> PersonaConfig { + let mut cfg = PersonaConfig::with_home(sources, "me@example.com"); + cfg.claude_code_root = Some(sources.join("claude/projects")); + cfg.codex_root = Some(sources.join("codex/sessions")); + cfg.project_roots = vec![sources.join("proj")]; + cfg.global_instruction_files = vec![]; + cfg.author_emails = vec!["me@work.com".into()]; + cfg +} + +fn provider(server: &MockServer, persona: &PersonaConfig) -> OpenRouterProvider { + OpenRouterProvider::new(OpenRouterConfig { + base_url: server.uri(), + api_key: SecretString::new("sk-test"), + chat_model: persona.chat_model.clone(), + run_cost_limit_usd: Some(persona.run_budget.max_cost_usd), + run_call_limit: Some(persona.run_budget.max_llm_calls), + ..Default::default() + }) + .unwrap() +} + +#[tokio::test] +async fn backfill_produces_structured_pack_within_budget() { + let server = mock_openrouter().await; + let sources = build_fixtures(); + let ws = TempDir::new().unwrap(); + let config = MemoryConfig::new(ws.path()); + let persona = persona_config(sources.path()); + let prov = provider(&server, &persona); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &prov, + summariser: &prov, + store: &store, + }; + let report = pipeline.run(RunMode::Backfill).await.unwrap(); + + // Transcripts (2 Claude + 1 Codex) + git batch were digested. + assert!(report.sessions_processed >= 3, "report: {report:?}"); + assert_eq!(report.directives_folded, 2, "two instruction rules folded"); + assert!(report.observations >= 3); + + // Pack structure: identity header, Directives section present, ordering. + let pack = std::fs::read_to_string(report.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack.starts_with("# Persona: me@example.com")); + assert!(pack.contains("## Directives")); + assert!(pack.contains("Always branch before writing code.")); + assert!(pack.contains("## Workflow")); + let d = pack.find("## Directives").unwrap(); + let w = pack.find("## Workflow").unwrap(); + assert!(d < w, "directives must precede workflow"); + + // Cost assertion: spend stayed under the configured run ceiling. + let usage = prov.usage(); + assert!(usage.requests > 0); + assert!(usage.cost_usd <= persona.run_budget.max_cost_usd); + assert!((usage.cost_usd - usage.requests as f64 * 0.0001).abs() < 1e-9); +} + +#[tokio::test] +async fn incremental_resume_after_partial_backfill() { + let server = mock_openrouter().await; + let sources = build_fixtures(); + let ws = TempDir::new().unwrap(); + let config = MemoryConfig::new(ws.path()); + let mut persona = persona_config(sources.path()); + // Partial: only one session may digest, forcing a checkpoint. + persona.run_budget.max_sessions = 1; + let prov = provider(&server, &persona); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + let first = { + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &prov, + summariser: &prov, + store: &store, + }; + pipeline.run(RunMode::Backfill).await.unwrap() + }; + assert_eq!(first.sessions_processed, 1); + assert!(first.budget_hit); + + // Lift the cap and resume incrementally: the remaining sessions process, + // the already-cursored ones are skipped. + persona.run_budget.max_sessions = 100; + let prov2 = provider(&server, &persona); + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &prov2, + summariser: &prov2, + store: &store, + }; + let second = pipeline.run(RunMode::Incremental).await.unwrap(); + assert!(second.sessions_processed >= 1, "remaining sessions resumed"); + assert!( + second.sessions_skipped >= 1, + "the checkpointed file was skipped" + ); + + // Final pack still well-formed. + let pack = std::fs::read_to_string(second.pack_path.as_ref().unwrap()).unwrap(); + assert!(pack.contains("# Persona: me@example.com")); +} + +#[tokio::test] +async fn offline_compile_only_needs_no_provider() { + // Prove the compile subcommand re-assembles from trees without any LLM. + let server = mock_openrouter().await; + let sources = build_fixtures(); + let ws = TempDir::new().unwrap(); + let config = MemoryConfig::new(ws.path()); + let persona = persona_config(sources.path()); + let prov = provider(&server, &persona); + let store = FileStateStore::open_in_workspace(ws.path()).unwrap(); + + { + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &prov, + summariser: &prov, + store: &store, + }; + pipeline.run(RunMode::Backfill).await.unwrap(); + } + + // Re-compile using a deterministic summariser and a never-called provider. + let concat = tinycortex::memory::tree::summarise::ConcatSummariser::new(); + let pipeline = Pipeline { + config: &config, + persona: &persona, + provider: &prov, // not invoked by compile_only + summariser: &concat, + store: &store, + }; + let path = pipeline.compile_only().unwrap(); + let pack = std::fs::read_to_string(path).unwrap(); + assert!(pack.contains("## Directives")); +}