feat: full-corpus reasoning mining — per-model targets + live progress (2.13.0)

CHANGELOG.md

@@ -5,7 +5,51 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.13.0] — Full-corpus reasoning mining: deep discovery, per-model targets, live progress
+
+### Added
+
+- **Full-corpus reasoning mining.** The miner now discovers transcripts
+  recursively — nested session directories and Task-tool subagent transcripts
+  (`projects/<project>/<session>/subagents/agent-*.jsonl`), not just the top
+  `projects/<project>/*.jsonl` layer — so the ~1000 previously-invisible files
+  are mined. Subagent traces are attributed to their real project, not a
+  literal "subagents" pseudo-project.
+- **Per-model pattern targets.** Each mineable model has a distillation target
+  (0–100), set via a slider on the dashboard Reasoning tab (or
+  `[reasoning_training.pattern_targets]` in config / `PUT /config`). A
+  preliminary Mine with no targets set only DETECTS models; raising a target
+  then distills that model's traces up to the target. Replaces the old global
+  `max_patterns_per_run`.
+- **Live mining progress.** `GET /api/dashboard/reasoning/status` and the
+  dashboard show the phase (scanning → ingesting → distilling → done), files
+  scanned/total, traces found/ingested, and per-model distillation progress
+  while a run is active. The `smem reasoning mine` CLI prints the same, and
+  `docker logs` shows INFO milestones at start / after ingest / at end.
+- **Backfill is a true full re-scan.** `--backfill` (CLI / MCP / dashboard) now
+  re-reads every transcript from the top, bypassing the incremental
+  size+mtime/line skip (trace-hash dedup keeps it idempotent) instead of only
+  widening the lookback window. It never deletes scan state, so a later normal
+  scan stays cheap.
+
+### Changed
+
+- **opus-4-8 is now mined.** It was wrongly denylisted as "signature-only
+  thinking"; the real corpus has ~933 opus-4-8 traces averaging ~1166 chars of
+  genuine reasoning. The prefix-denylist mechanism is kept (currently empty) for
+  a future thinking-less model.
+- **No per-scan trace cap.** `max_traces_per_scan` is removed — mining ingests
+  one transcript at a time (bounded memory) and sees ALL traces. The
+  `max_trace_chars` content-safety limit is raised 20k → 100k.
+
+### Fixed
+
+- **Docker: reasoning mining found zero traces.** `docker-compose.surrealdb.yml`
+  now mounts the host's `~/.claude/projects` read-only into the dashboard
+  container at `/home/appuser/.claude/projects`. Without it the in-container miner
+  scanned an empty `~/.claude` and every dashboard "Mine" finished instantly with
+  0 traces (no error) — looking like nothing happened. Projects-only, read-only
+  mount (not the whole `~/.claude`); override via `HOST_CLAUDE_PROJECTS` in `.env`.
 
 ## [2.12.1] — Release pipeline: npm packages actually ship again
 

dashboard/e2e/reasoning.spec.ts

@@ -27,19 +27,18 @@ const STATUS_BODY = {
     injection_map: {},
     categories: CATEGORIES,
     min_trace_chars: 200,
-    max_trace_chars: 20000,
-    max_traces_per_scan: 500,
+    max_trace_chars: 100000,
     scan_lookback_days: 30,
     retention_days: 90,
     max_traces_total: 20000,
     min_cluster_support: 3,
-    max_patterns_per_run: 10,
     min_confidence: 0.2,
     min_patterns_per_category: 3,
     injection_max_patterns: 5,
     injection_max_chars: 4000,
     distill_use_llm: false,
     redact_secrets: true,
+    pattern_targets: {},
   },
   detected_models: ["claude-fable-5"],
   per_model: [
@@ -67,8 +66,16 @@ const STATUS_BODY = {
     running: false,
     started_at: null,
     finished_at: null,
+    phase: "idle",
+    files_total: 0,
+    files_scanned: 0,
+    traces_found: 0,
     traces_ingested: 0,
+    traces_processed: 0,
     patterns_learned: 0,
+    current_model: null,
+    models_done: 0,
+    models_total: 0,
     dry_run: false,
     error: null,
   },

dashboard/src/api/hooks/useReasoning.ts

@@ -22,8 +22,8 @@ export function useReasoningStatus() {
   return useQuery({
     queryKey: keys.status,
     queryFn: () => api.get<ReasoningStatusResponse>("/api/dashboard/reasoning/status"),
-    // Poll while a mining job is running so counts/coverage update live.
-    refetchInterval: (query) => (query.state.data?.mining.running ? 3000 : false),
+    // Poll while a mining job is running so live progress + counts update.
+    refetchInterval: (query) => (query.state.data?.mining.running ? 1500 : false),
   })
 }
 

dashboard/src/api/types.ts

@@ -487,18 +487,18 @@ export interface ReasoningConfig {
   categories: string[]
   min_trace_chars: number
   max_trace_chars: number
-  max_traces_per_scan: number
   scan_lookback_days: number
   retention_days: number
   max_traces_total: number
   min_cluster_support: number
-  max_patterns_per_run: number
   min_confidence: number
   min_patterns_per_category: number
   injection_max_patterns: number
   injection_max_chars: number
   distill_use_llm: boolean
   redact_secrets: boolean
+  // Per-model distillation targets (model -> desired pattern count, 0..100).
+  pattern_targets: Record<string, number>
 }
 
 export interface ModelTraceStats {
@@ -517,12 +517,22 @@ export interface CategoryCoverage {
   covered: boolean
 }
 
+export type MiningPhase = "idle" | "scanning" | "ingesting" | "distilling" | "done"
+
 export interface MiningJobState {
   running: boolean
   started_at: string | null
   finished_at: string | null
+  phase: MiningPhase
+  files_total: number
+  files_scanned: number
+  traces_found: number
   traces_ingested: number
+  traces_processed: number
   patterns_learned: number
+  current_model: string | null
+  models_done: number
+  models_total: number
   dry_run: boolean
   error: string | null
 }
@@ -546,15 +556,17 @@ export interface ReasoningConfigUpdate {
   mining_models?: string[]
   injection_map?: Record<string, string>
   categories?: string[]
+  min_trace_chars?: number
+  max_trace_chars?: number
   scan_lookback_days?: number
   retention_days?: number
   max_traces_total?: number
   min_cluster_support?: number
-  max_patterns_per_run?: number
   min_confidence?: number
   min_patterns_per_category?: number
   injection_max_patterns?: number
   injection_max_chars?: number
+  pattern_targets?: Record<string, number>
 }
 
 export interface ReasoningConfigUpdateResponse {

dashboard/src/features/reasoning/MiningConfigCard.tsx

@@ -11,12 +11,66 @@ import {
   useUpdateReasoningConfig,
   useWipeTraces,
 } from "@/api/hooks/useReasoning"
-import type { ReasoningStatusResponse } from "@/api/types"
+import type { MiningJobState, ReasoningStatusResponse } from "@/api/types"
 
 interface Props {
   status: ReasoningStatusResponse
 }
 
+function MiningProgressBlock({ mining }: { mining: MiningJobState }) {
+  const { t } = useTranslation()
+  const { phase, files_total, files_scanned, traces_found, traces_ingested } = mining
+  const pct = files_total > 0 ? Math.round((files_scanned / files_total) * 100) : 0
+  const scanning = phase === "scanning" || phase === "ingesting"
+  const phaseLabel =
+    phase === "scanning"
+      ? t("reasoning.progressScanning")
+      : phase === "ingesting"
+        ? t("reasoning.progressIngesting")
+        : phase === "distilling"
+          ? t("reasoning.progressDistilling")
+          : t("reasoning.progressDone")
+
+  return (
+    <div
+      data-testid="mining-progress"
+      className="space-y-2 rounded-md border border-border bg-muted/40 p-3"
+    >
+      <div className="flex items-center justify-between text-xs">
+        <span className="font-medium">{phaseLabel}</span>
+        {scanning && files_total > 0 && (
+          <span className="font-mono tabular-nums text-muted-foreground">
+            {t("reasoning.progressFiles", { scanned: files_scanned, total: files_total })}
+          </span>
+        )}
+      </div>
+      {scanning && files_total > 0 && (
+        <div className="h-2 overflow-hidden rounded-full bg-muted">
+          <div
+            className="h-full rounded-full bg-primary transition-all duration-300"
+            style={{ width: `${pct}%` }}
+          />
+        </div>
+      )}
+      {scanning && (
+        <p className="text-xs text-muted-foreground">
+          {t("reasoning.progressTraces", { found: traces_found, ingested: traces_ingested })}
+        </p>
+      )}
+      {phase === "distilling" && (
+        <p className="font-mono text-xs text-muted-foreground">
+          {t("reasoning.progressModels", {
+            model: mining.current_model ?? "",
+            done: mining.models_done,
+            total: mining.models_total,
+            patterns: mining.patterns_learned,
+          })}
+        </p>
+      )}
+    </div>
+  )
+}
+
 export function MiningConfigCard({ status }: Props) {
   const { t } = useTranslation()
   const updateConfig = useUpdateReasoningConfig()
@@ -169,6 +223,8 @@ export function MiningConfigCard({ status }: Props) {
             {running ? t("reasoning.miningRunning") : t("reasoning.runMining")}
           </Button>
         </div>
+
+        {running && <MiningProgressBlock mining={status.mining} />}
       </CardContent>
 
       <ConfirmDialog

dashboard/src/features/reasoning/PatternTargetsCard.tsx

@@ -0,0 +1,96 @@
+import { useEffect, useState } from "react"
+import { toast } from "sonner"
+import { useTranslation } from "react-i18next"
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
+import { apiErrorMessage } from "@/api/client"
+import { useUpdateReasoningConfig } from "@/api/hooks/useReasoning"
+import type { ReasoningStatusResponse } from "@/api/types"
+
+interface Props {
+  status: ReasoningStatusResponse
+}
+
+const STEP = 10
+const MAX = 100
+
+/**
+ * Per-model distillation targets. Each mineable model gets a 0–100 (step 10)
+ * slider for how many strategy patterns to distill; 0 means "detect only".
+ * Changing a slider PUTs the full pattern_targets map (optimistic via the
+ * shared config mutation).
+ */
+export function PatternTargetsCard({ status }: Props) {
+  const { t } = useTranslation()
+  const updateConfig = useUpdateReasoningConfig()
+
+  // Only models that actually produce thinking text can be distilled.
+  const models = status.per_model.filter((m) => m.has_thinking_text)
+  const configTargets = status.config.pattern_targets
+
+  const [targets, setTargets] = useState<Record<string, number>>(configTargets)
+  // Re-sync local slider state when the server config changes.
+  useEffect(() => {
+    setTargets(configTargets)
+  }, [configTargets])
+
+  const setTarget = (model: string, value: number) => {
+    const next = { ...targets, [model]: value }
+    setTargets(next)
+    updateConfig.mutate(
+      { pattern_targets: next },
+      {
+        onSuccess: () => toast.success(t("reasoning.configSaved")),
+        onError: (err) => toast.error(apiErrorMessage(err, t("reasoning.configSaveFailed"))),
+      },
+    )
+  }
+
+  return (
+    <Card>
+      <CardHeader>
+        <CardTitle>{t("reasoning.targetsTitle")}</CardTitle>
+      </CardHeader>
+      <CardContent className="space-y-4 text-sm">
+        {models.length === 0 ? (
+          <p className="text-xs text-muted-foreground">{t("reasoning.noModels")}</p>
+        ) : (
+          <div className="space-y-4">
+            {models.map((m) => {
+              const value = targets[m.model] ?? 0
+              return (
+                <div key={m.model} className="space-y-1.5">
+                  <div className="flex items-center justify-between gap-2">
+                    <span className="font-mono text-xs">{m.model}</span>
+                    <span className="font-mono text-xs tabular-nums text-muted-foreground">
+                      {value}
+                    </span>
+                  </div>
+                  <input
+                    type="range"
+                    min={0}
+                    max={MAX}
+                    step={STEP}
+                    value={value}
+                    aria-label={m.model}
+                    onChange={(e) => setTarget(m.model, Number(e.target.value))}
+                    className="w-full cursor-pointer accent-primary"
+                  />
+                  <div className="flex items-center justify-between gap-2 text-xs text-muted-foreground">
+                    <span>
+                      {t("reasoning.targetCounts", {
+                        patterns: m.pattern_count,
+                        traces: m.trace_count,
+                      })}
+                    </span>
+                    {value === 0 && <span>{t("reasoning.targetsZeroHint")}</span>}
+                  </div>
+                </div>
+              )
+            })}
+          </div>
+        )}
+        <p className="pt-1 text-xs text-muted-foreground">{t("reasoning.targetsHint")}</p>
+      </CardContent>
+    </Card>
+  )
+}

dashboard/src/features/reasoning/ReasoningPage.tsx

@@ -8,6 +8,7 @@ import { Skeleton } from "@/components/ui/skeleton"
 import { useReasoningStatus } from "@/api/hooks/useReasoning"
 import type { ReasoningStatusResponse } from "@/api/types"
 import { MiningConfigCard } from "./MiningConfigCard"
+import { PatternTargetsCard } from "./PatternTargetsCard"
 import { InjectionMappingCard } from "./InjectionMappingCard"
 import { PatternsTable } from "./PatternsTable"
 
@@ -126,9 +127,11 @@ export default function ReasoningPage() {
 
           <div className="grid grid-cols-1 gap-6 lg:grid-cols-2">
             <MiningConfigCard status={status} />
-            <InjectionMappingCard status={status} />
+            <PatternTargetsCard status={status} />
           </div>
 
+          <InjectionMappingCard status={status} />
+
           <PatternsTable
             detectedModels={status.detected_models}
             categories={status.config.categories}

dashboard/src/i18n/en.json

@@ -96,7 +96,18 @@
     "wipeTracesTitle": "Wipe traces?",
     "wipeTracesDesc": "Permanently delete all staged reasoning traces for \"{{model}}\"? This cannot be undone.",
     "tracesWiped": "Wiped {{count}} trace(s).",
-    "duplicateTarget": "Duplicate target \"{{target}}\" — each target maps to only one source."
+    "duplicateTarget": "Duplicate target \"{{target}}\" — each target maps to only one source.",
+    "targetsTitle": "Pattern targets",
+    "targetsHint": "How many strategy patterns to distill per model. 0 = detect only; raise a slider, then run mining.",
+    "targetsZeroHint": "Set a target above 0 and run mining to distill this model.",
+    "targetCounts": "{{patterns}} patterns · {{traces}} traces",
+    "progressScanning": "Scanning transcripts",
+    "progressIngesting": "Ingesting traces",
+    "progressDistilling": "Distilling patterns",
+    "progressDone": "Done",
+    "progressFiles": "{{scanned}} / {{total}} files",
+    "progressTraces": "{{found}} found · {{ingested}} new",
+    "progressModels": "{{model}} ({{done}}/{{total}}) · {{patterns}} patterns"
   },
   "overview": {
     "title": "Overview",

docker-compose.surrealdb.yml

@@ -101,6 +101,13 @@ services:
         condition: service_healthy
     volumes:
       - nm_data:/data
+      # Reasoning-training mining reads the host's Claude transcripts from
+      # ~/.claude/projects/*/*.jsonl. Mounted READ-ONLY and projects-only (not the
+      # whole ~/.claude, which also holds settings/credentials). Without this the
+      # dashboard "Mine" runs in-container against an empty ~/.claude and silently
+      # finds zero traces. Container user is appuser (uid 1000); override the host
+      # path via HOST_CLAUDE_PROJECTS in .env if your home differs.
+      - ${HOST_CLAUDE_PROJECTS:-${HOME}/.claude/projects}:/home/appuser/.claude/projects:ro
     restart: unless-stopped
 
 volumes: