feat: token budget enforcement -- 100 line default cap, --verbose for full output (OPE-171)

ETH Zurich study (5,694 PRs): auto-generated context files >100 lines
reduced task success 3% and increased cost 20%. LLMs follow ~150
instructions. Claude Code burns ~50 slots before AGENTS.md even loads.
100-line cap leaves ~100 useful instruction slots.

Real-world impact:
- opencodeintel-fork-new: 304 lines -> 101 lines (default)
- Next.js: 3,941 lines -> ~101 lines (default)
- --verbose: full output restored at any time

Design:
- saar/formatters/budget.py: apply_budget(text, max_lines)
  - Splits on ## section headers
  - PROTECTED sections always included: Tribal Knowledge, Project-Specific
    Rules, How to Verify (human-written, highest value)
  - LOW PRIORITY sections cut first: Project Structure, Circular Deps
  - Truncation note tells users about --verbose
- saar/formatters/__init__.py: render() accepts budget= param
- saar/cli.py: --budget N (default 100), --verbose overrides to 0

Added tests/test_budget.py with 22 tests covering:
- Core budget logic (under/over/zero/negative budget)
- Protected sections always survive truncation
- Project Structure cut first
- Truncation note presence/absence/placement
- CLI flags: --verbose, --budget 0, --budget N
- Markdown format exempt from budget

Author: DevanshuNEU

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "saar"
-version = "0.3.2"
+version = "0.3.3"
 description = "Extract the essence of your codebase. Auto-generate AGENTS.md, CLAUDE.md, .cursorrules and more."
 readme = "README.md"
 license = "MIT"

saar/__init__.py

@@ -1,3 +1,3 @@
 """Saar -- extract the essence of your codebase."""
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"

saar/cli.py

@@ -257,12 +257,20 @@ def extract(
     verbose: bool = typer.Option(
         False,
         "--verbose", "-v",
-        help="Show detailed analysis progress.",
+        help="Show full output without line cap. Also enables debug logs.",
+    ),
+    budget: int = typer.Option(
+        100,
+        "--budget",
+        help="Max lines in generated file (default 100). 0 = unlimited. --verbose overrides to 0.",
+        min=0,
     ),
 ) -> None:
     """Analyze a codebase and extract its architectural DNA."""
     log_level = logging.DEBUG if verbose else logging.WARNING
     logging.basicConfig(level=log_level, format="%(message)s")
+    # --verbose disables line cap -- full output
+    effective_budget = 0 if verbose else budget
 
     console.print(f"[bold]saar[/bold] analyzing [cyan]{repo_path.name}[/cyan]...")
 
@@ -330,7 +338,7 @@ def extract(
     from saar.formatters import render
 
     for fmt in target_formats:
-        text = render(dna, fmt.value)
+        text = render(dna, fmt.value, budget=effective_budget)
         target = _resolve_output_path(fmt, output, repo_path)
 
         if target is None:

saar/formatters/__init__.py

@@ -20,9 +20,24 @@
 }
 
 
-def render(dna: CodebaseDNA, format: str) -> str:
-    """Render DNA in the given format. Raises KeyError for unknown formats."""
+def render(dna: CodebaseDNA, format: str, budget: int = 100) -> str:
+    """Render DNA in the given format, applying a line budget.
+
+    Args:
+        dna: Extracted codebase DNA.
+        format: Output format key (agents, claude, cursorrules, copilot, markdown).
+        budget: Max lines in output. 0 = unlimited (--verbose mode).
+
+    Raises:
+        KeyError: Unknown format string.
+    """
+    from saar.formatters.budget import apply_budget
+
     renderer = _RENDERERS.get(format)
     if renderer is None:
         raise KeyError(f"Unknown format: {format}. Options: {list(_RENDERERS.keys())}")
-    return renderer(dna)
+    text = renderer(dna)
+    # markdown format goes to stdout for human reading -- no budget applied
+    if format == "markdown":
+        return text
+    return apply_budget(text, budget)

saar/formatters/budget.py

@@ -0,0 +1,145 @@
+"""Token budget enforcement for generated AI context files.
+
+Research context (why this exists):
+    ETH Zurich study (5,694 PRs): auto-generated context files >100 lines
+    reduced agent task success by 3% and increased cost by 20%.
+    LLMs reliably follow ~150 instructions. Claude Code burns ~50 before
+    AGENTS.md even loads. That leaves ~100 slots for our content.
+
+Default cap: 100 lines.
+--verbose / --budget 0: unlimited.
+
+Design:
+    Sections marked as PROTECTED are always included regardless of budget.
+    These are human-written or contain the highest-value tribal knowledge.
+    Auto-generated bulk content (Project Structure, Circular Deps) is cut first.
+
+    The function works on already-rendered text -- it does NOT re-render.
+    It splits on ## section headers, applies priority ordering, and reassembles
+    within the budget. Protected sections are appended after the budget note.
+"""
+from __future__ import annotations
+
+# Lines cap below which we never bother truncating (avoids truncating small repos)
+_MIN_LINES_TO_TRUNCATE = 20
+
+# Section header prefixes that are ALWAYS included, never cut.
+# These contain human-written tribal knowledge and project-specific rules.
+_PROTECTED_SECTION_PREFIXES = (
+    "## Tribal Knowledge",
+    "## Project-Specific Rules",
+    "## How to Verify",
+)
+
+# Section header prefixes ranked lowest priority -- cut first when over budget.
+# Project Structure is the biggest offender: monorepos generate 100s of lines.
+_LOW_PRIORITY_SECTION_PREFIXES = (
+    "## Project Structure",
+    "## Circular Dependencies",
+    "## Preferred imports",
+)
+
+_TRUNCATION_NOTE = (
+    "\n> [{omitted} lines omitted -- run `saar extract --verbose` for full output]\n"
+)
+
+
+def apply_budget(text: str, max_lines: int) -> str:
+    """Apply a line budget to rendered AGENTS.md / CLAUDE.md content.
+
+    Args:
+        text: Fully rendered content string (without SAAR markers).
+        max_lines: Maximum lines allowed. 0 or negative = unlimited.
+
+    Returns:
+        Content string within budget, with a truncation note if lines were cut.
+        Protected sections (Tribal Knowledge, Project Rules) are always included.
+    """
+    if max_lines <= 0:
+        return text
+
+    lines = text.splitlines(keepends=True)
+    total = len(lines)
+
+    if total <= max_lines or total <= _MIN_LINES_TO_TRUNCATE:
+        return text
+
+    # Split into sections. Each section = (header_line_index, lines[])
+    sections = _split_into_sections(lines)
+
+    # Separate protected sections out -- they always appear at the end
+    protected: list[list[str]] = []
+    regular: list[list[str]] = []
+
+    for section_lines in sections:
+        header = section_lines[0].strip() if section_lines else ""
+        if any(header.startswith(p) for p in _PROTECTED_SECTION_PREFIXES):
+            protected.append(section_lines)
+        else:
+            regular.append(section_lines)
+
+    # Sort regular sections: low-priority ones go to the end (cut first)
+    def _priority(section_lines: list[str]) -> int:
+        header = section_lines[0].strip() if section_lines else ""
+        if any(header.startswith(p) for p in _LOW_PRIORITY_SECTION_PREFIXES):
+            return 99  # sort last = cut first
+        return 0
+
+    regular.sort(key=_priority)
+
+    # Count lines reserved for protected sections + truncation note
+    protected_line_count = sum(len(s) for s in protected) + 2  # +2 for note
+    available = max_lines - protected_line_count
+
+    # Fill regular sections within available budget
+    kept: list[list[str]] = []
+    used = 0
+    omitted = 0
+
+    for section_lines in regular:
+        section_len = len(section_lines)
+        if used + section_len <= available:
+            kept.append(section_lines)
+            used += section_len
+        else:
+            omitted += section_len
+
+    # Reassemble: kept sections (in original order) + note + protected
+    # Re-sort kept back to original document order
+    original_order = {id(s): i for i, s in enumerate(sections)}
+    kept.sort(key=lambda s: original_order.get(id(s), 999))
+
+    result_lines: list[str] = []
+    for section_lines in kept:
+        result_lines.extend(section_lines)
+
+    if omitted > 0:
+        note = _TRUNCATION_NOTE.format(omitted=omitted)
+        result_lines.append(note)
+
+    for section_lines in protected:
+        result_lines.extend(section_lines)
+
+    return "".join(result_lines)
+
+
+def _split_into_sections(lines: list[str]) -> list[list[str]]:
+    """Split a list of lines into sections delimited by ## headers.
+
+    The preamble (lines before the first ## header) is treated as
+    its own section with an empty header line.
+    """
+    sections: list[list[str]] = []
+    current: list[str] = []
+
+    for line in lines:
+        if line.startswith("## ") and current:
+            sections.append(current)
+            current = [line]
+        else:
+            current.append(line)
+
+    if current:
+        sections.append(current)
+
+    return sections