From 79c222f72fc86bfbd385151909d6c576441c71e1 Mon Sep 17 00:00:00 2001 From: Dmitry Teryaev Date: Wed, 8 Jul 2026 16:04:27 +0300 Subject: [PATCH] =?UTF-8?q?chore(absence):=20follow-ups=20=E2=80=94=20seri?= =?UTF-8?q?alization=20hardening,=20renderer=20DRY,=20hygiene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the deferred follow-up backlog from the absence-diagnosis review. No change to the diagnosis logic itself; full suite green (1261 passed). absence_vocab.py - FORMAT_VERSION constant + load() validation: a sidecar with an unknown/ changed JSON shape now raises VocabIndexStale (-> rebuild) instead of being misread. format_version was written but never checked. - Atomic save(): dump to a temp sibling + os.replace, so a crash mid-write leaves the previous complete file or the new one -- never a truncated sidecar. Explicit utf-8 / ensure_ascii=False. - Drop _name_index from the sidecar: it is derivable from records and is rebuilt in __init__ on load (single source of truth, no bloat). Backward compatible (old sidecars with name_index load fine; the field is ignored). - Collapse the redundant except-tuple in get_vocabulary_index (Exception subsumes VocabIndexStale/FileNotFoundError). jrag_render.py - Extract _ABSENCE_VERDICT_TEXT + _verdict_line() to DRY the verdict->label mapping that was triplicated across not-found / listing / traversal empty. - Render absence.message in the not-found block (the authored per-cause help was being dropped on the CLI surface). - Remove unreachable else branches (verdict is a closed Literal). absence_diagnosis.py - Correct the _threshold_verdict docstring: only absence_absent_floor decides; absence_close_threshold is transparency-only on this path (it decides the find path). The old "two-band" framing implied otherwise. Hygiene - Stop tracking the generated vocab_index.json (it dirtied every tree on a test/build run via a regenerated built_at). gitignore'd + untracked; the file is rebuilt on demand. Deliberately NOT done (would not improve correctness) - Compare sidecar ontology_version to the live graph version vs the code constant: the code-constant compare is the safer/correct choice for schema incompatibility; the live-graph variant could accept an index built under an older ontology. - Add a default to SymbolRecord.resolved: leaving it required is safer (explicit > implicit); _row_to_symbol_record already defaults missing->True. Co-Authored-By: Claude --- .gitignore | 3 ++ absence_diagnosis.py | 14 ++++--- absence_vocab.py | 44 +++++++++++++++------ java_codebase_rag/jrag_render.py | 67 ++++++++++++++++---------------- vocab_index.json | 1 - 5 files changed, 79 insertions(+), 50 deletions(-) delete mode 100644 vocab_index.json diff --git a/.gitignore b/.gitignore index e82ff355..9e6c4ad9 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ lancedb_data/ cocoindex_java_lance.db/ *.kuzu *.kuzu.wal +# Generated vocabulary-index sidecar (absence diagnosis; rebuilt on demand) +vocab_index.json +vocab_index.json.tmp # Local env files .env diff --git a/absence_diagnosis.py b/absence_diagnosis.py index 0772fdc9..9d42f81f 100644 --- a/absence_diagnosis.py +++ b/absence_diagnosis.py @@ -335,11 +335,15 @@ def _did_you_mean( def _threshold_verdict( best_sim: float, cfg: Any, *, identifier: bool, ) -> tuple[str, str, AbsenceProof | None]: - """Conservative two-band threshold policy. - - Returns (verdict, cause, proof). Middle band defaults to ``refine_query``; - ``not_in_project`` only when best similarity < ``absence_absent_floor`` AND - the query is identifier-shaped. + """Conservative threshold policy with a single deciding band. + + Returns (verdict, cause, proof). ``not_in_project`` only when best + similarity < ``absence_absent_floor`` AND the query is identifier-shaped; + everything else (a close hit OR the middle band) → ``refine_query``, so a + real symbol is never falsely declared absent. ``absence_close_threshold`` + is NOT a decider on this path (it decides the *find* path's "close hit + excluded by filter/scope" branch at the ``best_sim >= close`` check) — it + is recorded in ``proof.thresholds_applied`` for transparency only. """ close = float(getattr(cfg, "absence_close_threshold", 0.85)) floor = float(getattr(cfg, "absence_absent_floor", 0.40)) diff --git a/absence_vocab.py b/absence_vocab.py index b76e39eb..3fc1017c 100644 --- a/absence_vocab.py +++ b/absence_vocab.py @@ -10,6 +10,7 @@ import json import logging +import os from dataclasses import dataclass from pathlib import Path from typing import Any @@ -27,6 +28,10 @@ VOCAB_INDEX_FILENAME = "vocab_index.json" +# Sidecar schema version. Bump when the on-disk JSON shape changes; load() rejects +# a mismatch as stale (→ rebuild) so an old-format sidecar is never misread. +FORMAT_VERSION = 1 + class VocabIndexStale(Exception): """Raised when loading a vocab index with stale ontology_version.""" @@ -148,7 +153,7 @@ def save(self, path: Path, *, ontology_version: int) -> None: import time data = { - "format_version": 1, + "format_version": FORMAT_VERSION, "ontology_version": ontology_version, "built_at": int(time.time()), "symbol_count": self.symbol_count, @@ -168,12 +173,20 @@ def save(self, path: Path, *, ontology_version: int) -> None: for r in self.records ], "ngrams": self.ngram_index, - "name_index": self._name_index, + # _name_index is intentionally NOT persisted: it is derivable from + # records and rebuilt in __init__ on load (single source of truth, + # no sidecar bloat). } path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(data, f) + # Atomic write — dump to a temp sibling, then os.replace onto the target. + # A crash mid-write leaves either the previous complete file or the new + # complete file, never a truncated/corrupt sidecar (readers see one or + # the other atomically; os.replace is atomic on the same filesystem). + tmp_path = path.with_name(path.name + ".tmp") + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) + os.replace(tmp_path, path) log.debug(f"VocabularyIndex saved to {path} ({self.symbol_count} symbols)") @@ -188,14 +201,22 @@ def load(cls, path: Path) -> "VocabularyIndex": VocabularyIndex Raises: - VocabIndexStale: If sidecar ontology_version doesn't match expected + VocabIndexStale: If sidecar format_version or ontology_version + doesn't match expected """ from ast_java import ONTOLOGY_VERSION - with open(path) as f: + with open(path, encoding="utf-8") as f: data = json.load(f) - # Check ontology version + # Check format version (sidecar JSON schema) first. + if data.get("format_version") != FORMAT_VERSION: + raise VocabIndexStale( + f"Vocab index format_version {data.get('format_version')} " + f"does not match expected {FORMAT_VERSION}" + ) + + # Check ontology version (graph schema the index was built against). if data.get("ontology_version") != ONTOLOGY_VERSION: raise VocabIndexStale( f"Vocab index ontology version {data.get('ontology_version')} " @@ -217,11 +238,11 @@ def load(cls, path: Path) -> "VocabularyIndex": for r in data["records"] ] + # _name_index is rebuilt from records in __init__ (not persisted). return cls( records=records, ngram_index=data["ngrams"], q=data["q"], - _name_index=data.get("name_index"), ) def lookup(self, name: str, *, limit: int) -> list[SymbolRecord]: @@ -349,9 +370,10 @@ def get_vocabulary_index(graph: Any, cfg: Any) -> VocabularyIndex: _vocab_cache[db_path] = index log.debug(f"Loaded vocabulary index from {sidecar_path}") return index - except (VocabIndexStale, FileNotFoundError, Exception) as e: - # Stale, missing, or corrupt - rebuild from graph - log.debug(f"Vocab index missing/stale ({e}), rebuilding from graph") + except Exception as e: + # Stale (VocabIndexStale), missing (FileNotFoundError), or corrupt + # (JSONDecodeError/KeyError) — all subsumed by Exception; rebuild. + log.debug(f"Vocab index missing/stale/corrupt ({e}), rebuilding from graph") # Build from graph index = VocabularyIndex.build(graph, q=cfg.absence_ngram_q) diff --git a/java_codebase_rag/jrag_render.py b/java_codebase_rag/jrag_render.py index 824bf2e8..f010d8f4 100644 --- a/java_codebase_rag/jrag_render.py +++ b/java_codebase_rag/jrag_render.py @@ -33,6 +33,23 @@ # none) and are left untagged. _ROUTE_KIND_TAGS: dict[str, str] = {"kafka_topic": "kafka", "http_endpoint": "http"} +# Absence verdict → human-readable label, shared by the not-found / listing / +# traversal empty-result renderers. ``AbsenceVerdict`` is a closed Literal of +# these four values. +_ABSENCE_VERDICT_TEXT: dict[str, str] = { + "not_in_project": "not in project", + "external_dependency": "external dependency", + "refine_query": "refine your query", + "correct_empty": "correct empty", +} + + +def _verdict_line(absence: AbsenceDiagnosis) -> str | None: + """A ``Verdict: