diff --git a/absence_diagnosis.py b/absence_diagnosis.py new file mode 100644 index 00000000..0772fdc9 --- /dev/null +++ b/absence_diagnosis.py @@ -0,0 +1,696 @@ +"""Stateless absence diagnosis (PR-ABS-2) — the feature's core. + +``diagnose(...)`` is the single place empty-MCP-result logic lives. It classifies +an empty exploration result by cause and emits cause-specific help. Pure function +of its inputs (incl. the :class:`VocabularyIndex`); no I/O, no mutation. Consumed +by PR-ABS-3 (MCP wiring) and PR-ABS-4 (CLI). + +Similarity metric +----------------- +Identifier did-you-mean uses ``difflib.SequenceMatcher(None, a, b).ratio()`` +(stdlib, ∈ [0,1]) on the query's normalized name vs each candidate's +``normalized_name``; ``distance = 1.0 - similarity``. ``difflib`` is stdlib and +adequate for identifier typo/misremember detection — Jaro-Winkler would be +marginally better but is not stdlib and not worth a dependency. See the task +brief's resolution 1. + +Conservative absence +-------------------- +False-absent (declaring a real symbol absent) is the catastrophic failure mode. +The two-band threshold policy defaults the middle band to ``refine_query`` and +commits to ``not_in_project`` only when best similarity < ``absence_absent_floor`` +AND the query is identifier-shaped. ``closest_symbols``/``distances`` are ALWAYS +populated regardless of verdict. +""" + +from __future__ import annotations + +import logging +from collections import Counter +from difflib import SequenceMatcher +from typing import Any, Literal + +from absence_types import ( + AbsenceDiagnosis, + AbsenceProof, + ExternalIdentity, + FilterRelaxation, + FilterRelaxationDim, + VocabularyContext, +) +from absence_vocab import SymbolRecord, VocabularyIndex, _normalize_name +from graph_types import NodeRef +from mcp_hints import _IDENTIFIER_FILTER_FIELDS + +log = logging.getLogger(__name__) + +__all__ = ["diagnose"] + +# Dimensions whose relaxation _filter_relaxation can probe on the graph. These +# mirror the single-dim filters handled by ``_zero_result_guidance`` (jrag.py). +_RELAXABLE_DIMS: tuple[str, ...] = ("role", "microservice", "module") + +# Node-id prefixes (graph_types._node_kind_from_id). Used to tell a describe +# node_id miss apart from an FQN lookup. +_NODE_ID_PREFIXES: tuple[str, ...] = ( + "ucs:", "sym:", "route:", "r:", "client:", "c:", "producer:", "p:", +) + +# Small English stopword set; a single stopword is treated as NL, not identifier. +_STOPWORDS: frozenset[str] = frozenset({ + "the", "a", "an", "and", "or", "of", "in", "to", "for", "with", "on", "at", + "by", "is", "are", "be", "was", "were", "how", "does", "do", "what", "where", + "which", "who", "why", "when", "find", "show", "get", "list", "all", "any", + "this", "that", "these", "those", "from", "into", "use", "using", "used", +}) + + +# --------------------------------------------------------------------------- # +# Public entry point # +# --------------------------------------------------------------------------- # + + +def diagnose( + *, + tool: Literal["search", "find", "neighbors", "describe", "resolve"], + query: str | None, + filt: dict | None, # find's model_dump'd filter + filter_kind: str | None, # find's kind, for identifier-shape test + root_node: NodeRef | None, # neighbors/describe subject + scope: dict[str, str], # {"microservice":..,"module":..} + vocab: VocabularyIndex, + graph: Any, # LadybugGraph + cfg: Any, # ResolvedOperatorConfig (thresholds) +) -> AbsenceDiagnosis | None: + """Classify an empty result and emit cause-specific help. + + Returns ``None`` when the master toggle is off or on unrecoverable error. + Never raises: any exception is logged and degrades to a minimal + ``refine_query`` (or ``None`` if even that cannot be built). + """ + try: + if not getattr(cfg, "absence_diag_enabled", True): + return None + return _diagnose_inner( + tool=tool, + query=query, + filt=filt, + filter_kind=filter_kind, + root_node=root_node, + scope=scope, + vocab=vocab, + graph=graph, + cfg=cfg, + ) + except Exception: # noqa: BLE001 — diagnosis must never fail the tool + log.exception("absence diagnosis failed; degrading to refine_query") + return _fallback_refine() + + +# --------------------------------------------------------------------------- # +# Decision procedure # +# --------------------------------------------------------------------------- # + + +def _diagnose_inner( + *, + tool: str, + query: str | None, + filt: dict | None, + filter_kind: str | None, + root_node: NodeRef | None, + scope: dict[str, str], + vocab: VocabularyIndex, + graph: Any, + cfg: Any, +) -> AbsenceDiagnosis | None: + # --- External-wins: emit external_dependency first for any external target. + ext = _detect_external(query, filt, filter_kind, root_node, vocab) + if ext is not None: + return AbsenceDiagnosis( + verdict="external_dependency", + cause="external", + message=( + f"`{ext.fqn}` is referenced by this project but not defined in it " + f"({ext.reason}). It is an external dependency." + ), + external_identity=ext, + ) + + # --- neighbors/describe subject present. + if root_node is not None: + return _diagnose_neighbors(root_node, graph) + + # --- describe by node_id (not fqn): an unknown id, not a misspelled name. + if tool == "describe" and query and _looks_like_node_id(query): + return AbsenceDiagnosis( + verdict="refine_query", + cause="identifier_miss", + message=( + f"No node with id `{query}`. Run `resolve` to map a name/FQN to an id, " + "or `search` to discover symbols." + ), + ) + + # --- find (filter) path. + if filt is not None and filter_kind is not None: + return _diagnose_find(filt, filter_kind, scope, vocab, graph, cfg) + + # --- search/resolve/describe-by-fqn (query) path. + if query: + return _diagnose_query(query, vocab, graph, cfg) + + # Nothing to classify on (no query, no filt, no root_node). Be conservative. + return _fallback_refine() + + +def _diagnose_query( + query: str, vocab: VocabularyIndex, graph: Any, cfg: Any, +) -> AbsenceDiagnosis: + # Empty vocab guard: never declare not_in_project on an unindexed/empty graph + if vocab.symbol_count == 0: + return AbsenceDiagnosis( + verdict="refine_query", + cause="identifier_miss", + message=( + "Index appears empty/unindexed — verify the project was indexed " + "before concluding a symbol is absent." + ), + ) + + if _is_identifier_shaped(query): + closest, distances, best_sim = _did_you_mean(query, vocab, cfg) + verdict, cause, proof = _threshold_verdict(best_sim, cfg, identifier=True) + # False-absent guard: if the query exactly resolves to a real project + # symbol (simple name OR FQN, case-insensitive), never declare not_in_project. + if verdict == "not_in_project" and _exact_symbol_exists(query, vocab): + verdict, proof = "refine_query", None + if proof is not None: + proof.symbol_count_scanned = vocab.symbol_count + message = _identifier_message(query, verdict, closest) + return AbsenceDiagnosis( + verdict=verdict, + cause=cause, + message=message, + closest_symbols=closest, + distances=distances, + proof=proof, + ) + # Natural language → assemble vocabulary context, no did-you-mean. + ctx = _build_vocabulary_context(graph, vocab) + return AbsenceDiagnosis( + verdict="refine_query", + cause="nl_miss", + message=( + f"No symbol matches `{query}`. Refine the query — try an identifier " + "(class/method/FQN) or browse the project vocabulary below." + ), + vocabulary_context=ctx, + ) + + +def _diagnose_find( + filt: dict, + filter_kind: str, + scope: dict[str, str], + vocab: VocabularyIndex, + graph: Any, + cfg: Any, +) -> AbsenceDiagnosis: + # Empty vocab guard: never declare not_in_project on an unindexed/empty graph + if vocab.symbol_count == 0: + return AbsenceDiagnosis( + verdict="refine_query", + cause="identifier_miss", + message=( + "Index appears empty/unindexed — verify the project was indexed " + "before concluding a symbol is absent." + ), + ) + + identifier = _extract_identifier(filt, filter_kind) + + if identifier is not None: + # Identifier-shaped filter: run did-you-mean on the identifier value. + closest, distances, best_sim = _did_you_mean(identifier, vocab, cfg) + if best_sim >= cfg.absence_close_threshold: + # Close hit exists → the filter (or scope) excluded it. Show where it lives. + relax = _filter_relaxation(filt, filter_kind, scope, graph, identifier) + return AbsenceDiagnosis( + verdict="refine_query", + cause="filter_miss", + message=( + f"No results for `{identifier}` under the current filter. " + "Close matches exist — try relaxing a dimension (see filter_relaxation)." + ), + closest_symbols=closest, + distances=distances, + filter_relaxation=relax, + ) + # No close hit → identifier miss; apply the conservative threshold. + verdict, cause, proof = _threshold_verdict(best_sim, cfg, identifier=True) + if verdict == "not_in_project" and _exact_symbol_exists(identifier, vocab): + verdict, proof = "refine_query", None + if proof is not None: + proof.symbol_count_scanned = vocab.symbol_count + return AbsenceDiagnosis( + verdict=verdict, + cause=cause, + message=_identifier_message(identifier, verdict, closest), + closest_symbols=closest, + distances=distances, + proof=proof, + ) + + # Broad / non-identifier filter → filter_miss with relaxation suggestions. + relax = _filter_relaxation(filt, filter_kind, scope, graph, None) + return AbsenceDiagnosis( + verdict="refine_query", + cause="filter_miss", + message=( + "No results under the current filter. Matches exist under other values " + "(see filter_relaxation)." + ), + filter_relaxation=relax, + ) + + +def _diagnose_neighbors(root_node: NodeRef, graph: Any) -> AbsenceDiagnosis: + if _neighbors_meaningful_empty(root_node, graph): + return AbsenceDiagnosis( + verdict="correct_empty", + cause="meaningful_empty", + message=( + f"`{root_node.fqn or root_node.id}` has no neighbors of the requested " + "type here — this is a genuine leaf / external entrypoint, not an error." + ), + ) + return AbsenceDiagnosis( + verdict="refine_query", + cause="identifier_miss", + message=( + f"No neighbors for `{root_node.fqn or root_node.id}` with the requested " + "edge type/direction. Run `describe` and inspect `edge_summary` for the " + "edge types this node actually participates in." + ), + ) + + +# --------------------------------------------------------------------------- # +# Did-you-mean + thresholds # +# --------------------------------------------------------------------------- # + + +def _did_you_mean( + identifier: str, vocab: VocabularyIndex, cfg: Any, +) -> tuple[list[NodeRef], list[float], float]: + """Rank vocabulary candidates by SequenceMatcher similarity to ``identifier``. + + Returns (closest_symbols, distances, best_similarity). When n-gram lookup + yields no candidates (no q-gram overlap at all), falls back to a bounded + linear scan over all records so ``closest_symbols`` is still populated — + this backs the "nearest-by-name" guarantee on the ``not_in_project`` path. + """ + limit = int(getattr(cfg, "absence_candidate_count", 5)) + query_norm = _normalize_name(identifier) + + candidates = vocab.lookup(identifier, limit=limit) + if not candidates and vocab.records: + # Rare: totally novel token with zero q-gram overlap. Bounded scan for + # the nearest-by-name records so not_in_project still shows nearest names. + candidates = vocab.records + + scored: list[tuple[SymbolRecord, float]] = [ + (rec, _similarity(query_norm, rec.normalized_name)) for rec in candidates + ] + scored.sort(key=lambda pair: pair[1], reverse=True) + + top = scored[:limit] + closest = [_build_node_ref(rec) for rec, _ in top] + distances = [round(1.0 - sim, 4) for _, sim in top] + best_sim = top[0][1] if top else 0.0 + return closest, distances, best_sim + + +def _threshold_verdict( + best_sim: float, cfg: Any, *, identifier: bool, +) -> tuple[str, str, AbsenceProof | None]: + """Conservative two-band threshold policy. + + Returns (verdict, cause, proof). Middle band defaults to ``refine_query``; + ``not_in_project`` only when best similarity < ``absence_absent_floor`` AND + the query is identifier-shaped. + """ + close = float(getattr(cfg, "absence_close_threshold", 0.85)) + floor = float(getattr(cfg, "absence_absent_floor", 0.40)) + if best_sim < floor and identifier: + proof = AbsenceProof( + nearest_distance=round(1.0 - best_sim, 4), + symbol_count_scanned=0, # filled by caller via vocab + thresholds_applied={"absence_close_threshold": close, "absence_absent_floor": floor}, + query_shape="identifier", + ) + return "not_in_project", "identifier_miss", proof + # close band OR middle band → refine_query (conservative; never false-absent). + return "refine_query", "identifier_miss", None + + +def _identifier_message( + query: str, verdict: str, closest: list[NodeRef], +) -> str: + if verdict == "not_in_project": + return ( + f"No symbol matching `{query}` was found in the project vocabulary. " + "It does not appear to be defined here." + ) + if closest: + names = ", ".join(s.name or s.fqn for s in closest[:3]) + return ( + f"No exact match for `{query}`. Closest symbols: {names}. " + "Refine the query (typo? scope?) and retry." + ) + return f"No match for `{query}`. Refine the query and retry." + + +# --------------------------------------------------------------------------- # +# External detection # +# --------------------------------------------------------------------------- # + + +def _detect_external( + query: str | None, + filt: dict | None, + filter_kind: str | None, + root_node: NodeRef | None, + vocab: VocabularyIndex, +) -> ExternalIdentity | None: + """External-wins: if the target is external/phantom, emit its identity first.""" + # root_node (neighbors/describe subject). + if root_node is not None: + if root_node.kind == "unresolved_call_site": + return ExternalIdentity(fqn=root_node.fqn, reason="unresolved-call") + if root_node.fqn: + ext = _external_identity_for(root_node.fqn, vocab) + if ext is not None: + return ext + + # free-text query (search/resolve/describe-by-fqn). + if query: + ext = _external_identity_for(query, vocab) + if ext is not None: + return ext + + # find identifier filter value. + identifier = _extract_identifier(filt, filter_kind) if filt is not None else None + if identifier: + ext = _external_identity_for(identifier, vocab) + if ext is not None: + return ext + + return None + + +def _external_identity_for(name: str, vocab: VocabularyIndex) -> ExternalIdentity | None: + is_ext, reason = vocab.is_external(name) + if is_ext and reason in ("prefix", "phantom"): + # Prefer the corpus FQN when we can resolve it (richer than the bare query). + fqn = name + for rec in vocab.records: + if rec.simple_name == name or rec.fqn == name: + fqn = rec.fqn or name + break + return ExternalIdentity(fqn=fqn, reason=reason) + return None + + +# --------------------------------------------------------------------------- # +# Neighbors meaningful-empty detection # +# --------------------------------------------------------------------------- # + + +def _neighbors_meaningful_empty(root_node: NodeRef, graph: Any) -> bool: + """A genuine leaf or external entrypoint → ``correct_empty``. + + Reuses the conditions behind ``is_external_entrypoint`` (jrag.py:2452): an + HTTP ``http_endpoint`` route with inbound handlers is an external entrypoint, + so zero callers is meaningful. A symbol with no edges at all is an isolated + leaf. Everything else (has edges, just not the requested type) → refine. + + Kafka topics are not considered external entrypoints: their empty-callers + semantics differ from HTTP routes (per is_external_entrypoint precedent). + """ + try: + if root_node.kind == "route": + # Fetch the route's kind property (http_endpoint vs kafka_topic). + kind_rows = graph._rows( + "MATCH (r:Route) WHERE r.id = $id RETURN r.kind AS k", + {"id": root_node.id}, + ) + route_kind = kind_rows[0].get("k") if kind_rows else "" + # Only http_endpoint routes with handlers are external entrypoints. + if route_kind == "http_endpoint": + handlers = graph.find_route_handlers(route_id=root_node.id) + if handlers: + return True + # kafka_topic routes and handler-less routes are NOT meaningful empty. + return False + # Symbol/other: meaningful empty only if it has zero edges (isolated leaf). + rows = graph._rows( # noqa: SLF001 - same pattern as graph_types helpers + "MATCH (n)--(m) WHERE n.id = $id RETURN count(*) AS c", + {"id": root_node.id}, + ) + if rows: + return int(rows[0].get("c") or 0) == 0 + except Exception: # noqa: BLE001 - degrade to refine_query on graph error + log.debug("neighbors meaningful-empty probe failed", exc_info=True) + return False + + +# --------------------------------------------------------------------------- # +# Filter relaxation (ported from _zero_result_guidance, jrag.py:4187) # +# --------------------------------------------------------------------------- # + + +def _filter_relaxation( + filt: dict | None, + filter_kind: str | None, + scope: dict[str, str], + graph: Any, + identifier: str | None, +) -> FilterRelaxation: + """For each constrained scope dim, tally where identifier/all matches live. + + Ports ``_zero_result_guidance``'s tally-and-suggest-most-common logic into the + structured ``FilterRelaxation`` payload, parameterized on a filter-dims dict + + graph (NOT ``argparse.Namespace``). + """ + constrained: dict[str, str] = {} + for source in (filt or {}, scope or {}): + for dim in _RELAXABLE_DIMS: + val = source.get(dim) if isinstance(source, dict) else None + if isinstance(val, str) and val.strip() and dim not in constrained: + constrained[dim] = val.strip() + + per_dimension: list[FilterRelaxationDim] = [] + for dim, val in constrained.items(): + try: + total, suggested = _tally_dim(graph, dim, identifier) + except Exception: # noqa: BLE001 - relaxation is best-effort + log.debug("filter relaxation tally failed for dim=%s", dim, exc_info=True) + total, suggested = 0, None + per_dimension.append( + FilterRelaxationDim( + dimension=dim, + constrained_value=val, + matches_under_relaxation=total, + suggested_value=suggested, + ) + ) + return FilterRelaxation(per_dimension=per_dimension) + + +def _tally_dim( + graph: Any, dim: str, identifier: str | None, +) -> tuple[int, str | None]: + """Count symbols (optionally matching ``identifier``) grouped by ``dim``. + + Returns (total_matches, most_common_bucket). Mirrors the probe+tally+top-3 + shape of ``_zero_result_guidance`` but reads the graph directly (no mcp_v2 + import) and returns structured values instead of a human string. + """ + params: dict[str, Any] = {} + where = ["s.module IS NOT NULL"] if dim == "module" else [] + # module_counts/microservice_counts count resolved type-symbols; mirror that + # by restricting to resolved symbols for scope dims so suggestions are stable. + if dim in ("module", "microservice", "role"): + where.append("s.resolved = true") + if identifier: + where.append( + "(toLower(s.name) CONTAINS toLower($needle) " + "OR toLower(s.fqn) CONTAINS toLower($needle))" + ) + params["needle"] = identifier + where_clause = ("WHERE " + " AND ".join(where)) if where else "" + query = ( + f"MATCH (s:Symbol) {where_clause} " + f"RETURN s.{dim} AS bucket, count(*) AS n ORDER BY n DESC LIMIT 10" + ) + rows = graph._rows(query, params) # noqa: SLF001 + buckets = [ + (str(r.get("bucket") or ""), int(r.get("n") or 0)) + for r in rows + if r.get("bucket") + ] + total = sum(n for _, n in buckets) + suggested = buckets[0][0] if buckets else None + return total, suggested + + +# --------------------------------------------------------------------------- # +# Vocabulary context (nl_miss) # +# --------------------------------------------------------------------------- # + + +def _build_vocabulary_context(graph: Any, vocab: VocabularyIndex) -> VocabularyContext: + """Assemble project vocabulary stats to inform query refinement.""" + top_modules = sorted(graph.module_counts().items(), key=lambda kv: -kv[1])[:5] + top_microservices = sorted(graph.microservice_counts().items(), key=lambda kv: -kv[1])[:5] + + role_counts: Counter = Counter() + token_counts: Counter = Counter() + for rec in vocab.records: + if rec.role: + role_counts[rec.role] += 1 + for tok in _camel_tokens(rec.simple_name): + token_counts[tok] += 1 + + roles = sorted(role_counts.items(), key=lambda kv: -kv[1])[:5] + tokens = [tok for tok, _ in token_counts.most_common(10)] + return VocabularyContext( + top_modules=[(k, int(v)) for k, v in top_modules], + top_microservices=[(k, int(v)) for k, v in top_microservices], + roles_present=[(k, int(v)) for k, v in roles], + frequent_name_tokens=tokens, + ) + + +# --------------------------------------------------------------------------- # +# Small helpers # +# --------------------------------------------------------------------------- # + + +def _similarity(a: str, b: str) -> float: + """difflib SequenceMatcher ratio on normalized names (∈ [0,1]).""" + return SequenceMatcher(None, a, b).ratio() + + +def _is_identifier_shaped(query: str) -> bool: + """Predicate: does ``query`` look like an identifier (not natural language)? + + Identifier-shaped = a CamelCase token, dotted FQN, or ``Cls#member`` with no + spaces, no NL punctuation, at least one alphanumeric, and not a lone stopword. + Extends the spirit of ``_find_has_identifier_shaped_filter`` to free text. + """ + q = query.strip() + if not q or " " in q: + return False + if any(ch in q for ch in "?,;:!'\""): + return False + if not any(ch.isalnum() for ch in q): + return False + if q.lower() in _STOPWORDS: + return False + return True + + +def _looks_like_node_id(s: str) -> bool: + """True if ``s`` carries a Ladybug node-id prefix (sym:/route:/ucs:/...).""" + return any(s.startswith(pfx) for pfx in _NODE_ID_PREFIXES) + + +def _extract_identifier(filt: dict | None, filter_kind: str | None) -> str | None: + """Pull the identifier filter value (fqn_contains/path_contains/...) out of filt.""" + if not filt or not filter_kind: + return None + for fname in _IDENTIFIER_FILTER_FIELDS.get(filter_kind, ()): + val = filt.get(fname) + if isinstance(val, str) and val.strip(): + return val.strip() + return None + + +def _exact_symbol_exists(query: str, vocab: VocabularyIndex) -> bool: + """True if ``query`` exactly resolves to a real (resolved) project symbol. + + Conservative false-absent guard: compares the query verbatim AND normalized + against every record's simple_name and fqn. Only invoked on the + ``not_in_project`` path (rare), so the O(n) scan is acceptable; it makes + declaring a real symbol absent impossible. Resolved-only because a phantom + match is handled as ``external`` upstream. + """ + q = query.strip() + if not q: + return False + q_norm = _normalize_name(q) + for rec in vocab.records: + if not rec.resolved: + continue + if rec.simple_name == q or rec.fqn == q: + return True + if _normalize_name(rec.simple_name) == q_norm or _normalize_name(rec.fqn) == q_norm: + return True + return False + + +def _build_node_ref(rec: SymbolRecord) -> NodeRef: + """Build a NodeRef from a SymbolRecord (per the brief's field mapping).""" + return NodeRef( + id=rec.node_id, + kind="symbol", + fqn=rec.fqn, + name=rec.simple_name, + symbol_kind=rec.kind or None, + module=rec.module, + microservice=rec.microservice, + role=rec.role, + ) + + +def _camel_tokens(name: str) -> list[str]: + """Split a CamelCase identifier into tokens for vocabulary statistics.""" + if not name: + return [] + tokens: list[str] = [] + cur = "" + prev_lower = False + for ch in name: + if ch.isupper(): + if prev_lower and cur: + tokens.append(cur.lower()) + cur = ch + else: + cur += ch + prev_lower = False + elif ch.isalnum(): + cur += ch + prev_lower = ch.islower() + else: + if cur: + tokens.append(cur.lower()) + cur = "" + prev_lower = False + if cur: + tokens.append(cur.lower()) + return [t for t in tokens if len(t) > 1] + + +def _fallback_refine() -> AbsenceDiagnosis | None: + """Minimal refine_query when diagnosis cannot complete (never raises).""" + try: + return AbsenceDiagnosis( + verdict="refine_query", + cause="identifier_miss", + message="Unable to diagnose the empty result; refine the query and retry.", + ) + except Exception: # noqa: BLE001 + return None diff --git a/absence_types.py b/absence_types.py new file mode 100644 index 00000000..98fec95f --- /dev/null +++ b/absence_types.py @@ -0,0 +1,124 @@ +"""Absence diagnosis data transfer objects. + +These types define the contract for explaining empty MCP tool results. +Later PRs (ABS-2, ABS-3) populate these fields; ABS-0 only declares them. +""" + +from typing import Literal + +from pydantic import BaseModel, Field + +from graph_types import NodeRef + +__all__ = [ + "AbsenceVerdict", + "AbsenceCause", + "ExternalReason", + "AbsenceProof", + "ExternalIdentity", + "VocabularyContext", + "FilterRelaxationDim", + "FilterRelaxation", + "AbsenceDiagnosis", +] + +# Literal types for verdicts and causes +AbsenceVerdict = Literal["refine_query", "not_in_project", "external_dependency", "correct_empty"] +AbsenceCause = Literal["identifier_miss", "nl_miss", "filter_miss", "external", "meaningful_empty"] +ExternalReason = Literal["prefix", "phantom", "unresolved-call"] + + +class AbsenceProof(BaseModel): + """Evidence backing a hard 'not_in_project' verdict. + + Attributes: + nearest_distance: Distance to the closest symbol found (0-1) + symbol_count_scanned: Total symbols examined during search + thresholds_applied: The similarity thresholds used in the decision + query_shape: Shape of the original query (currently only "identifier") + """ + nearest_distance: float + symbol_count_scanned: int + thresholds_applied: dict[str, float] + query_shape: Literal["identifier"] + + +class ExternalIdentity(BaseModel): + """Identifies an external dependency that caused the empty result. + + Attributes: + fqn: Fully qualified name of the external symbol + reason: Why we believe this is external (prefix, phantom, unresolved call) + source: Optional source name (e.g., "maven", "gradle") + """ + fqn: str + reason: ExternalReason + source: str | None = None + + +class VocabularyContext(BaseModel): + """Project vocabulary statistics to inform query refinement. + + Attributes: + top_modules: Most frequent modules with counts + top_microservices: Most frequent microservices with counts + roles_present: Symbol roles present with counts + frequent_name_tokens: Common tokens in symbol names + """ + top_modules: list[tuple[str, int]] + top_microservices: list[tuple[str, int]] + roles_present: list[tuple[str, int]] + frequent_name_tokens: list[str] + + +class FilterRelaxationDim(BaseModel): + """Relaxation analysis for a single filter dimension. + + Attributes: + dimension: The filter dimension (e.g., "microservice", "role") + constrained_value: The value that constrained results + matches_under_relaxation: Results if this dimension were relaxed + suggested_value: Optional alternative value to try + """ + dimension: str + constrained_value: str | None + matches_under_relaxation: int + suggested_value: str | None + + +class FilterRelaxation(BaseModel): + """Analysis of how relaxing filters would affect results. + + Attributes: + per_dimension: List of relaxation options per dimension + """ + per_dimension: list[FilterRelaxationDim] + + +class AbsenceDiagnosis(BaseModel): + """Explains why an MCP tool returned no results. + + This is the main DTO that the 5 MCP output models optionally carry. + In PR-ABS-0, the `absence` field stays None everywhere — later PRs + populate it with diagnosis logic. + + Attributes: + verdict: High-level judgment on the empty result + cause: Specific cause that led to this verdict + message: Human-readable explanation + closest_symbols: Symbols closest to the query (if any) + distances: Corresponding distance values + proof: Evidence for not_in_project verdict + external_identity: External dependency info + vocabulary_context: Project vocabulary for refinement + filter_relaxation: Filter relaxation suggestions + """ + verdict: AbsenceVerdict + cause: AbsenceCause + message: str + closest_symbols: list[NodeRef] = Field(default_factory=list) + distances: list[float] = Field(default_factory=list) + proof: AbsenceProof | None = None + external_identity: ExternalIdentity | None = None + vocabulary_context: VocabularyContext | None = None + filter_relaxation: FilterRelaxation | None = None diff --git a/absence_vocab.py b/absence_vocab.py new file mode 100644 index 00000000..b76e39eb --- /dev/null +++ b/absence_vocab.py @@ -0,0 +1,433 @@ +"""Vocabulary index for absence diagnosis (PR-ABS-1). + +A VocabularyIndex builds a search-optimized projection from a LadybugGraph's +Symbol nodes, persisting as a versioned JSON sidecar. It provides bounded-time +lookup for did-you-mean candidates and external membership checks. + +Consumed by PR-ABS-2 (diagnosis ranking) and PR-ABS-3 (MCP tools). +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +log = logging.getLogger(__name__) + +__all__ = [ + "SymbolRecord", + "VocabularyIndex", + "VocabIndexStale", + "get_vocabulary_index", + "reset_cache", + "VOCAB_INDEX_FILENAME", +] + +VOCAB_INDEX_FILENAME = "vocab_index.json" + + +class VocabIndexStale(Exception): + """Raised when loading a vocab index with stale ontology_version.""" + + pass + + +@dataclass +class SymbolRecord: + """A single symbol record from the graph. + + Attributes: + node_id: Ladybug node ID + fqn: Fully qualified name + simple_name: Simple name (last segment) + normalized_name: Lowercased simple name with signatures stripped + kind: Symbol kind (class, method, field, etc.) + module: Maven module (if available) + microservice: Microservice label (if available) + role: Symbol role (Controller, Service, Repository, etc.) + resolved: Whether the symbol resolved to a source location + """ + node_id: str + fqn: str + simple_name: str + normalized_name: str + kind: str + module: str | None + microservice: str | None + role: str | None + resolved: bool + + +class VocabularyIndex: + """Search-optimized vocabulary index built from LadybugGraph Symbol nodes. + + The index stores a flat list of SymbolRecords and an n-gram inverted index + mapping q-grams to record indexes. This allows bounded-time lookup for + did-you-mean candidates without scanning the entire vocabulary. + + Built at the end of graph build; persisted as a sidecar JSON; lazily rebuilt + if missing or stale (ontology_version mismatch). + """ + + def __init__( + self, + records: list[SymbolRecord], + ngram_index: dict[str, list[int]], + q: int, + _name_index: dict[str, list[int]] | None = None, + ) -> None: + self.records = records + self.ngram_index = ngram_index + self.q = q + # Build name index for O(1) exact lookups (key: normalized_name -> record indices) + if _name_index is None: + self._name_index: dict[str, list[int]] = {} + for idx, record in enumerate(records): + norm = record.normalized_name + if norm not in self._name_index: + self._name_index[norm] = [] + self._name_index[norm].append(idx) + else: + self._name_index = _name_index + + @property + def symbol_count(self) -> int: + return len(self.records) + + @classmethod + def build(cls, graph: Any, *, q: int) -> "VocabularyIndex": + """Build a vocabulary index from a LadybugGraph. + + Enumerates all Symbol nodes, builds SymbolRecords with normalized names, + and constructs a q-gram inverted index for candidate lookup. + + Args: + graph: LadybugGraph instance + q: N-gram length (typically 3) + + Returns: + VocabularyIndex ready for queries + """ + # Query all Symbol nodes with proper column aliases + query = """ + MATCH (s:Symbol) + RETURN s.id AS id, s.kind AS kind, s.name AS name, s.fqn AS fqn, + s.package AS package, s.module AS module, s.microservice AS microservice, + s.filename AS filename, s.start_line AS start_line, s.end_line AS end_line, + s.start_byte AS start_byte, s.end_byte AS end_byte, s.modifiers AS modifiers, + s.annotations AS annotations, s.capabilities AS capabilities, s.role AS role, + s.signature AS signature, s.parent_id AS parent_id, s.resolved AS resolved + """ + rows = graph._rows(query, {}) + + records: list[SymbolRecord] = [] + for row in rows: + record = _row_to_symbol_record(row) + records.append(record) + + # Build n-gram index from normalized names + ngram_index: dict[str, list[int]] = {} + for idx, record in enumerate(records): + grams = _qgrams(record.normalized_name, q) + for gram in grams: + if gram not in ngram_index: + ngram_index[gram] = [] + ngram_index[gram].append(idx) + + return cls(records=records, ngram_index=ngram_index, q=q) + + def save(self, path: Path, *, ontology_version: int) -> None: + """Save the vocabulary index to a JSON sidecar. + + Args: + path: Destination path for the sidecar + ontology_version: Current graph ontology version (for staleness detection) + """ + import time + + data = { + "format_version": 1, + "ontology_version": ontology_version, + "built_at": int(time.time()), + "symbol_count": self.symbol_count, + "q": self.q, + "records": [ + { + "node_id": r.node_id, + "fqn": r.fqn, + "simple_name": r.simple_name, + "normalized_name": r.normalized_name, + "kind": r.kind, + "module": r.module, + "microservice": r.microservice, + "role": r.role, + "resolved": r.resolved, + } + for r in self.records + ], + "ngrams": self.ngram_index, + "name_index": self._name_index, + } + + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(data, f) + + log.debug(f"VocabularyIndex saved to {path} ({self.symbol_count} symbols)") + + @classmethod + def load(cls, path: Path) -> "VocabularyIndex": + """Load a vocabulary index from a JSON sidecar. + + Args: + path: Path to the sidecar file + + Returns: + VocabularyIndex + + Raises: + VocabIndexStale: If sidecar ontology_version doesn't match expected + """ + from ast_java import ONTOLOGY_VERSION + + with open(path) as f: + data = json.load(f) + + # Check ontology version + if data.get("ontology_version") != ONTOLOGY_VERSION: + raise VocabIndexStale( + f"Vocab index ontology version {data.get('ontology_version')} " + f"does not match expected {ONTOLOGY_VERSION}" + ) + + records = [ + SymbolRecord( + node_id=r["node_id"], + fqn=r["fqn"], + simple_name=r["simple_name"], + normalized_name=r["normalized_name"], + kind=r["kind"], + module=r.get("module"), + microservice=r.get("microservice"), + role=r.get("role"), + resolved=r["resolved"], + ) + for r in data["records"] + ] + + return cls( + records=records, + ngram_index=data["ngrams"], + q=data["q"], + _name_index=data.get("name_index"), + ) + + def lookup(self, name: str, *, limit: int) -> list[SymbolRecord]: + """Lookup candidate records by name using n-gram overlap. + + This returns candidate records ONLY; ranking by similarity is done + in PR-ABS-2 (absence_diagnosis module) to avoid circular imports. + + Args: + name: Query name (can be typoed) + limit: Maximum number of candidates to return + + Returns: + List of candidate SymbolRecord (up to limit), ordered by n-gram overlap count + """ + # First, check for exact match on simple_name using O(1) dict lookup (fast path) + # Return ALL matching records since overloaded names matter + normalized = _normalize_name(name) + if normalized in self._name_index: + exact_matches = [self.records[idx] for idx in self._name_index[normalized]] + # Filter to only those where simple_name matches exactly (case-sensitive) + exact_simple_matches = [r for r in exact_matches if r.simple_name == name] + if exact_simple_matches: + log.debug(f"lookup({name}): exact match found") + return exact_simple_matches + + # No exact match, use n-gram overlap + # Extract q-grams from query + grams = _qgrams(normalized, self.q) + + # Count n-gram matches per record index + match_counts: dict[int, int] = {} + for gram in grams: + if gram in self.ngram_index: + for idx in self.ngram_index[gram]: + match_counts[idx] = match_counts.get(idx, 0) + 1 + + # Sort by match count (descending) to get candidates with most overlap first + sorted_idxs = sorted(match_counts.keys(), key=lambda idx: match_counts[idx], reverse=True) + + # Debug logging + log.debug(f"lookup({name}): normalized={normalized}, grams={grams[:5]}, candidates={len(sorted_idxs)}") + + # Return top candidates + candidates = [self.records[idx] for idx in sorted_idxs[:limit]] + return candidates + + def is_external(self, name: str) -> tuple[bool, str | None]: + """Check if a name refers to an external symbol. + + Returns (is_external, reason) where reason is one of: + - "prefix": FQN matches an external library prefix (java.*, javax.*, etc.) + - "phantom": Symbol exists in graph but is unresolved (phantom) + - None: Symbol is a real project symbol + + Args: + name: Simple name or FQN to check + + Returns: + (is_external, reason) tuple + """ + from ladybug_queries import _is_external_fqn, _EXTERNAL_PREFIXES + + # First, check if it's an external prefix (highest priority) + if _is_external_fqn(name): + return (True, "prefix") + + # Also check simple name against external prefixes + for prefix in _EXTERNAL_PREFIXES: + if name.startswith(prefix): + return (True, "prefix") + + # Check if name matches any record in our vocabulary using O(1) dict lookup + normalized = _normalize_name(name) + matching_indices = self._name_index.get(normalized, []) + matching_record = None + for idx in matching_indices: + rec = self.records[idx] + if rec.simple_name == name or rec.fqn == name: + matching_record = rec + break + + if matching_record: + # If the symbol is unresolved, it's a phantom + if not matching_record.resolved: + return (True, "phantom") + # Otherwise it's a real project symbol + return (False, None) + + # Not found and doesn't look external + return (False, None) + + +# Module-level cache for get_vocabulary_index +_vocab_cache: dict[str, VocabularyIndex] = {} + + +def get_vocabulary_index(graph: Any, cfg: Any) -> VocabularyIndex: + """Get or build a vocabulary index for the given graph. + + This is the primary entry point for the diagnosis layer (PR-ABS-2) and + tools (PR-ABS-3). It implements lazy backfill: tries to load from sidecar, + builds from graph on miss/stale, and caches the result. + + Args: + graph: LadybugGraph instance + cfg: ResolvedOperatorConfig instance + + Returns: + VocabularyIndex (cached or newly built) + """ + from ast_java import ONTOLOGY_VERSION + + # Determine graph db path for cache key + db_path = graph.db_path if hasattr(graph, 'db_path') else str(cfg.ladybug_path) + sidecar_path = Path(db_path).parent / VOCAB_INDEX_FILENAME + + # Check cache + if db_path in _vocab_cache: + return _vocab_cache[db_path] + + # Try loading from sidecar + try: + index = VocabularyIndex.load(sidecar_path) + _vocab_cache[db_path] = index + log.debug(f"Loaded vocabulary index from {sidecar_path}") + return index + except (VocabIndexStale, FileNotFoundError, Exception) as e: + # Stale, missing, or corrupt - rebuild from graph + log.debug(f"Vocab index missing/stale ({e}), rebuilding from graph") + + # Build from graph + index = VocabularyIndex.build(graph, q=cfg.absence_ngram_q) + + # Save to sidecar (best-effort) + try: + index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION) + except Exception as save_err: + log.warning(f"Failed to save vocab index to {sidecar_path}: {save_err}") + + # Cache and return + _vocab_cache[db_path] = index + return index + + +def reset_cache() -> None: + """Reset the module-level vocabulary index cache. + + Exposed for tests that need to simulate a fresh start or different graph paths. + """ + global _vocab_cache + _vocab_cache = {} + + +# ---- Helper functions ---- + + +def _row_to_symbol_record(row: dict[str, Any]) -> SymbolRecord: + """Convert a Ladybug row to a SymbolRecord.""" + from ladybug_queries import _type_part_fqn + + fqn = row.get("fqn") or "" + name = row.get("name") or "" + + return SymbolRecord( + node_id=row.get("id") or "", + fqn=fqn, + simple_name=name, + normalized_name=_normalize_name(name), + kind=row.get("kind") or "", + module=row.get("module"), + microservice=row.get("microservice"), + role=row.get("role"), + resolved=bool(row.get("resolved", True)), + ) + + +def _normalize_name(name: str) -> str: + """Normalize a symbol name for n-gram indexing. + + Strips: + - Generic signatures (e.g., List → List) + - Method signatures (e.g., method(Param) → method) + - Parentheses and angle brackets + + Returns lowercase result. + """ + # Remove generic signatures + normalized = name.split("<")[0] + # Remove method signatures + normalized = normalized.split("(")[0] + # Remove hash suffix (e.g., method#signature) + normalized = normalized.split("#")[0] + return normalized.lower() + + +def _qgrams(text: str, q: int) -> list[str]: + """Extract q-grams from text. + + Args: + text: Input string + q: Gram length + + Returns: + List of q-grams (substrings of length q) + """ + if len(text) < q: + return [text] if text else [] + return [text[i:i + q] for i in range(len(text) - q + 1)] diff --git a/build_ast_graph.py b/build_ast_graph.py index 4576f659..9dd22227 100644 --- a/build_ast_graph.py +++ b/build_ast_graph.py @@ -4208,9 +4208,57 @@ def write_ladybug( _write_meta(conn, tables, source_root) conn.close() db.close() + + # Build vocabulary index (best-effort, failure doesn't fail the graph build) + _try_build_vocabulary_index(db_path, source_root, verbose) _init_hash_tracker(source_root, db_path) +def _try_build_vocabulary_index(db_path: Path, source_root: Path, verbose: bool) -> None: + """Build and save the vocabulary index as a sidecar (best-effort). + + This is called after write_ladybug() completes. A build failure must not + fail the graph build, so this is wrapped in try/except and logged. + + Args: + db_path: Path to the LadybugDB database file + source_root: Source repository root + verbose: Whether to emit verbose progress + """ + try: + from absence_vocab import VocabularyIndex, VOCAB_INDEX_FILENAME + from ladybug_queries import LadybugGraph + + t0 = time.time() + if verbose: + _verbose_stderr_line("[vocab] building vocabulary index") + + # Open graph for reading + graph = LadybugGraph.get(str(db_path)) + + # Read q from env var set by ResolvedOperatorConfig.subprocess_env() + raw_q = os.environ.get("JAVA_CODEBASE_RAG_ABSENCE_NGRAM_Q", "3").strip() + try: + q = int(raw_q) if raw_q else 3 + except ValueError: + q = 3 # Invalid env value falls back to default + # Build index with configured q (or default 3) + index = VocabularyIndex.build(graph, q=q) + + # Save to sidecar next to the graph db + sidecar_path = Path(db_path).parent / VOCAB_INDEX_FILENAME + index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION) + + if verbose: + _verbose_stderr_line(f"[vocab] index built with {index.symbol_count} symbols in {time.time() - t0:.2f}s") + + except Exception as e: + # Log but don't fail - graph build is the primary concern + log.warning(f"Vocabulary index build failed (non-critical): {e}") + if verbose: + _verbose_stderr_line(f"[vocab] build failed (graph still written): {e}") + + # ---------- CLI ---------- diff --git a/docs/AGENT-GUIDE.md b/docs/AGENT-GUIDE.md index 266f12c1..4bebfc7e 100644 --- a/docs/AGENT-GUIDE.md +++ b/docs/AGENT-GUIDE.md @@ -26,7 +26,7 @@ Copy the block between `