feat: classify, tag, and filter generated Java sources

ast_java.py

@@ -84,7 +84,7 @@
 # Phase 11: `EDGE_SCHEMA` in `java_ontology.py` (canonical edge navigation schema; v14 re-index).
 # Phase 12: CALLS `callee_declaring_role`, supertype-walk dedup, pass3 unresolved counters (v15 re-index).
 # Bumps whenever extraction / enrichment semantics change.
-ONTOLOGY_VERSION = 17
+ONTOLOGY_VERSION = 18
 
 ROLE_ANNOTATIONS: dict[str, str] = {
     # Spring Web

build_ast_graph.py

@@ -54,8 +54,10 @@
 )
 from graph_enrich import (
     _load_config_cross_service_resolution,
+    classify_java_file,
     collect_annotation_meta_chain,
     load_brownfield_overrides,
+    load_generated_detection,
     microservice_for_path,
     module_for_path,
     phantom_id,
@@ -465,6 +467,8 @@ class GraphTables:
     cross_service_resolution: str = "auto"
     # Populated in _write_nodes (same overrides + meta_chain as Symbol.role).
     type_role_by_node_id: dict[str, str] = field(default_factory=dict)
+    # Populated in pass 1 (classify_java_file) and _load_existing_types for incremental rebuilds.
+    type_generated_by_node_id: dict[str, tuple[bool, str | None]] = field(default_factory=dict)
 
 
 @dataclass
@@ -598,7 +602,7 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
     query = f"""
     MATCH (s:Symbol)
     {where}
-    RETURN s.kind, s.fqn, s.name, s.filename, s.module, s.microservice, s.id, s.role
+    RETURN s.kind, s.fqn, s.name, s.filename, s.module, s.microservice, s.id, s.role, s.generated, s.generated_by
     """
     result = conn.execute(query, params)
     while result.has_next():
@@ -608,6 +612,8 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
         microservice = row[5] if len(row) > 5 else ""
         node_id = row[6] if len(row) > 6 else ""
         role = row[7] if len(row) > 7 else ""
+        generated = row[8] if len(row) > 8 else False
+        generated_by = row[9] if len(row) > 9 else ""
 
         decl = TypeDecl(name, kind, fqn)
         package = fqn[: -(len(name) + 1)] if fqn.endswith("." + name) else ""
@@ -629,6 +635,8 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
         # the default during node staging (issue #352 divergence #2).
         if role:
             tables.type_role_by_node_id[node_id] = role
+        # Seed the persisted generated/generated_by so stubs retain their values
+        tables.type_generated_by_node_id[node_id] = (bool(generated), generated_by)
 
 
 def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
@@ -1061,6 +1069,12 @@ def pass1_parse(
             microservice = microservice_for_path(str(p), root)
             asts[rel] = ast
 
+            # Classify the file once (generated or not, and which tool generated it)
+            generated_config = load_generated_detection(str(root))
+            file_generated, file_generated_by = classify_java_file(
+                content, ast, config=generated_config, project_root=root
+            )
+
             # file node
             file_id = symbol_id("file", rel, rel, 0)
             tables.files[rel] = file_id
@@ -1075,6 +1089,12 @@ def pass1_parse(
                     module=module, microservice=microservice, outer_fqn=None,
                 )
 
+            # Seed generated/generated_by for all types in this file (including nested)
+            for t in ast.all_types:
+                if t.fqn in tables.types:
+                    node_id = tables.types[t.fqn].node_id
+                    tables.type_generated_by_node_id[node_id] = (file_generated, file_generated_by)
+
     if verbose:
         elapsed = time.time() - t0
         _emit_graph_progress(
@@ -2906,7 +2926,8 @@ def _micro_factor(member: MemberEntry | None) -> float:
     "filename STRING, start_line INT64, end_line INT64, "
     "start_byte INT64, end_byte INT64, "
     "modifiers STRING[], annotations STRING[], capabilities STRING[], "
-    "role STRING, signature STRING, parent_id STRING, resolved BOOLEAN"
+    "role STRING, signature STRING, parent_id STRING, resolved BOOLEAN, "
+    "generated BOOLEAN, generated_by STRING"
     ")"
 )
 
@@ -3088,6 +3109,7 @@ def _node_row(**kwargs) -> dict:
         "start_byte": 0, "end_byte": 0,
         "modifiers": [], "annotations": [], "capabilities": [],
         "role": "OTHER", "signature": "", "parent_id": "", "resolved": True,
+        "generated": False, "generated_by": None,
     }
     base.update(kwargs)
     return base
@@ -3138,7 +3160,8 @@ def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
 _NODE_COLUMNS = [
     "id", "kind", "name", "fqn", "package", "module", "microservice",
     "filename", "start_line", "end_line", "start_byte", "end_byte",
-    "modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved"
+    "modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved",
+    "generated", "generated_by"
 ]
 
 # Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
@@ -3161,7 +3184,8 @@ def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
     "n.start_byte = $start_byte, n.end_byte = $end_byte, "
     "n.modifiers = $modifiers, n.annotations = $annotations, "
     "n.capabilities = $capabilities, n.role = $role, "
-    "n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved"
+    "n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved, "
+    "n.generated = $generated, n.generated_by = $generated_by"
 )
 
 # Refresh every mutable Route field on an existing Route node by id. Mirrors the
@@ -3240,6 +3264,9 @@ def _write_nodes_impl(
             overrides=overrides,
             meta_chain=mch,
         )
+        # Read generated/generated_by from pass-1 classification or stub persistence
+        generated, generated_by = tables.type_generated_by_node_id.get(entry.node_id, (False, None))
+
         if entry.loaded_from_db:
             stub_ids.add(entry.node_id)
             # Out-of-scope stub: its annotation-less decl collapses role to the
@@ -3250,6 +3277,7 @@ def _write_nodes_impl(
             # capabilities placeholder never reaches the graph.
             role = tables.type_role_by_node_id.get(entry.node_id, role)
             capabilities = []
+            # For stubs, trust the persisted generated/generated_by (seeded by _load_existing_types)
         else:
             tables.type_role_by_node_id[entry.node_id] = role
         rows.append(_node_row(
@@ -3265,6 +3293,8 @@ def _write_nodes_impl(
             role=role,
             signature="",
             parent_id=tables.types[entry.outer_fqn].node_id if entry.outer_fqn and entry.outer_fqn in tables.types else "",
+            generated=generated,
+            generated_by=generated_by,
         ))
     # members (methods / constructors)
     for m in tables.members:
@@ -3774,7 +3804,7 @@ def incremental_rebuild(
     Returns IncrementalResult with statistics about the rebuild.
     Falls back to full rebuild if:
     - No previous graph exists
-    - Ontology version < 17 (missing source_file on edges)
+    - Ontology version < ONTOLOGY_VERSION (stale schema; rebuild for current columns)
     - Crash marker exists (previous incremental run failed)
     - Dependent expansion exceeds expansion_cap
     """
@@ -3812,9 +3842,9 @@ def incremental_rebuild(
         if meta_result.has_next():
             row = meta_result.get_next()
             version = row[0] if row else 0
-            if version < 17:
+            if version < ONTOLOGY_VERSION:
                 if verbose:
-                    _verbose_stderr_line(f"[increment] ontology version {version} < 17; falling back to full rebuild")
+                    _verbose_stderr_line(f"[increment] ontology version {version} < {ONTOLOGY_VERSION}; falling back to full rebuild")
                 conn.close()
                 db.close()
                 del conn, db

docs/AGENT-GUIDE.md

@@ -14,7 +14,7 @@ Copy the block between `<!-- BEGIN` and `<!-- END` into your project's `AGENTS.m
 
 **Indexed content:** Java production sources plus SQL and YAML (use `search` `table`: `java`, `sql`, `yaml`, or `all`).
 
-**Ontology: 17** — if results look structurally wrong or empty across tools, the index may be missing, stale, or built with a different `ontology_version`; you cannot re-index via MCP — ask the operator to rebuild.
+**Ontology: 18** — if results look structurally wrong or empty across tools, the index may be missing, stale, or built with a different `ontology_version`; you cannot re-index via MCP — ask the operator to rebuild.
 
 **Responses:** On success, `search`, `find`, `describe`, `neighbors`, and `resolve` may include two top-level fields: `hints_structured` (≤5 suggested next-tool calls) and `advisories` (≤5 pure informational strings). Each `hints_structured` entry has `tool`, `args`, `actionable`, `label`, and `reason`. `actionable=true` means you can call the tool directly with `args`; `actionable=false` means partial/advisory — fill missing values or use as guidance. `reason` explains why the hint was emitted. `advisories` carry context education (fuzzy strategy warnings, role collision explanations, etc.) with no tool call suggestion. For `search`/`find`, echoed `limit`/`offset`. Hints are advisory; ignore them when `success` is false.
 
@@ -136,7 +136,7 @@ For **`find`**, `filter` is required — `{}` means no predicates (all nodes of
 | Keys | Applies to |
 | ---- | ---------- |
 | `microservice`, `module` | All kinds |
-| `role`, `exclude_roles`, `annotation`, `capability`, `fqn_contains`, `symbol_kind`, `symbol_kinds` | **symbol** |
+| `role`, `exclude_roles`, `annotation`, `capability`, `fqn_contains`, `symbol_kind`, `symbol_kinds`, `generated_only`, `exclude_generated` | **symbol** |
 | `http_method`, `path_contains`, `framework` | **route** |
 | `source_layer`, `client_kind`, `target_service`, `target_path_contains`, `http_method` | **client** |
 | `source_layer`, `producer_kind`, `topic_contains` | **producer** |
@@ -145,6 +145,21 @@ For **`find`**, `filter` is required — `{}` means no predicates (all nodes of
 
 **Strict frame:** one populated field → one stored attribute for that kind. Unknown keys or inapplicable populated fields → `success=false` with a teaching `message`. Invalid enum values (e.g. wrong case) are rejected earlier at the schema layer with the valid set listed. The substring fields (`fqn_contains`, `path_contains`, `target_path_contains`, `topic_contains`) match literally via `CONTAINS` — no `*`/`?` metacharacters; use `search(query=…)` for ranked text instead. `search.query` is opaque text, not a DSL.
 
+### Generated source detection
+
+**Generated sources** (MapStruct mappers, OpenAPI clients, protobuf stubs, etc.) are **auto-detected by content** (not by path). Every MCP search/find/describe/neighbors result row carries two fields:
+
+- `generated` (bool) — `true` if the source file is generated.
+- `generated_by` (string | null) — the generator family slug (`openapi`, `jsonschema2pojo`, `protobuf`, `mapstruct`, `wsimport`, `querydsl`, `jooq`, `immutables`, `autovalue`, `lombok`), or `null` for unrecognized generators.
+
+**Detection criteria:** A Java source file is classified as generated when it carries a `@Generated` annotation (javax/jakarta.annotation.processing.Generated and equivalents: lombok.Generated, org.immutables.value.Generated, com.squareup.javapoet.Generated) OR a recognized generator header banner (OpenAPI, jsonschema2pojo, protobuf, MapStruct, wsimport).
+
+**Note:** The detector matches `@Generated` by simple annotation name. If your project defines its own unrelated `@Generated` annotation (e.g., `@com.example.Generated`), it will be flagged as generated code. To exclude a specific FQN from detection, add it to `generated_detection.exclude_fqns` in your `.java-codebase-rag.yml` configuration.
+
+**Equal-treatment default:** Generated sources are indexed and returned **exactly** like hand-written code by default — they are **not** ranked down and **not** excluded from graph traversal. The existing role-aware ranking already down-ranks non-actionable roles (DTOs/mappers), which covers most generated code.
+
+**Filtering:** Use `filter={"exclude_generated": true}` on the MCP `NodeFilter` to exclude generated sources when you only want hand-written code. Use `filter={"generated_only": true}` to show only generated sources. On the CLI, use `--exclude-generated` or `--generated-only` flags.
+
 ### Identifier resolution (`resolve`)
 
 **Input:** FQN or suffix, `sym:`/`route:`/`client:`/`producer:` id, `METHOD /path`, route path template, client `target_service`, `target_service` + path prefix, or producer topic.

docs/CODEBASE_REQUIREMENTS.md

@@ -60,10 +60,12 @@ inside the MCP.
   - See: `graph_enrich.py::module_for_path`,
     `graph_enrich.py::microservice_for_path`, constant `BUILD_MARKERS`.
 - **Build outputs out of the way.** `target/`, `build/`, `node_modules/`,
-  `.idea/`, `.venv/` are pruned during the graph walk. Don't keep
-  generated `.java` (e.g., MapStruct, Lombok delombok output, OpenAPI
-  generated clients) in committed source trees — they balloon the graph
-  with phantom edges.
+  `.idea/`, `.venv/` are pruned during the graph walk. Generated sources
+  (e.g., MapStruct mappers, OpenAPI clients, protobuf stubs) in committed
+  source trees are now **auto-detected and tagged** as `generated=true`
+  with a `generated_by` field — they are indexed and surfaced by default.
+  Use `exclude_generated` on the MCP `NodeFilter` (or CLI `--exclude-generated`)
+  to filter them out when you only want hand-written code.
 
 ### A.2 Call graph (static `CALLS` / `DECLARES`)
 
@@ -441,8 +443,8 @@ You'd do this if:
   group symbols correctly;
 - you use a build system the MCP doesn't recognise (`package.json`,
   `Cargo.toml`, `BUILD.bazel`, custom marker file);
-- you want to exclude additional directories (generated code, vendored
-  forks).
+- you want to exclude additional directories (vendored forks,
+  legacy modules).
 
 **No-code option (recommended first):** drop a `.java-codebase-rag.yml` at
 the project root listing the directory names that should be treated as
@@ -456,14 +458,19 @@ microservice_roots:
   - notifications
 ```
 
+**Generated code:** Java sources generated by tools (MapStruct, OpenAPI,
+protobuf, etc.) are **auto-detected by content** (not by path) and tagged
+with `generated=true` and `generated_by=<family>`. Use the `exclude_generated`
+filter on the MCP `NodeFilter` or `--exclude-generated` CLI flag to filter
+them out when you only want hand-written code. See `generated_detection`
+in `.java-codebase-rag.yml` to customize detection patterns.
+
 **Code-level changes:**
 
 - `graph_enrich.py::BUILD_MARKERS` — add new marker filenames so both
   `module_for_path` and `microservice_for_path` discover them.
 - `graph_enrich.py::microservice_for_path` — adjust the fallback rules
   (e.g. promote `services/<name>/...` segments).
-- `java_index_v1_common.py::COMMON_EXCLUDED_PATH_PATTERNS` — append
-  globs like `**/generated/**`, `**/openapi/**`, `**/legacy/**`.
 - `java_index_v1_common.iter_java_source_files` / `build_ast_graph.py` — extra hard-coded directory
   names to prune (`target`, `build`, `node_modules`, ...).
 

docs/CONFIGURATION.md

@@ -262,6 +262,39 @@ async_producer_overrides:
     com.legacy.EventBus:
       client_kind: kafka_send
       topic: chat.follow-up
+
+# -------- Generated source detection --------
+
+# Customize how generated Java sources are detected (content-based, not path-based).
+# By default, the indexer recognizes @Generated annotations and common generator
+# header banners (OpenAPI, jsonschema2pojo, protobuf, MapStruct, wsimport, etc.).
+generated_detection:
+  # Additional regex patterns for generator header banners (matched case-insensitive).
+  # Each pattern is applied to the full file content; use .* for wildcard.
+  # Example: '^// Generated by acme-codegen v' for a custom tool.
+  header_patterns:
+    - '^// Generated by acme-codegen v'
+
+  # Additional annotation fully-qualified names that signal generated code.
+  # Annotation simple names are also recognized (e.g., '@Generated' matches
+  # javax.annotation.processing.Generated, jakarta.annotation.processing.Generated).
+  # Use FQNs to avoid collisions with non-generated annotations of the same name.
+  # Note: Simple-name matching means your project's own @com.example.Generated
+  # would be flagged as generated. To exclude a specific FQN, add it to exclude_fqns below.
+  annotation_patterns:
+    - 'com.example.custom.Generated'
+
+  # Force specific types (by FQN) to be treated as generated, even if no
+  # @Generated annotation or header banner is present. Useful for types
+  # from known generators that don't emit standard markers.
+  force_fqns:
+    - 'com.example.autogen.AutoModel'
+
+  # Exclude specific types (by FQN) from generated detection, even if they
+  # carry @Generated or match a header pattern. Use sparingly — prefer to
+  # fix the generator or filter with exclude_generated at query time.
+  exclude_fqns:
+    - 'com.example.manual.ExampleImpl'
 ```
 
 **Path expansion (what gets `~` / `$VAR` treatment):**
@@ -289,7 +322,7 @@ Deeper documentation for the brownfield blocks (`role_overrides`, `route_overrid
 
 ## 3. Graph layer
 
-A deterministic property graph derived from tree-sitter Java parsing lives next to the LanceDB tables under the index directory (default `${JAVA_CODEBASE_RAG_INDEX_DIR:-./.java-codebase-rag}/code_graph.lbug`). Current ontology version: **17** (see [`EDGE-NAVIGATION.md`](./EDGE-NAVIGATION.md) for MCP-traversable edge shapes).
+A deterministic property graph derived from tree-sitter Java parsing lives next to the LanceDB tables under the index directory (default `${JAVA_CODEBASE_RAG_INDEX_DIR:-./.java-codebase-rag}/code_graph.lbug`). Current ontology version: **18** (see [`EDGE-NAVIGATION.md`](./EDGE-NAVIGATION.md) for MCP-traversable edge shapes).
 
 ### Node kinds
 
@@ -353,7 +386,7 @@ Resolution order for `microservice`:
 
 ### Re-index required when ontology changes
 
-Current ontology version is **17**. Any index built before this version must be rebuilt via `cocoindex update ... --full-reprocess -f` or a full `java-codebase-rag reprocess` (no selective flags) so vectors and graph stay aligned. Until re-indexed, the server defensively JSON-decodes string-form list columns so nothing explodes, but filters like `array_contains` will not work.
+Current ontology version is **18**. Any index built before this version must be rebuilt via `cocoindex update ... --full-reprocess -f` or a full `java-codebase-rag reprocess` (no selective flags) so vectors and graph stay aligned. Until re-indexed, the server defensively JSON-decodes string-form list columns so nothing explodes, but filters like `array_contains` will not work.
 
 Ontology **15** (CALLS-NOISE) adds `CALLS.callee_declaring_role`, `GraphMeta.pass3_unresolved_phantom_receiver` / `pass3_unresolved_chained`, and **supertype-walk dedup** at build time. PR-2 adds `edge_filter` on `neighbors`. **PR-3 (breaking):** receiver-failure sites (`chained_receiver`, unresolved-receiver `phantom`) are no longer `CALLS` rows — they live on `UnresolvedCallSite` + `UNRESOLVED_AT`. Default `neighbors(..., ['CALLS'])` returns fewer rows; use `include_unresolved=True` for a source-ordered interleaved transcript (`row_kind`), `describe(method_id).unresolved_call_sites` (capped), or `java-codebase-rag unresolved-calls list|stats`. Known-receiver-external JDK rows stay on `CALLS` with `resolved=false`.