feat: native ast generalization

docs/wasm-plugins.md

@@ -39,73 +39,86 @@ The module is instantiated once per scan and its `parse*` functions are called m
 times (it's a long-lived "reactor", not a per-file process). Compile a library, not
 a command.
 
-> **Scope:** `codesight` invokes a plugin only at its own extraction points and only
-> for a language identifier it recognizes. It provides no WASM-based parsers itself.
-> Where no plugin is configured, it falls back to its built-in extraction. This contract
-> defines the boundary; it does not promise any particular language is wired up.
+> **Scope:** `codesight` ships no parsers itself. Dispatch is **language-driven**: a
+> plugin declares the file extensions it handles (via `describe()`, below), and
+> `codesight` routes matching files to it — so *any* language works, not just the ones
+> with built-in extractors. Where no plugin handles a file, built-in extraction stands.
 
 ---
 
 ## Discovery & naming
 
-A plugin is identified by a language identifier `<lang>`. Per identifier,
-`codesight` looks for a single self-describing module — no sidecar manifest:
+Plugin binaries must match the template (anything else on the path is ignored):
 
 ```
-codesight-<lang>-ast.wasm            # the module (required)
+codesight-<lang>-ast.wasm            # `<lang>` ∈ [a-z0-9_-]+ ; the `-ast` capability namespace is reserved
 ```
 
-The `-ast` segment is the capability namespace (reserved to allow the same plugin
-mechanism to host other capabilities later if/when it becomes beneficial or desirable
-to do so). The module declares its own contract version and capabilities through
-its exports (see below), so there is nothing else to ship or keep in sync.
+The module is **fully self-describing** — no sidecar manifest. Its `describe()`
+export (below) declares the authoritative `languageId` and the file `extensions`
+it parses; the `<lang>` in the filename is only a discovery key + fallback id (if
+`describe().languageId` is set and differs, the declared id wins, with a warning).
+For the built-in language ids (`rust`/`go`/`python`) a default extension map
+(`.rs`/`.go`/`.py`) applies when `describe()` is omitted; **any other language must
+declare `extensions`** or it can't be routed.
 
-Directories are searched in this order; the **first** directory containing a
-matching `.wasm` binary wins (\*nix `PATH`-style waterfall):
+Directories are searched in PATH-style waterfall order:
 
 1. `--plugin-dir <dir>` / `CODESIGHT_PLUGIN_DIR` (relative paths resolve against the project root)
 2. `~/.codesight/plugins`
 3. `${XDG_DATA_HOME:-~/.local/share}/codesight/plugins`
 4. `<codesight install dir>/plugins`
 
+**Precedence:** per language id, the first dir wins (lower dirs shadowed). If two
+*different* languages claim the same extension, an explicitly-named language beats
+an `all`-discovered one, else first-registered wins — with a warning either way.
+
 ---
 
 ## Enabling native parsing
 
 Native parsing is off unless explicitly enabled. Precedence is `CLI` > `env` > `config file`.
 
-| Mechanism | Example                                                                                                                         |
-|-----------|---------------------------------------------------------------------------------------------------------------------------------|
-| CLI       | `codesight --native-ast` · `codesight --native-ast <langs>` · `codesight --native-ast-strict` · `codesight --plugin-dir ./wasm` |
-| Env       | `CODESIGHT_NATIVE_AST=1` (or `strict`, or a comma list of language identifiers) · `CODESIGHT_PLUGIN_DIR=/path`                  |
-| Config    | `codesight.config.{json,js,ts}` → `{ "nativeAst": { "enabled": true, "languages": ["<lang>"], "pluginDir": "./wasm" } }`        |
-
-`enabled` may be `true`, `false`, or `"strict"`. Omitting `languages` (or passing
-an empty list) enables every language identifier `codesight` recognizes; supplying a
-list restricts native parsing to those identifiers.
+| Mechanism | Example                                                                                                                            |
+|-----------|------------------------------------------------------------------------------------------------------------------------------------|
+| CLI       | `codesight --native-ast` (= all) · `codesight --native-ast=rust,go` · `codesight --native-ast=none` · `codesight --native-ast-strict` · `codesight --plugin-dir ./wasm` |
+| Env       | `CODESIGHT_NATIVE_AST=all` (or `1`/`true`/`strict`/`none`, or a comma list of ids) · `CODESIGHT_PLUGIN_DIR=/path`                  |
+| Config    | `codesight.config.{json,js,ts}` → `{ "nativeAst": { "enabled": true, "languages": ["rust"], "pluginDir": "./wasm" } }`             |
+
+`enabled` may be `true`, `false`, or `"strict"`. `none` forces off (overriding the
+config file). **Dispatch mode depends on how you enable it:**
+
+- **`all` / bare flag / empty `languages`** → *additive*: every discovered plugin is
+  consulted, and results are **unioned** with the built-in extractors (native wins on
+  a `method:path` / model-name conflict).
+- **An explicit language list** (`--native-ast=rust,go`) → *authoritative*: for files
+  a named plugin handles, its results **replace** the built-in routes from that file
+  (dropping regex over-matches). If the plugin returns nothing for a file the built-in
+  did extract, the built-in stands and a warning is emitted. (Authoritative replacement
+  is route-only — `SchemaModel` carries no file provenance, so schemas are always
+  native-preferred dedup-by-name.)
 
 ---
 
 ## The WASM ABI
 
 The host instantiates every module with a **minimal WASI import object** (Node's
 built-in `node:wasi` — clock/random/exit/stderr only; **no filesystem, no network,
-no env/args**). Pure-compute modules with no imports (Rust/AssemblyScript on
-`wasm32-unknown-unknown`) ignore it; modules whose runtime needs WASI (e.g. Go
-`//go:wasmexport` reactors) get exactly those minimal capabilities. A module that
-exports `_initialize` is initialized before any other export is called. The module
-must not require JS-binding glue or host functions beyond WASI. `node:wasi` is built
-in, so codesight keeps its zero-dependency guarantee.
+no env/args**). Pure-compute modules (Rust/AssemblyScript, no imports) ignore it;
+modules whose runtime needs WASI (e.g. Go `//go:wasmexport` reactors) get exactly
+those minimal capabilities. A reactor that exports `_initialize` is initialized
+before its other exports are called. "Plugins are pure compute" — if a kind ever
+needs project context, it flows through the ABI, not the filesystem.
 
-A conforming module exports a small fixed core plus one optional `parse*` function
-per capability it provides:
+A conforming module exports a small fixed core plus optional capability functions:
 
 | Export            | Signature (wasm types)                | Purpose                                         |
 |-------------------|---------------------------------------|-------------------------------------------------|
 | `memory`          | linear memory                         | the module's exported memory                    |
 | `alloc`           | `(len: i32) -> i32`                   | reserve `len` bytes, return a pointer           |
 | `dealloc`         | `(ptr: i32, len: i32) -> ()`          | release a prior allocation                      |
 | `contractVersion` | `() -> i32`                           | the contract version this plugin implements     |
+| `describe`        | `() -> i64`                           | *optional* — packed JSON metadata (see below)   |
 | `parseRoutes`     | `(srcPtr: i32, srcLen: i32) -> i64`   | *optional* — extract routes                     |
 | `parseSchemas`    | `(srcPtr: i32, srcLen: i32) -> i64`   | *optional* — extract schema models              |
 | `parseImports`    | `(srcPtr: i32, srcLen: i32) -> i64`   | *optional* — extract imports (defined, not yet dispatched — see below) |
@@ -116,6 +129,20 @@ also makes `contractVersion` a "this is a codesight plugin" marker. **Capability
 detected by export presence:** a plugin supports a kind iff it exports the matching
 `parse*` function. There are no kind codes and no manifest.
 
+### `describe()` — self-description
+
+Optional. Returns packed `(outPtr << 32) | outLen` pointing at UTF-8 JSON (same
+convention as `parse*`), e.g.:
+
+```jsonc
+{ "languageId": "ruby", "extensions": [".rb", ".rake"] }
+```
+
+The host reads `languageId` (authoritative over the filename) and `extensions` (how
+files are routed to this plugin); other fields are carried but unused for now. A
+plugin for a non-built-in language **must** declare `extensions` here, or it has no
+files to receive.
+
 As WebAssembly text:
 
 ```wat
@@ -286,10 +313,10 @@ from built-in `ast`/`regex` results in the scan summary.
 Implement the module in any toolchain that targets `wasm32` and imports **at most
 `wasi_snapshot_preview1`** (no JS-binding glue, no other host functions). Pure-compute
 languages (Rust/AssemblyScript via `wasm32-unknown-unknown`) need no imports at all;
-full-runtime languages (Go via `GOOS=wasip1 -buildmode=c-shared`) import WASI, which
-the host supplies minimally. The allocator and the per-kind marshalling are
-boilerplate; the only part that changes is your extraction logic. Export only the
-`parse*` functions for the kinds you support.
+full-runtime languages (Go via `GOOS=wasip1 -buildmode=c-shared` + `//go:wasmexport`)
+import WASI, which the host supplies minimally. The allocator and per-kind marshalling
+are boilerplate; only your extraction logic changes. Export `describe()` (for routing)
+plus the `parse*` functions for the kinds you support.
 
 Required exports and their behavior, in pseudocode:
 

reference/ast-plugin/assembly/index.ts

@@ -26,6 +26,11 @@ export function contractVersion(): i32 {
   return 1;
 }
 
+/** Optional self-description: language id + the extensions this plugin parses. */
+export function describe(): i64 {
+  return emit("{\"languageId\":\"reference\",\"extensions\":[\".ref\"]}");
+}
+
 // ─── ABI: memory management ───
 
 export function alloc(len: i32): i32 {

reference/ast-plugin/checksums.json

@@ -1,7 +1,7 @@
 {
   "algorithm": "sha256",
   "files": {
-    "assembly/index.ts": "4d2f5917df6f2ca85c33fa59b390a4a87331812a0a2522528aa9c3a35d7b2229",
-    "codesight-reference-ast.wasm": "31c49dfa343a8fec0578c062a7f5bac013260de3e1852551a020ebc278b3b843"
+    "assembly/index.ts": "7dbd49e902cbc8c51a7aa19010a4893f1aab7c0ddba32ad5219cbfe988545ee3",
+    "codesight-reference-ast.wasm": "a8a0429b4a72f6937d26d455b60593b7aa8593a937c65c7d1f18dda20af484ad"
   }
 }

src/ast/native-loader.ts

@@ -23,17 +23,25 @@ import type {
   ORM,
   Framework,
 } from "../types.js";
-import { loadPlugin } from "../wasm/plugin-host.js";
+import { loadPlugin, listPluginFiles, type LoadedPlugin } from "../wasm/plugin-host.js";
 
 /** Effective, resolved native-AST settings for one scan. */
 export interface NativeAstResolved {
   mode: false | "on" | "strict";
-  /** Languages native parsing applies to. Empty set = all languages. */
+  /** Languages native parsing applies to. Empty set = all discovered languages. */
   languages: Set<NativeLang>;
+  /**
+   * True when an explicit language list was given (so those languages are
+   * AUTHORITATIVE — they preempt built-in extraction). Empty list / "all" =
+   * additive. Equivalent to `languages.size > 0`, captured for clarity.
+   */
+  authoritative: boolean;
   /** Plugin directories searched in order (PATH-style waterfall). */
   pluginDirs: string[];
   /** Shared, mutable sink for strict-mode diagnostics. */
   diagnostics: NativeDiagnostic[];
+  /** Shared, mutable sink for non-fatal warnings (collisions, name mismatch, etc.). */
+  warnings: string[];
 }
 
 /** A plugin adapted to codesight's domain types. Methods return null when the
@@ -47,8 +55,18 @@ export interface NativePlugin {
 const DISABLED: NativeAstResolved = {
   mode: false,
   languages: new Set(),
+  authoritative: false,
   pluginDirs: [],
   diagnostics: [],
+  warnings: [],
+};
+
+/** Built-in default extensions for the historically-known language ids, used when
+ *  a plugin omits `describe().extensions`. New languages must self-report. */
+const DEFAULT_EXTENSIONS: Record<string, string[]> = {
+  rust: [".rs"],
+  go: [".go"],
+  python: [".py"],
 };
 
 // Memoized by the `nativeAst` config object reference. scan() passes the same
@@ -70,11 +88,14 @@ export function resolveNativeAst(
   const cached = resolvedCache.get(cfg);
   if (cached) return cached;
 
+  const languages = new Set(cfg.languages ?? []);
   const resolved: NativeAstResolved = {
     mode: cfg.enabled === "strict" ? "strict" : "on",
-    languages: new Set(cfg.languages ?? []),
+    languages,
+    authoritative: languages.size > 0,
     pluginDirs: defaultPluginDirs(projectRoot, cfg.pluginDir),
     diagnostics: [],
+    warnings: [],
   };
   resolvedCache.set(cfg, resolved);
   return resolved;
@@ -108,6 +129,94 @@ export function isStrict(r: NativeAstResolved): boolean {
   return r.mode === "strict";
 }
 
+/** A language's adapted plugin + whether it preempts built-in extraction. */
+export interface NativeRegistryEntry {
+  lang: string;
+  authoritative: boolean;
+  plugin: NativePlugin;
+}
+
+/** Extension → owning plugin, for one scan. */
+export interface NativeRegistry {
+  /** Lowercased extension (leading dot) → the plugin that handles it. */
+  byExt: Map<string, NativeRegistryEntry>;
+}
+
+/**
+ * Build the extension→plugin registry: target the explicit languages (or
+ * enumerate all discovered plugins for "all"), resolve each language's
+ * identity (`describe().languageId` wins over the filename) and extensions
+ * (`describe().extensions`, else the built-in default map), and route extensions
+ * with the collision rule. Records strict diagnostics for enabled-but-unavailable
+ * or unroutable languages, and warnings for name mismatches / extension collisions.
+ */
+export function buildNativeRegistry(r: NativeAstResolved): NativeRegistry {
+  const byExt = new Map<string, NativeRegistryEntry>();
+  if (r.mode === false) return { byExt };
+
+  const candidates = r.authoritative
+    ? [...r.languages]
+    : listPluginFiles(r.pluginDirs).map((f) => f.lang);
+
+  for (const requested of candidates) {
+    const loaded = loadPlugin(requested, r.pluginDirs);
+    if (!loaded) {
+      if (r.mode === "strict") recordUnavailableLang(r, requested);
+      continue;
+    }
+
+    // `describe().languageId` is authoritative for identity; filename is fallback.
+    const reported = loaded.metadata?.languageId;
+    const lang = reported && reported.length > 0 ? reported : requested;
+    if (reported && reported.length > 0 && reported !== requested) {
+      addWarning(
+        r,
+        `plugin "codesight-${requested}-ast.wasm" self-reports languageId "${reported}" — honoring "${reported}" (rename the file to match for explicit --native-ast=${reported})`
+      );
+      // In explicit mode the user asked for `requested`; a file claiming a
+      // different id doesn't satisfy that request.
+      if (r.authoritative && !r.languages.has(lang)) {
+        if (r.mode === "strict") recordUnavailableLang(r, requested);
+        continue;
+      }
+    }
+
+    const declared = loaded.metadata?.extensions;
+    const exts = (declared && declared.length > 0 ? declared : DEFAULT_EXTENSIONS[lang]) ?? [];
+    if (exts.length === 0) {
+      if (r.mode === "strict") {
+        r.diagnostics.push({
+          lang,
+          kind: "routes",
+          reason: "no extensions declared (describe().extensions is required for non-built-in languages)",
+        });
+      }
+      continue;
+    }
+
+    const entry: NativeRegistryEntry = { lang, authoritative: r.authoritative, plugin: adaptPlugin(loaded) };
+    for (const raw of exts) {
+      const ext = (raw.startsWith(".") ? raw : "." + raw).toLowerCase();
+      const existing = byExt.get(ext);
+      if (!existing || existing.lang === entry.lang) {
+        byExt.set(ext, entry);
+        continue;
+      }
+      const winner = resolveCollision(existing, entry);
+      byExt.set(ext, winner);
+      addWarning(r, `extension "${ext}" claimed by both "${existing.lang}" and "${entry.lang}" — using "${winner.lang}"`);
+    }
+  }
+
+  return { byExt };
+}
+
+/** Collision tiebreak: explicit (authoritative) beats discovered; else first-registered wins. */
+function resolveCollision(existing: NativeRegistryEntry, candidate: NativeRegistryEntry): NativeRegistryEntry {
+  if (candidate.authoritative && !existing.authoritative) return candidate;
+  return existing;
+}
+
 /**
  * Get the native plugin for (lang, kind), or null when native is disabled for
  * the language, no plugin is available, or the plugin doesn't support the kind.
@@ -162,6 +271,17 @@ function recordUnavailable(r: NativeAstResolved, lang: NativeLang, kind: NativeK
   if (!dup) r.diagnostics.push({ lang, kind, reason: "plugin unavailable" });
 }
 
+/** Language-level "no plugin found" diagnostic (deduped per language). */
+function recordUnavailableLang(r: NativeAstResolved, lang: NativeLang): void {
+  const dup = r.diagnostics.some((d) => d.lang === lang && !d.file && d.reason === "plugin unavailable");
+  if (!dup) r.diagnostics.push({ lang, kind: "routes", reason: "plugin unavailable" });
+}
+
+/** Append a non-fatal warning (deduped). */
+function addWarning(r: NativeAstResolved, message: string): void {
+  if (!r.warnings.includes(message)) r.warnings.push(message);
+}
+
 /**
  * Build a domain-typed adapter around the raw plugin, or null if unavailable.
  * A method is exposed only when the plugin exports the matching capability, so
@@ -170,7 +290,12 @@ function recordUnavailable(r: NativeAstResolved, lang: NativeLang, kind: NativeK
 function getNativePlugin(lang: NativeLang, r: NativeAstResolved): NativePlugin | null {
   const loaded = loadPlugin(lang, r.pluginDirs);
   if (!loaded) return null;
+  return adaptPlugin(loaded);
+}
 
+/** Map a raw LoadedPlugin to codesight's domain types. A method is exposed only
+ *  when the plugin exports the matching capability (so callers can detect it). */
+function adaptPlugin(loaded: LoadedPlugin): NativePlugin {
   const np: NativePlugin = {};
 
   if (loaded.routes) {