From 4dc21dd7c2dd1868e50745e5444a860bdd7c7556 Mon Sep 17 00:00:00 2001 From: Mark S Date: Thu, 25 Jun 2026 09:21:48 -0400 Subject: [PATCH 1/4] feat(ast): self-describing plugins, registry discovery, all/none/explicit selection Add an optional describe()->i64 export (JSON {languageId, extensions}) that the host reads into LoadedPlugin.metadata, making modules fully self-describing alongside contractVersion() and parse* capability probes. - types: NativeLang widened to string; add PluginMetadata. - plugin-host: read describe() during bindExports; listPluginFiles globs the template-guarded ^codesight-()-ast.wasm$ across the dir waterfall. - native-loader: buildNativeRegistry maps extension -> plugin (describe-wins languageId, DEFAULT_EXTENSIONS fallback, first-match precedence, collision rule); NativeAstResolved gains authoritative + warnings. - index: --native-ast[=ids|all|none] selection (drops KNOWN_NATIVE_LANGS). - reference plugin: export describe(); rebuild wasm + checksums. Foundation only: built-in in-detector dispatch is still live, so behavior is unchanged. --- reference/ast-plugin/assembly/index.ts | 5 + reference/ast-plugin/checksums.json | 4 +- .../ast-plugin/codesight-reference-ast.wasm | Bin 4245 -> 4385 bytes src/ast/native-loader.ts | 131 +++++++++++++++++- src/index.ts | 107 +++++++------- src/types.ts | 22 ++- src/wasm/plugin-host.ts | 91 +++++++++--- 7 files changed, 289 insertions(+), 71 deletions(-) diff --git a/reference/ast-plugin/assembly/index.ts b/reference/ast-plugin/assembly/index.ts index 9d8596d..8975256 100644 --- a/reference/ast-plugin/assembly/index.ts +++ b/reference/ast-plugin/assembly/index.ts @@ -26,6 +26,11 @@ export function contractVersion(): i32 { return 1; } +/** Optional self-description: language id + the extensions this plugin parses. */ +export function describe(): i64 { + return emit("{\"languageId\":\"reference\",\"extensions\":[\".ref\"]}"); +} + // ─── ABI: memory management ─── export function alloc(len: i32): i32 { diff --git a/reference/ast-plugin/checksums.json b/reference/ast-plugin/checksums.json index f41e742..83fc48e 100644 --- a/reference/ast-plugin/checksums.json +++ b/reference/ast-plugin/checksums.json @@ -1,7 +1,7 @@ { "algorithm": "sha256", "files": { - "assembly/index.ts": "4d2f5917df6f2ca85c33fa59b390a4a87331812a0a2522528aa9c3a35d7b2229", - "codesight-reference-ast.wasm": "31c49dfa343a8fec0578c062a7f5bac013260de3e1852551a020ebc278b3b843" + "assembly/index.ts": "7dbd49e902cbc8c51a7aa19010a4893f1aab7c0ddba32ad5219cbfe988545ee3", + "codesight-reference-ast.wasm": "a8a0429b4a72f6937d26d455b60593b7aa8593a937c65c7d1f18dda20af484ad" } } diff --git a/reference/ast-plugin/codesight-reference-ast.wasm b/reference/ast-plugin/codesight-reference-ast.wasm index 8a2a1e847d94478ef8d232ed453ba0c497d9e48a..cf8ab86031ba63e97ce6ad1923d4612dbf2e8f2e 100644 GIT binary patch delta 1283 zcmZWnJ!~9B6n<}J_ilG*_I3iH;T&go6{IT+6{HBw4H7?z0-~b8fa07lmL10z?i>jz zz+D-n6pmIP3xQ;d#)XR@BV-DM3j~d8Xtr8F)m_-5DgS)q6{`@Z+y_rCXL zKhHLEMRS8&Ohnw)57f&yuMvMLvy_QhFnNTK?~YoXMG!_(N>aijUm~NX#E>ni@~C%% zq|;kj?ay^q-|6-Ti@g<^4(7XqPJi)wm!|x=<>g+7&e!L=)z`TPo0oJe+J81}vp2Nk z@Y}yO`KiMwW7$s0a4N2NZNSBs;<*s(GC_{nerA%m6 z3bDhecr5w?^$YUcE8`H9aT75xW@I+1xo!@G&3r=>X!><|8 zH$O)1Dt=NQJUUZCpMuzZ2-URYew&0&&LOK2QPxdrn#-^T0tV*QZ z62-e(ojukdCK}d~LdMyfPtyIs?BmrgFjVY?;F93-N-l{h+~#HK<9@b^Nt#r*FTxZK7{}OfxVLiRmZCZMi>9Y7l(iP zEwc9m#UGPh57dGnqHD0bl+!ZJ(F!fl9o!4lrOPxA^b)KO?*`1>E7ZZ6(+sRjAJJ-w z4QP>ihz-h$S)4B%SLO6ReS)5$;wc*1^-w9-)Pv0;WVg$J-3?XfaAy(O{R%z6CCEOi zyrWY&SkV2*QQGxLovEq_EqjpfB-w35XUOhGs_CLTVk^M@QEL*DhqtP7h!$Ze|Q#lyzQH36zLT8~)SL*bH4|29EboUgB9qd==fu2lq z3zN)aj%Dy8iqa!ptR+)HtoSO~t+@CudCfU45*NbcK|?VVZ#9Y;bJaOedM_zP=9&Kh DeNEC# delta 1137 zcmZWnJ!lj`6n<}J_ilIJ-rZpQ0rR(e#E>GU5{qbYW1%rb!Nwv{A;f?=@<(!DC3kia zf+P%rO(4N0h($s`B3N9JB1ObhlENaDSP0sgLc}+_A$P=LX6JqL=KGo1wc@LoV03;8 z0RXSlGF=;yQ6~bTKyV;|!G%l~h35xSN)QgXk{dE4s;GgIDLb4kjLl3}=0?XV_s8ev zCuXLh-5s5roEd|T?Bnrt)waCZA9e>j7duPnRCV3k{Jn-}_Wc-TDFTI&7<5Vuo44Z9 zm3?2i+=Ol<^)FtNQ;pEBz>Sb1sUYS98q2W=k%;&9eIw+gaDiv^ej#zGZF+%y0GiE69CsJ_Yt;KGUPibx%1IWeTK0WhjJ%J*Qe# zfnM;fr)oKkU8q+*GaR;7^F`BM_&!{|dp0Wds?V~hgz^;Vr9d&G7RP0s zODos7Cw&-jPU6;8o>+ADw5H; + /** + * True when an explicit language list was given (so those languages are + * AUTHORITATIVE — they preempt built-in extraction). Empty list / "all" = + * additive. Equivalent to `languages.size > 0`, captured for clarity. + */ + authoritative: boolean; /** Plugin directories searched in order (PATH-style waterfall). */ pluginDirs: string[]; /** Shared, mutable sink for strict-mode diagnostics. */ diagnostics: NativeDiagnostic[]; + /** Shared, mutable sink for non-fatal warnings (collisions, name mismatch, etc.). */ + warnings: string[]; } /** A plugin adapted to codesight's domain types. Methods return null when the @@ -47,8 +55,18 @@ export interface NativePlugin { const DISABLED: NativeAstResolved = { mode: false, languages: new Set(), + authoritative: false, pluginDirs: [], diagnostics: [], + warnings: [], +}; + +/** Built-in default extensions for the historically-known language ids, used when + * a plugin omits `describe().extensions`. New languages must self-report. */ +const DEFAULT_EXTENSIONS: Record = { + rust: [".rs"], + go: [".go"], + python: [".py"], }; // Memoized by the `nativeAst` config object reference. scan() passes the same @@ -70,11 +88,14 @@ export function resolveNativeAst( const cached = resolvedCache.get(cfg); if (cached) return cached; + const languages = new Set(cfg.languages ?? []); const resolved: NativeAstResolved = { mode: cfg.enabled === "strict" ? "strict" : "on", - languages: new Set(cfg.languages ?? []), + languages, + authoritative: languages.size > 0, pluginDirs: defaultPluginDirs(projectRoot, cfg.pluginDir), diagnostics: [], + warnings: [], }; resolvedCache.set(cfg, resolved); return resolved; @@ -108,6 +129,94 @@ export function isStrict(r: NativeAstResolved): boolean { return r.mode === "strict"; } +/** A language's adapted plugin + whether it preempts built-in extraction. */ +export interface NativeRegistryEntry { + lang: string; + authoritative: boolean; + plugin: NativePlugin; +} + +/** Extension → owning plugin, for one scan. */ +export interface NativeRegistry { + /** Lowercased extension (leading dot) → the plugin that handles it. */ + byExt: Map; +} + +/** + * Build the extension→plugin registry: target the explicit languages (or + * enumerate all discovered plugins for "all"), resolve each language's + * identity (`describe().languageId` wins over the filename) and extensions + * (`describe().extensions`, else the built-in default map), and route extensions + * with the collision rule. Records strict diagnostics for enabled-but-unavailable + * or unroutable languages, and warnings for name mismatches / extension collisions. + */ +export function buildNativeRegistry(r: NativeAstResolved): NativeRegistry { + const byExt = new Map(); + if (r.mode === false) return { byExt }; + + const candidates = r.authoritative + ? [...r.languages] + : listPluginFiles(r.pluginDirs).map((f) => f.lang); + + for (const requested of candidates) { + const loaded = loadPlugin(requested, r.pluginDirs); + if (!loaded) { + if (r.mode === "strict") recordUnavailableLang(r, requested); + continue; + } + + // `describe().languageId` is authoritative for identity; filename is fallback. + const reported = loaded.metadata?.languageId; + const lang = reported && reported.length > 0 ? reported : requested; + if (reported && reported.length > 0 && reported !== requested) { + addWarning( + r, + `plugin "codesight-${requested}-ast.wasm" self-reports languageId "${reported}" — honoring "${reported}" (rename the file to match for explicit --native-ast=${reported})` + ); + // In explicit mode the user asked for `requested`; a file claiming a + // different id doesn't satisfy that request. + if (r.authoritative && !r.languages.has(lang)) { + if (r.mode === "strict") recordUnavailableLang(r, requested); + continue; + } + } + + const declared = loaded.metadata?.extensions; + const exts = (declared && declared.length > 0 ? declared : DEFAULT_EXTENSIONS[lang]) ?? []; + if (exts.length === 0) { + if (r.mode === "strict") { + r.diagnostics.push({ + lang, + kind: "routes", + reason: "no extensions declared (describe().extensions is required for non-built-in languages)", + }); + } + continue; + } + + const entry: NativeRegistryEntry = { lang, authoritative: r.authoritative, plugin: adaptPlugin(loaded) }; + for (const raw of exts) { + const ext = (raw.startsWith(".") ? raw : "." + raw).toLowerCase(); + const existing = byExt.get(ext); + if (!existing || existing.lang === entry.lang) { + byExt.set(ext, entry); + continue; + } + const winner = resolveCollision(existing, entry); + byExt.set(ext, winner); + addWarning(r, `extension "${ext}" claimed by both "${existing.lang}" and "${entry.lang}" — using "${winner.lang}"`); + } + } + + return { byExt }; +} + +/** Collision tiebreak: explicit (authoritative) beats discovered; else first-registered wins. */ +function resolveCollision(existing: NativeRegistryEntry, candidate: NativeRegistryEntry): NativeRegistryEntry { + if (candidate.authoritative && !existing.authoritative) return candidate; + return existing; +} + /** * Get the native plugin for (lang, kind), or null when native is disabled for * the language, no plugin is available, or the plugin doesn't support the kind. @@ -162,6 +271,17 @@ function recordUnavailable(r: NativeAstResolved, lang: NativeLang, kind: NativeK if (!dup) r.diagnostics.push({ lang, kind, reason: "plugin unavailable" }); } +/** Language-level "no plugin found" diagnostic (deduped per language). */ +function recordUnavailableLang(r: NativeAstResolved, lang: NativeLang): void { + const dup = r.diagnostics.some((d) => d.lang === lang && !d.file && d.reason === "plugin unavailable"); + if (!dup) r.diagnostics.push({ lang, kind: "routes", reason: "plugin unavailable" }); +} + +/** Append a non-fatal warning (deduped). */ +function addWarning(r: NativeAstResolved, message: string): void { + if (!r.warnings.includes(message)) r.warnings.push(message); +} + /** * Build a domain-typed adapter around the raw plugin, or null if unavailable. * A method is exposed only when the plugin exports the matching capability, so @@ -170,7 +290,12 @@ function recordUnavailable(r: NativeAstResolved, lang: NativeLang, kind: NativeK function getNativePlugin(lang: NativeLang, r: NativeAstResolved): NativePlugin | null { const loaded = loadPlugin(lang, r.pluginDirs); if (!loaded) return null; + return adaptPlugin(loaded); +} +/** Map a raw LoadedPlugin to codesight's domain types. A method is exposed only + * when the plugin exports the matching capability (so callers can detect it). */ +function adaptPlugin(loaded: LoadedPlugin): NativePlugin { const np: NativePlugin = {}; if (loaded.routes) { diff --git a/src/index.ts b/src/index.ts index 07a1e41..5e555c4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,8 +40,8 @@ function printHelp() { --since Show only routes from files changed since git ref/commit --mode Scan mode: code (default) | knowledge (map .md notes) --refresh [pkg] Rebuild monorepo package context (all or named package) - --native-ast [langs] Use WASM AST plugins when present (optional list: rust,go,python) - --native-ast-strict Like --native-ast, but report + fail if a plugin is missing + --native-ast[=langs] Use WASM AST plugins (=all default, =none to force off, or =rust,go,…) + --native-ast-strict Like --native-ast, but report + fail if a named plugin is missing --plugin-dir Extra directory to search for WASM plugins -v, --version Show version -h, --help Show this help @@ -80,52 +80,73 @@ async function fileExists(path: string): Promise { } } -const KNOWN_NATIVE_LANGS: NativeLang[] = ["rust", "go", "python"]; - -/** True when `token` is a comma list of known native languages (and only those). */ -function isLangList(token: string): boolean { - if (!token || token.startsWith("-")) return false; - return token - .split(",") - .every((t) => (KNOWN_NATIVE_LANGS as string[]).includes(t.trim().toLowerCase())); -} - +/** Split a comma list of language ids → normalized string[]. */ function parseLangs(token: string): NativeLang[] { return token .split(",") .map((t) => t.trim().toLowerCase()) - .filter(Boolean) as NativeLang[]; + .filter(Boolean); +} + +/** Accumulated native-AST CLI intent. `none` forces off; empty `languages` = all. */ +interface NativeAstCli { + enabled?: boolean | "strict"; + none?: boolean; + languages?: NativeLang[]; +} + +/** + * Apply one `--native-ast[=val]` / `--native-ast-strict[=val]` flag. + * `val`: "" or "all" → all; "none" → force off; otherwise a comma list of ids. + */ +function applyNativeAstArg(cli: NativeAstCli, val: string, strict: boolean): void { + const v = val.trim().toLowerCase(); + if (v === "none") { + cli.none = true; + return; + } + cli.enabled = cli.enabled === "strict" || strict ? "strict" : true; + if (v !== "" && v !== "all") cli.languages = parseLangs(v); } -/** Parse CODESIGHT_NATIVE_AST: "1"|"true"|"on"|"strict"|"rust,go" → partial config. */ +/** Parse CODESIGHT_NATIVE_AST: ""|none|off|0|false → off; 1|true|on|all → all; strict; or a comma id list. */ function parseNativeAstEnv( v: string | undefined ): { enabled: boolean | "strict"; languages?: NativeLang[] } | undefined { if (!v) return undefined; const t = v.trim().toLowerCase(); - if (t === "" || t === "0" || t === "false" || t === "off") return undefined; + if (t === "") return undefined; + if (t === "0" || t === "false" || t === "off" || t === "none") return { enabled: false }; if (t === "strict") return { enabled: "strict" }; - if (t === "1" || t === "true" || t === "on") return { enabled: true }; - if (isLangList(t)) return { enabled: true, languages: parseLangs(t) }; - return undefined; + if (t === "1" || t === "true" || t === "on" || t === "all") return { enabled: true }; + const langs = parseLangs(t); + return langs.length ? { enabled: true, languages: langs } : undefined; } -/** Merge native-AST CLI flags with env vars (CLI wins per field). */ -function resolveNativeAstCli( - cliEnabled: boolean | "strict" | undefined, - cliLangs: NativeLang[] | undefined, - cliPluginDir: string -): NativeAstConfig | undefined { +/** Merge native-AST CLI flags with env vars into a config (CLI > env > config file). */ +function resolveNativeAstCli(cli: NativeAstCli, cliPluginDir: string): NativeAstConfig | undefined { + const pickDir = (cfg: NativeAstConfig) => { + const dir = cliPluginDir || process.env.CODESIGHT_PLUGIN_DIR; + if (dir) cfg.pluginDir = dir; + return cfg; + }; + + if (cli.none) return { enabled: false }; // explicit off, overrides env + config file + if (cli.enabled) { + const cfg: NativeAstConfig = { enabled: cli.enabled }; + if (cli.languages?.length) cfg.languages = cli.languages; + return pickDir(cfg); + } + const env = parseNativeAstEnv(process.env.CODESIGHT_NATIVE_AST); - const enabled = cliEnabled ?? env?.enabled; - if (!enabled) return undefined; - - const cfg: NativeAstConfig = { enabled }; - const languages = cliLangs ?? env?.languages; - if (languages && languages.length) cfg.languages = languages; - const dir = cliPluginDir || process.env.CODESIGHT_PLUGIN_DIR; - if (dir) cfg.pluginDir = dir; - return cfg; + if (env) { + if (env.enabled === false) return { enabled: false }; + const cfg: NativeAstConfig = { enabled: env.enabled }; + if (env.languages?.length) cfg.languages = env.languages; + return pickDir(cfg); + } + + return undefined; // no CLI/env opinion → config file decides } async function installGitHook(root: string, outputDirName: string) { @@ -319,8 +340,7 @@ async function main() { let mode = "code"; let doRefresh = false; let refreshPackage = ""; - let nativeAstEnabled: boolean | "strict" | undefined; - let nativeAstLangs: NativeLang[] | undefined; + const nativeAstCli: NativeAstCli = {}; let pluginDir = ""; for (let i = 0; i < args.length; i++) { @@ -367,17 +387,10 @@ async function main() { if (args[i + 1] && !args[i + 1].startsWith("-")) { refreshPackage = args[++i]; } - } else if (arg === "--native-ast") { - if (nativeAstEnabled !== "strict") nativeAstEnabled = true; - // Optional next token: a comma list of known languages (must not swallow - // the target directory, e.g. `--native-ast ./project`). - const next = args[i + 1]; - if (next && isLangList(next)) { - nativeAstLangs = parseLangs(next); - i++; - } - } else if (arg === "--native-ast-strict") { - nativeAstEnabled = "strict"; + } else if (arg === "--native-ast" || arg.startsWith("--native-ast=")) { + applyNativeAstArg(nativeAstCli, arg.includes("=") ? arg.slice(arg.indexOf("=") + 1) : "", false); + } else if (arg === "--native-ast-strict" || arg.startsWith("--native-ast-strict=")) { + applyNativeAstArg(nativeAstCli, arg.includes("=") ? arg.slice(arg.indexOf("=") + 1) : "", true); } else if (arg === "--plugin-dir" && args[i + 1]) { pluginDir = args[++i]; } else if (!arg.startsWith("-")) { @@ -403,7 +416,7 @@ async function main() { // Resolve native-AST settings: CLI takes precedence over env, both over the // config file (merged below). Undefined unless explicitly enabled somewhere. - const nativeAst = resolveNativeAstCli(nativeAstEnabled, nativeAstLangs, pluginDir); + const nativeAst = resolveNativeAstCli(nativeAstCli, pluginDir); // Load config file const fileConfig = await loadConfig(root); diff --git a/src/types.ts b/src/types.ts index ae3d059..3ffecc8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -232,12 +232,30 @@ export interface CodesightConfig { rokuScreenHelpers?: string[]; } -/** Native language to which a WASM AST plugin applies. */ -export type NativeLang = "rust" | "go" | "python"; +/** + * A native language identifier. Open-ended: a plugin declares the language it + * handles (via `describe().languageId`, else its filename), so any string is a + * valid id — `rust`/`go`/`python` are merely the ids with built-in extractors. + */ +export type NativeLang = string; /** Extraction capability a native plugin can provide. */ export type NativeKind = "routes" | "schemas" | "imports"; +/** + * Optional self-reported plugin metadata (from a `describe()` export). The host + * consumes `languageId` + `extensions`; other fields are carried but unused for + * now (e.g. `frameworks`, reserved for future framework labeling). + */ +export interface PluginMetadata { + /** Authoritative language id when present/non-empty (else the filename ``). */ + languageId?: string; + /** File extensions this plugin parses, e.g. [".rs"]. Required for non-built-in languages. */ + extensions?: string[]; + /** Frameworks the plugin can label results with — carried, not yet consumed. */ + frameworks?: string[]; +} + export interface NativeAstConfig { /** * true → try the plugin, silently fall back to the built-in parser on miss. diff --git a/src/wasm/plugin-host.ts b/src/wasm/plugin-host.ts index 6d5941d..a25385b 100644 --- a/src/wasm/plugin-host.ts +++ b/src/wasm/plugin-host.ts @@ -26,10 +26,10 @@ * instantiated once and the parse functions are called per file — a reactor, * not a per-file process. */ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { createRequire } from "node:module"; -import type { NativeLang } from "../types.js"; +import type { NativeLang, PluginMetadata } from "../types.js"; // `node:wasi` is loaded lazily (below) rather than via a static import: importing // it emits a one-time ExperimentalWarning at module-load, which would fire on @@ -52,11 +52,52 @@ export interface LoadedPlugin { routes?(source: string): unknown; schemas?(source: string): unknown; imports?(source: string): unknown; + /** Self-reported metadata from the optional `describe()` export (if any). */ + metadata?: PluginMetadata; } /** Resolves a language to a LoadedPlugin (or null if none found). Swappable for tests. */ export type PluginProvider = (lang: NativeLang, pluginDirs: string[]) => LoadedPlugin | null; +/** A plugin binary found on the search path: its filename language id + path. */ +export interface PluginFile { + /** The `` captured from the filename — a discovery hint / fallback id. */ + lang: string; + path: string; +} + +/** Only files matching this template are ever considered plugins. */ +const PLUGIN_FILENAME = /^codesight-([a-z0-9_-]+)-ast\.wasm$/; + +/** + * Enumerate plugin binaries across the search path (for `all` mode). Returns one + * entry per filename language id, first-match-wins by waterfall dir order (lower + * dirs shadowed), deterministic within a dir. Only template-matching files are + * returned — arbitrary `.wasm` is ignored. Filesystem-only (a test provider is + * for load-by-id, not enumeration). + */ +export function listPluginFiles(pluginDirs: string[]): PluginFile[] { + const seen = new Set(); + const out: PluginFile[] = []; + for (const dir of pluginDirs) { + let entries: string[]; + try { + entries = readdirSync(dir).sort(); + } catch { + continue; // dir doesn't exist / unreadable + } + for (const name of entries) { + const m = PLUGIN_FILENAME.exec(name); + if (!m) continue; + const lang = m[1]; + if (seen.has(lang)) continue; // waterfall first-wins + seen.add(lang); + out.push({ lang, path: join(dir, name) }); + } + } + return out; +} + // ─── Test seam ─── let providerOverride: PluginProvider | null = null; /** Install a fake provider (tests). */ @@ -148,6 +189,7 @@ interface WasmExports { alloc(len: number): number; dealloc(ptr: number, len: number): void; contractVersion(): number; + describe?(): bigint; parseRoutes?(srcPtr: number, srcLen: number): bigint; parseSchemas?(srcPtr: number, srcLen: number): bigint; parseImports?(srcPtr: number, srcLen: number): bigint; @@ -176,25 +218,14 @@ export function bindExports(rawExports: unknown): LoadedPlugin | null { const dec = new TextDecoder(); const enc = new TextEncoder(); - const bind = (fn: (p: number, l: number) => bigint) => (source: string): unknown => { - const bytes = enc.encode(source); - const ptr = ex.alloc(bytes.length) >>> 0; - // Re-acquire the view after alloc — memory.grow detaches the ArrayBuffer. - new Uint8Array(ex.memory.buffer, ptr, bytes.length).set(bytes); - - let packed: bigint; - try { - packed = fn(ptr, bytes.length); // may trap → throws - } finally { - ex.dealloc(ptr, bytes.length); - } - + // Read a packed (outPtr<<32)|outLen return value into parsed JSON, freeing the + // output buffer. Returns null on an empty result or invalid JSON. + const readPacked = (packed: bigint): unknown => { const p = BigInt(packed); const outPtr = Number(BigInt.asUintN(32, p >> 32n)); const outLen = Number(BigInt.asUintN(32, p)); if (outLen === 0) return null; - - // Re-acquire again — parse may have grown memory while building output. + // Re-acquire the view — the call may have grown memory (detaches ArrayBuffer). const json = dec.decode(new Uint8Array(ex.memory.buffer, outPtr, outLen)); ex.dealloc(outPtr, outLen); try { @@ -204,9 +235,35 @@ export function bindExports(rawExports: unknown): LoadedPlugin | null { } }; + const bind = (fn: (p: number, l: number) => bigint) => (source: string): unknown => { + const bytes = enc.encode(source); + const ptr = ex.alloc(bytes.length) >>> 0; + // Re-acquire the view after alloc — memory.grow detaches the ArrayBuffer. + new Uint8Array(ex.memory.buffer, ptr, bytes.length).set(bytes); + let packed: bigint; + try { + packed = fn(ptr, bytes.length); // may trap → throws + } finally { + ex.dealloc(ptr, bytes.length); + } + return readPacked(packed); + }; + const plugin: LoadedPlugin = {}; if (typeof ex.parseRoutes === "function") plugin.routes = bind(ex.parseRoutes); if (typeof ex.parseSchemas === "function") plugin.schemas = bind(ex.parseSchemas); if (typeof ex.parseImports === "function") plugin.imports = bind(ex.parseImports); + + // Optional self-description (no input). Carried for discovery/routing; a plugin + // that omits it falls back to its filename id + a default extension map. + if (typeof ex.describe === "function") { + try { + const meta = readPacked(ex.describe()); + if (meta && typeof meta === "object") plugin.metadata = meta as PluginMetadata; + } catch { + /* ignore a broken describe() — treat as no metadata */ + } + } + return plugin; } From a2669165d390a3b2f1f303b105da98febbcfb114 Mon Sep 17 00:00:00 2001 From: Mark S Date: Thu, 25 Jun 2026 09:21:57 -0400 Subject: [PATCH 2/4] feat(detectors): language-driven generic native pass; migrate off in-detector dispatch Introduce src/detectors/native.ts as the single point where WASM plugins are dispatched: group files by extension via the registry, call parseRoutes/ parseSchemas, stamp confidence "native" (framework "unknown" interim), and record strict diagnostics. mergeNativeRoutes is native-preferred-dedup (additive) or per-file replace (authoritative, with an empty-with-builtin warning); mergeNativeSchemas dedups by name (SchemaModel lacks file provenance). - routes/schema: drop the per-detector nativePluginFor calls; built-in extractors remain as pure fallback. Export detectTags for the pass. - core: run detectNative after the built-in detectors and merge. - tests: native-ast dispatch tests move to the generic pass (detectNative). --- src/core.ts | 17 ++++- src/detectors/native.ts | 145 +++++++++++++++++++++++++++++++++++++++ src/detectors/routes.ts | 85 +++-------------------- src/detectors/schema.ts | 63 +++-------------- tests/native-ast.test.ts | 59 ++++++++-------- 5 files changed, 209 insertions(+), 160 deletions(-) create mode 100644 src/detectors/native.ts diff --git a/src/core.ts b/src/core.ts index ae232a9..cc36c55 100644 --- a/src/core.ts +++ b/src/core.ts @@ -16,6 +16,7 @@ import { detectOpenAPISpec } from "./detectors/openapi.js"; import { writeOutput, computeCrudGroups } from "./formatter.js"; import { resolveNativeAst } from "./ast/native-loader.js"; import { createBuiltinPlugins } from "./plugins/index.js"; +import { detectNative, mergeNativeRoutes, mergeNativeSchemas } from "./detectors/native.js"; import { createRequire } from "node:module"; import type { ScanResult, CodesightConfig } from "./types.js"; @@ -72,7 +73,7 @@ export async function scan( // diagnostics sink) via reference memoization. const nativeResolved = resolveNativeAst(userConfig.nativeAst, project.root); - const [rawHttpRoutes, schemas, components, libs, configResult, middleware, graph, + const [rawHttpRoutes, builtinSchemas, components, libs, configResult, middleware, graph, graphqlRoutes, grpcRoutes, wsRoutes, events, openapi] = await Promise.all([ disabled.has("routes") ? Promise.resolve([]) : detectRoutes(files, project, userConfig), @@ -89,8 +90,19 @@ export async function scan( detectOpenAPISpec(root, project), ]); + // Generic language-driven native pass — the single place WASM plugins are + // dispatched. Merge into the built-in results (native-preferred; authoritative + // languages replace built-ins per file — routes only, see native.ts). + const native = await detectNative(files, project, nativeResolved); + const httpRoutes = disabled.has("routes") + ? rawHttpRoutes + : mergeNativeRoutes(rawHttpRoutes, native.routes, nativeResolved, native.registry); + const schemas = disabled.has("schema") + ? builtinSchemas + : mergeNativeSchemas(builtinSchemas, native.schemas); + // Merge OpenAPI routes and schemas if spec found - const rawRoutes = [...rawHttpRoutes, ...graphqlRoutes, ...grpcRoutes, ...wsRoutes]; + const rawRoutes = [...httpRoutes, ...graphqlRoutes, ...grpcRoutes, ...wsRoutes]; if (openapi.routes.length > 0) { if (rawRoutes.length === 0) rawRoutes.push(...openapi.routes); const existingModelNames = new Set(schemas.map((m) => m.name.toLowerCase())); @@ -160,6 +172,7 @@ export async function scan( } else { console.log(" done"); } + for (const w of nativeResolved.warnings) console.warn(` Native-AST: ${w}`); } // Step 5: Write output diff --git a/src/detectors/native.ts b/src/detectors/native.ts new file mode 100644 index 0000000..713ba6f --- /dev/null +++ b/src/detectors/native.ts @@ -0,0 +1,145 @@ +/** + * Generic, language-driven native-AST pass. + * + * Unlike the built-in detectors (which run per detected framework/ORM), this + * pass routes files to plugins purely by file extension via the plugin registry, + * so any language a plugin declares is handled — no framework detection needed. + * It is the single place native plugins are dispatched. + * + * Results are merged with the built-in extractors in core.ts: + * - routes: additive (union, native-preferred dedup on method:path) when enabled + * via `all`/config; authoritative (native replaces built-in per file, with a + * fallback + warning when the plugin is empty for a file the built-in handled) + * when languages are explicitly named. + * - schemas: native-preferred dedup by name in both modes (SchemaModel carries no + * file provenance, so per-file authoritative replacement isn't possible). + */ +import { relative, extname } from "node:path"; +import { readFileSafe } from "../scanner.js"; +import { detectTags } from "./routes.js"; +import { + buildNativeRegistry, + recordParseError, + type NativeAstResolved, + type NativeRegistry, +} from "../ast/native-loader.js"; +import type { ProjectInfo, RouteInfo, SchemaModel, Framework } from "../types.js"; + +export interface NativeExtraction { + routes: RouteInfo[]; + schemas: SchemaModel[]; + /** The registry used, for ownership checks during the merge. */ + registry: NativeRegistry; +} + +/** + * Run every enabled plugin over the files it claims (by extension). Returns native + * routes/schemas (already stamped `confidence: "native"` by the adapter) plus the + * registry. Inert (empty) when native AST is disabled or no plugin is registered. + */ +export async function detectNative( + files: string[], + project: ProjectInfo, + resolved: NativeAstResolved +): Promise { + const registry = buildNativeRegistry(resolved); + const routes: RouteInfo[] = []; + const schemas: SchemaModel[] = []; + if (registry.byExt.size === 0) return { routes, schemas, registry }; + + for (const file of files) { + const ext = extname(file).toLowerCase(); + const entry = registry.byExt.get(ext); + if (!entry) continue; + + const content = await readFileSafe(file); + if (!content) continue; + const rel = relative(project.root, file); + + if (entry.plugin.routes) { + try { + const r = entry.plugin.routes(rel, content, "unknown" as Framework, detectTags(content)); + if (r && r.length) routes.push(...r); + } catch (e) { + recordParseError(resolved, entry.lang, "routes", rel, e); + } + } + if (entry.plugin.schemas) { + try { + const s = entry.plugin.schemas(rel, content); + if (s && s.length) schemas.push(...s); + } catch (e) { + recordParseError(resolved, entry.lang, "schemas", rel, e); + } + } + } + + return { routes, schemas, registry }; +} + +/** + * Merge native routes with built-in routes. + * - additive: union, native-preferred dedup on method:path. + * - authoritative: for a file owned by a native plugin, native results replace + * built-in results from that file; if the plugin was empty for an owned file + * the built-in DID extract, built-in stands and a warning is recorded. + */ +export function mergeNativeRoutes( + builtin: RouteInfo[], + native: RouteInfo[], + resolved: NativeAstResolved, + registry: NativeRegistry +): RouteInfo[] { + if (native.length === 0 && !resolved.authoritative) return builtin; + + const seen = new Set(); + const result: RouteInfo[] = []; + const add = (r: RouteInfo) => { + const key = `${r.method}:${r.path}`; + if (seen.has(key)) return; + seen.add(key); + result.push(r); + }; + + for (const r of native) add(r); // native preferred + + if (resolved.authoritative) { + const nativeFiles = new Set(native.map((r) => r.file)); + const warned = new Set(); + for (const r of builtin) { + const owned = registry.byExt.has(extname(r.file).toLowerCase()); + if (owned) { + if (nativeFiles.has(r.file)) continue; // native authoritative for this file + if (!warned.has(r.file)) { + warned.add(r.file); + warn(resolved, `native plugin found nothing in ${r.file}; using built-in extraction`); + } + } + add(r); + } + } else { + for (const r of builtin) add(r); // additive union + } + + return result; +} + +/** Merge native + built-in schemas, native-preferred dedup by model name. */ +export function mergeNativeSchemas(builtin: SchemaModel[], native: SchemaModel[]): SchemaModel[] { + if (native.length === 0) return builtin; + const seen = new Set(); + const result: SchemaModel[] = []; + const add = (m: SchemaModel) => { + const key = m.name.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + result.push(m); + }; + for (const m of native) add(m); + for (const m of builtin) add(m); + return result; +} + +function warn(resolved: NativeAstResolved, message: string): void { + if (!resolved.warnings.includes(message)) resolved.warnings.push(message); +} diff --git a/src/detectors/routes.ts b/src/detectors/routes.ts index 0814f4c..346b237 100644 --- a/src/detectors/routes.ts +++ b/src/detectors/routes.ts @@ -10,7 +10,6 @@ import { extractFlutterRoutes } from "../ast/extract-dart.js"; import { extractVaporRoutes } from "../ast/extract-swift.js"; import { extractRetrofitRoutes, extractNavigationRoutes, extractActivitiesFromManifest } from "../ast/extract-android.js"; import type { RouteInfo, Framework, ProjectInfo, CodesightConfig } from "../types.js"; -import { resolveNativeAst, nativePluginFor, recordParseError, type NativeAstResolved } from "../ast/native-loader.js"; const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"]; @@ -25,7 +24,7 @@ const TAG_PATTERNS: [string, RegExp[]][] = [ ["ai", [/openai/i, /anthropic/i, /claude/i, /\.chat\.completions/i, /\.messages\.create/i]], ]; -function detectTags(content: string): string[] { +export function detectTags(content: string): string[] { const tags: string[] = []; for (const [tag, patterns] of TAG_PATTERNS) { if (patterns.some((p) => p.test(content))) { @@ -41,7 +40,6 @@ export async function detectRoutes( config?: CodesightConfig ): Promise { const routes: RouteInfo[] = []; - const native = resolveNativeAst(config?.nativeAst, project.root); for (const fw of project.frameworks) { switch (fw) { @@ -85,20 +83,20 @@ export async function detectRoutes( routes.push(...(await detectNuxtRoutes(files, project))); break; case "fastapi": - routes.push(...(await detectFastAPIRoutes(files, project, native))); + routes.push(...(await detectFastAPIRoutes(files, project))); break; case "flask": - routes.push(...(await detectFlaskRoutes(files, project, native))); + routes.push(...(await detectFlaskRoutes(files, project))); break; case "django": - routes.push(...(await detectDjangoRoutes(files, project, native))); + routes.push(...(await detectDjangoRoutes(files, project))); break; case "gin": case "go-net-http": case "fiber": case "echo": case "chi": - routes.push(...(await detectGoRoutes(files, project, fw, native))); + routes.push(...(await detectGoRoutes(files, project, fw))); break; case "rails": routes.push(...(await detectRailsRoutes(files, project))); @@ -114,7 +112,7 @@ export async function detectRoutes( break; case "actix": case "axum": - routes.push(...(await detectRustRoutes(files, project, fw, native))); + routes.push(...(await detectRustRoutes(files, project, fw))); break; case "raw-http": routes.push(...(await detectRawHttpRoutes(files, project))); @@ -789,12 +787,10 @@ async function detectNuxtRoutes( // --- FastAPI --- async function detectFastAPIRoutes( files: string[], - project: ProjectInfo, - native?: NativeAstResolved + project: ProjectInfo ): Promise { const pyFiles = files.filter((f) => f.endsWith(".py")); const routes: RouteInfo[] = []; - const np = native ? nativePluginFor("python", "routes", native) : null; for (const file of pyFiles) { const content = await readFileSafe(file); @@ -803,16 +799,6 @@ async function detectFastAPIRoutes( const rel = relative(project.root, file); const tags = detectTags(content); - // Native WASM plugin first (when enabled + present); falls through on miss. - if (np?.routes) { - try { - const r = np.routes(rel, content, "fastapi", tags); - if (r && r.length) { routes.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "python", "routes", rel, e); - } - } - // Try Python AST first const astRoutes = await extractPythonRoutesAST(rel, content, "fastapi", tags); if (astRoutes && astRoutes.length > 0) { @@ -841,12 +827,10 @@ async function detectFastAPIRoutes( // --- Flask --- async function detectFlaskRoutes( files: string[], - project: ProjectInfo, - native?: NativeAstResolved + project: ProjectInfo ): Promise { const pyFiles = files.filter((f) => f.endsWith(".py")); const routes: RouteInfo[] = []; - const np = native ? nativePluginFor("python", "routes", native) : null; for (const file of pyFiles) { const content = await readFileSafe(file); @@ -855,16 +839,6 @@ async function detectFlaskRoutes( const rel = relative(project.root, file); const tags = detectTags(content); - // Native WASM plugin first (when enabled + present); falls through on miss. - if (np?.routes) { - try { - const r = np.routes(rel, content, "flask", tags); - if (r && r.length) { routes.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "python", "routes", rel, e); - } - } - // Try Python AST first const astRoutes = await extractPythonRoutesAST(rel, content, "flask", tags); if (astRoutes && astRoutes.length > 0) { @@ -900,30 +874,18 @@ async function detectFlaskRoutes( // --- Django --- async function detectDjangoRoutes( files: string[], - project: ProjectInfo, - native?: NativeAstResolved + project: ProjectInfo ): Promise { const pyFiles = files.filter( (f) => f.endsWith(".py") && (basename(f) === "urls.py" || basename(f) === "views.py") ); const routes: RouteInfo[] = []; - const np = native ? nativePluginFor("python", "routes", native) : null; for (const file of pyFiles) { const content = await readFileSafe(file); const rel = relative(project.root, file); const tags = detectTags(content); - // Native WASM plugin first (when enabled + present); falls through on miss. - if (np?.routes) { - try { - const r = np.routes(rel, content, "django", tags); - if (r && r.length) { routes.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "python", "routes", rel, e); - } - } - // Try Python AST first const astRoutes = await extractPythonRoutesAST(rel, content, "django", tags); if (astRoutes && astRoutes.length > 0) { @@ -952,28 +914,16 @@ async function detectDjangoRoutes( async function detectGoRoutes( files: string[], project: ProjectInfo, - fw: Framework, - native?: NativeAstResolved + fw: Framework ): Promise { const goFiles = files.filter((f) => f.endsWith(".go")); const routes: RouteInfo[] = []; - const np = native ? nativePluginFor("go", "routes", native) : null; for (const file of goFiles) { const content = await readFileSafe(file); const rel = relative(project.root, file); const tags = detectTags(content); - // Native WASM plugin first (when enabled + present); falls through on miss. - if (np?.routes) { - try { - const r = np.routes(rel, content, fw, tags); - if (r && r.length) { routes.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "go", "routes", rel, e); - } - } - // Use structured parser (brace-tracking + group prefix resolution) const structuredRoutes = extractGoRoutesStructured(rel, content, fw, tags); if (structuredRoutes.length > 0) { @@ -1243,28 +1193,15 @@ function extractKtorParams(path: string): string[] { async function detectRustRoutes( files: string[], project: ProjectInfo, - fw: Framework, - native?: NativeAstResolved + fw: Framework ): Promise { const rsFiles = files.filter((f) => f.endsWith(".rs")); const routes: RouteInfo[] = []; - const np = native ? nativePluginFor("rust", "routes", native) : null; for (const file of rsFiles) { const content = await readFileSafe(file); const rel = relative(project.root, file); - // Native WASM plugin first (when enabled + present). Rust currently has no - // built-in AST path — this is the only AST-grade option for Rust routes. - if (np?.routes) { - try { - const r = np.routes(rel, content, fw, detectTags(content)); - if (r && r.length) { routes.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "rust", "routes", rel, e); - } - } - if (fw === "actix") { // #[get("/path")], #[post("/path")], etc. const attrPattern = /#\[(get|post|put|patch|delete)\s*\(\s*"([^"]+)"\s*\)\s*\]/gi; diff --git a/src/detectors/schema.ts b/src/detectors/schema.ts index abcddd6..204ccce 100644 --- a/src/detectors/schema.ts +++ b/src/detectors/schema.ts @@ -8,7 +8,6 @@ import { extractEloquentModels } from "../ast/extract-php.js"; import { extractEntityFrameworkModels } from "../ast/extract-csharp.js"; import { extractRoomEntities } from "../ast/extract-android.js"; import type { SchemaModel, SchemaField, ProjectInfo, CodesightConfig } from "../types.js"; -import { resolveNativeAst, nativePluginFor, recordParseError, type NativeAstResolved } from "../ast/native-loader.js"; const AUDIT_FIELDS = new Set([ "createdAt", @@ -25,7 +24,6 @@ export async function detectSchemas( config?: CodesightConfig ): Promise { const models: SchemaModel[] = []; - const native = resolveNativeAst(config?.nativeAst, project.root); for (const orm of project.orms) { switch (orm) { @@ -39,13 +37,13 @@ export async function detectSchemas( models.push(...(await detectTypeORMSchemas(files, project))); break; case "sqlalchemy": - models.push(...(await detectSQLAlchemySchemas(files, project, native))); + models.push(...(await detectSQLAlchemySchemas(files, project))); break; case "gorm": - models.push(...(await detectGORMSchemas(files, project, native))); + models.push(...(await detectGORMSchemas(files, project))); break; case "ent": - models.push(...(await detectEntSchemas(files, project, native))); + models.push(...(await detectEntSchemas(files, project))); break; case "activerecord": models.push(...(await detectActiveRecordSchemas(project))); @@ -54,7 +52,7 @@ export async function detectSchemas( models.push(...(await detectEctoSchemas(files, project))); break; case "django": - models.push(...(await detectDjangoSchemas(files, project, native))); + models.push(...(await detectDjangoSchemas(files, project))); break; case "eloquent": models.push(...(await detectEloquentSchemas(files, project))); @@ -358,27 +356,15 @@ async function detectTypeORMSchemas( // --- SQLAlchemy --- async function detectSQLAlchemySchemas( files: string[], - project: ProjectInfo, - native?: NativeAstResolved + project: ProjectInfo ): Promise { const pyFiles = files.filter((f) => f.endsWith(".py")); const models: SchemaModel[] = []; - const np = native ? nativePluginFor("python", "schemas", native) : null; for (const file of pyFiles) { const content = await readFileSafe(file); const rel = relative(project.root, file); - // Native WASM plugin first (when enabled + present); falls through on miss. - if (np?.schemas) { - try { - const r = np.schemas(rel, content); - if (r && r.length) { models.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "python", "schemas", rel, e); - } - } - // SQLModel: class X(SQLModel, table=True) with typed annotations if (content.includes("SQLModel") && content.includes("table=True")) { const sqlmodelModels = await extractSQLModelAST(rel, content); @@ -450,12 +436,10 @@ async function detectSQLAlchemySchemas( // --- GORM --- async function detectGORMSchemas( files: string[], - _project: ProjectInfo, - native?: NativeAstResolved + _project: ProjectInfo ): Promise { const goFiles = files.filter((f) => f.endsWith(".go")); const models: SchemaModel[] = []; - const np = native ? nativePluginFor("go", "schemas", native) : null; for (const file of goFiles) { const content = await readFileSafe(file); @@ -463,15 +447,6 @@ async function detectGORMSchemas( const rel = relative(_project.root, file); - if (np?.schemas) { - try { - const r = np.schemas(rel, content); - if (r && r.length) { models.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "go", "schemas", rel, e); - } - } - const structModels = extractGORMModelsStructured(rel, content); models.push(...structModels); } @@ -482,14 +457,12 @@ async function detectGORMSchemas( // --- Ent (Go) --- async function detectEntSchemas( files: string[], - _project: ProjectInfo, - native?: NativeAstResolved + _project: ProjectInfo ): Promise { const goFiles = files.filter( (f) => f.endsWith(".go") && (f.includes("/ent/schema/") || f.includes("/schema/")) ); const models: SchemaModel[] = []; - const np = native ? nativePluginFor("go", "schemas", native) : null; for (const file of goFiles) { const content = await readFileSafe(file); @@ -497,15 +470,6 @@ async function detectEntSchemas( const rel = relative(_project.root, file); - if (np?.schemas) { - try { - const r = np.schemas(rel, content); - if (r && r.length) { models.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "go", "schemas", rel, e); - } - } - const structModels = extractEntSchemasStructured(rel, content); models.push(...structModels); } @@ -643,15 +607,13 @@ async function detectActiveRecordSchemas( // --- Django ORM --- async function detectDjangoSchemas( files: string[], - project: ProjectInfo, - native?: NativeAstResolved + project: ProjectInfo ): Promise { // Django models live in models.py or models/ directories const modelFiles = files.filter( (f) => f.endsWith("/models.py") || f.includes("/models/") && f.endsWith(".py") ); const models: SchemaModel[] = []; - const np = native ? nativePluginFor("python", "schemas", native) : null; for (const file of modelFiles) { const content = await readFileSafe(file); @@ -659,15 +621,6 @@ async function detectDjangoSchemas( const rel = relative(project.root, file); - if (np?.schemas) { - try { - const r = np.schemas(rel, content); - if (r && r.length) { models.push(...r); continue; } - } catch (e) { - if (native) recordParseError(native, "python", "schemas", rel, e); - } - } - const astModels = await extractDjangoModelsAST(rel, content); if (astModels && astModels.length > 0) { models.push(...astModels); diff --git a/tests/native-ast.test.ts b/tests/native-ast.test.ts index c4e2321..5fa39b4 100644 --- a/tests/native-ast.test.ts +++ b/tests/native-ast.test.ts @@ -13,12 +13,12 @@ async function loadModules() { await import("../dist/wasm/plugin-host.js"); const { collectFiles, detectProject } = await import("../dist/scanner.js"); const { detectRoutes } = await import("../dist/detectors/routes.js"); - const { detectSchemas } = await import("../dist/detectors/schema.js"); + const { detectNative } = await import("../dist/detectors/native.js"); return { loadConfig, mergeCliConfig, safeParseConfigText, resolveNativeAst, nativeEnabledFor, isStrict, reportNativeDiagnostics, setNativePluginProvider, resetNativePluginProvider, - collectFiles, detectProject, detectRoutes, detectSchemas, + collectFiles, detectProject, detectRoutes, detectNative, }; } @@ -129,48 +129,52 @@ describe("native-ast config resolution", () => { }); }); -describe("native-ast dispatch", () => { +describe("native-ast dispatch (generic pass)", () => { let mods: any; before(async () => { mods = await loadModules(); }); afterEach(() => { mods.resetNativePluginProvider(); }); - it("prefers the native plugin and tags confidence 'native' (routes)", async () => { - const dir = await writeFixture("routes", { "main.py": FASTAPI_FIXTURE, "requirements.txt": "fastapi\n" }); + // Explicit languages target the plugin by name (the mocked provider answers + // loadPlugin) and route .py via the default extension map — no real .wasm. + const pythonExplicit = (mods: any, dir: string) => + mods.resolveNativeAst({ enabled: true, languages: ["python"] }, dir); + + it("dispatches by extension and tags confidence 'native' (routes)", async () => { + const dir = await writeFixture("routes", { "main.py": FASTAPI_FIXTURE }); try { mods.setNativePluginProvider(fakeProvider()); const project = await mods.detectProject(dir); const files = await mods.collectFiles(dir, 10, []); - const routes = await mods.detectRoutes(files, project, { nativeAst: { enabled: true } }); - const native = routes.filter((r: any) => r.confidence === "native"); - assert.ok(native.length > 0, "expected native-tagged routes"); - assert.ok(native.some((r: any) => r.path === "/native"), "expected route from the plugin"); + const { routes } = await mods.detectNative(files, project, pythonExplicit(mods, dir)); + assert.ok(routes.length > 0, "expected native routes"); + assert.ok(routes.every((r: any) => r.confidence === "native")); + assert.ok(routes.some((r: any) => r.path === "/native")); } finally { await rm(dir, { recursive: true, force: true }); } }); - it("prefers the native plugin and tags confidence 'native' (schemas)", async () => { - const dir = await writeFixture("schemas", { "models.py": SQLALCHEMY_FIXTURE, "requirements.txt": "sqlalchemy\n" }); + it("dispatches by extension and tags confidence 'native' (schemas)", async () => { + const dir = await writeFixture("schemas", { "models.py": SQLALCHEMY_FIXTURE }); try { mods.setNativePluginProvider(fakeProvider()); const project = await mods.detectProject(dir); const files = await mods.collectFiles(dir, 10, []); - const schemas = await mods.detectSchemas(files, project, { nativeAst: { enabled: true } }); - const native = schemas.filter((s: any) => s.confidence === "native"); - assert.ok(native.length > 0, "expected native-tagged schemas"); - assert.ok(native.some((s: any) => s.name === "NativeModel")); + const { schemas } = await mods.detectNative(files, project, pythonExplicit(mods, dir)); + assert.ok(schemas.length > 0, "expected native schemas"); + assert.ok(schemas.every((s: any) => s.confidence === "native")); + assert.ok(schemas.some((s: any) => s.name === "NativeModel")); } finally { await rm(dir, { recursive: true, force: true }); } }); - it("regression: with native disabled, no result is tagged 'native'", async () => { + it("regression: with native disabled, built-in extraction is unaffected and untagged", async () => { const dir = await writeFixture("off", { "main.py": FASTAPI_FIXTURE, "requirements.txt": "fastapi\n" }); try { - // no provider installed; no nativeAst config const project = await mods.detectProject(dir); const files = await mods.collectFiles(dir, 10, []); - const routes = await mods.detectRoutes(files, project); + const routes = await mods.detectRoutes(files, project); // no nativeAst config assert.equal(routes.filter((r: any) => r.confidence === "native").length, 0); assert.ok(routes.length > 0, "built-in extraction should still find routes"); } finally { @@ -178,22 +182,19 @@ describe("native-ast dispatch", () => { } }); - it("strict mode collects a diagnostic when the plugin is unavailable", async () => { - const dir = await writeFixture("strict", { "main.py": FASTAPI_FIXTURE, "requirements.txt": "fastapi\n" }); + it("strict mode records an 'unavailable' diagnostic for a named language with no plugin", async () => { + const dir = await writeFixture("strict", { "main.py": FASTAPI_FIXTURE }); try { - // No provider → real loader finds no .wasm → unavailable. - const cfg = { nativeAst: { enabled: "strict" as const } }; + // No provider → real loader finds no codesight-python-ast.wasm → unavailable. + const resolved = mods.resolveNativeAst({ enabled: "strict", languages: ["python"] }, dir); const project = await mods.detectProject(dir); const files = await mods.collectFiles(dir, 10, []); - await mods.detectRoutes(files, project, cfg); - - // Same config object → same memoized resolved instance → shared diagnostics. - const resolved = mods.resolveNativeAst(cfg.nativeAst, project.root); + await mods.detectNative(files, project, resolved); assert.ok( - resolved.diagnostics.some((d: any) => d.lang === "python" && d.kind === "routes" && d.reason === "plugin unavailable"), - `expected a python/routes unavailable diagnostic, got ${JSON.stringify(resolved.diagnostics)}` + resolved.diagnostics.some((d: any) => d.lang === "python" && d.reason === "plugin unavailable"), + `expected a python unavailable diagnostic, got ${JSON.stringify(resolved.diagnostics)}` ); - assert.ok(mods.reportNativeDiagnostics(resolved.diagnostics).includes("python/routes")); + assert.ok(mods.reportNativeDiagnostics(resolved.diagnostics).includes("python")); } finally { await rm(dir, { recursive: true, force: true }); } From c0c5dcc22717338cb3cdf4e354e544df0d0885ae Mon Sep 17 00:00:00 2001 From: Mark S Date: Thu, 25 Jun 2026 09:22:04 -0400 Subject: [PATCH 3/4] test(ast): cover describe() metadata, registry routing, and the generic pass Add a generalization suite to reference-plugin.test.ts: reads describe() metadata off the loaded plugin, asserts buildNativeRegistry routes the declared extension (explicit => authoritative), drives the generic pass over a .ref file, and verifies `all` mode discovers the plugin by globbing the plugin dir (=> additive). --- tests/reference-plugin.test.ts | 57 ++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/tests/reference-plugin.test.ts b/tests/reference-plugin.test.ts index 7183ed7..c401a68 100644 --- a/tests/reference-plugin.test.ts +++ b/tests/reference-plugin.test.ts @@ -1,6 +1,8 @@ import { describe, it, before } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, copyFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; +import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; // ABI conformance test for the reference WASM plugin. @@ -28,9 +30,14 @@ const SRC = [ ].join("\n"); async function loadModules() { - const { loadPlugin, bindExports, resetNativePluginProvider } = await import("../dist/wasm/plugin-host.js"); - const { resolveNativeAst, nativePluginFor } = await import("../dist/ast/native-loader.js"); - return { loadPlugin, bindExports, resetNativePluginProvider, resolveNativeAst, nativePluginFor }; + const { loadPlugin, bindExports, listPluginFiles, resetNativePluginProvider } = await import("../dist/wasm/plugin-host.js"); + const { resolveNativeAst, nativePluginFor, buildNativeRegistry } = await import("../dist/ast/native-loader.js"); + const { detectNative } = await import("../dist/detectors/native.js"); + const { detectProject } = await import("../dist/scanner.js"); + return { + loadPlugin, bindExports, listPluginFiles, resetNativePluginProvider, + resolveNativeAst, nativePluginFor, buildNativeRegistry, detectNative, detectProject, + }; } describe("reference WASM plugin — raw ABI (plugin-host)", () => { @@ -154,3 +161,47 @@ describe("plugin-host — contractVersion gating (bindExports)", () => { assert.equal(plugin, null); }); }); + +describe("native generalization — discovery + generic pass (reference plugin)", () => { + let mods: any; + before(async () => { mods = await loadModules(); mods.resetNativePluginProvider(); }); + + const MARKERS = ["route GET /health", "model User id email", "import ./db"].join("\n"); + + it("reads describe() metadata (languageId + extensions)", () => { + const p = mods.loadPlugin("reference", [PLUGIN_DIR]); + assert.ok(p, "reference plugin should load"); + assert.deepEqual(p.metadata, { languageId: "reference", extensions: [".ref"] }); + }); + + it("buildNativeRegistry routes the declared extension to the plugin (explicit)", () => { + const resolved = mods.resolveNativeAst({ enabled: true, languages: ["reference"], pluginDir: PLUGIN_DIR }, PLUGIN_DIR); + const reg = mods.buildNativeRegistry(resolved); + const entry = reg.byExt.get(".ref"); + assert.ok(entry, "expected .ref → reference in the registry"); + assert.equal(entry.lang, "reference"); + assert.equal(entry.authoritative, true); // explicit list ⇒ authoritative + }); + + it("the generic pass dispatches a .ref file by extension", async () => { + const tmp = mkdtempSync(join(tmpdir(), "cs-gen-")); + writeFileSync(join(tmp, "thing.ref"), MARKERS); + const resolved = mods.resolveNativeAst({ enabled: true, languages: ["reference"], pluginDir: PLUGIN_DIR }, tmp); + const { routes, schemas } = await mods.detectNative([join(tmp, "thing.ref")], { root: tmp }, resolved); + assert.deepEqual(routes.map((r: any) => `${r.method} ${r.path}`), ["GET /health"]); + assert.ok(routes.every((r: any) => r.confidence === "native" && r.framework === "unknown")); + assert.deepEqual(schemas.map((s: any) => s.name), ["User"]); + }); + + it("`all` mode discovers the plugin by globbing the plugin dir", () => { + const tmp = mkdtempSync(join(tmpdir(), "cs-plugins-")); + copyFileSync(join(PLUGIN_DIR, "codesight-reference-ast.wasm"), join(tmp, "codesight-reference-ast.wasm")); + // no `languages` ⇒ all/additive; the override dir is searched first + const resolved = mods.resolveNativeAst({ enabled: true, pluginDir: tmp }, tmp); + const reg = mods.buildNativeRegistry(resolved); + const entry = reg.byExt.get(".ref"); + assert.ok(entry, "all-mode should discover the reference plugin"); + assert.equal(entry.lang, "reference"); + assert.equal(entry.authoritative, false); // all ⇒ additive + }); +}); From 143e1bd730fa13e244d2f13d49e48452c347cbf1 Mon Sep 17 00:00:00 2001 From: Mark S Date: Thu, 25 Jun 2026 09:22:04 -0400 Subject: [PATCH 4/4] docs: document language-driven dispatch, describe(), and WASI instantiation Rework the plugin contract for the generalization: any user-declared language, describe()-driven discovery (id/extensions, template guard, precedence/collision), all/none/=ids enabling with additive-vs-authoritative merge, and the WASI instantiation row in the ABI. --- docs/wasm-plugins.md | 95 ++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/docs/wasm-plugins.md b/docs/wasm-plugins.md index 23afa4f..57819fc 100644 --- a/docs/wasm-plugins.md +++ b/docs/wasm-plugins.md @@ -39,50 +39,64 @@ The module is instantiated once per scan and its `parse*` functions are called m times (it's a long-lived "reactor", not a per-file process). Compile a library, not a command. -> **Scope:** `codesight` invokes a plugin only at its own extraction points and only -> for a language identifier it recognizes. It provides no WASM-based parsers itself. -> Where no plugin is configured, it falls back to its built-in extraction. This contract -> defines the boundary; it does not promise any particular language is wired up. +> **Scope:** `codesight` ships no parsers itself. Dispatch is **language-driven**: a +> plugin declares the file extensions it handles (via `describe()`, below), and +> `codesight` routes matching files to it — so *any* language works, not just the ones +> with built-in extractors. Where no plugin handles a file, built-in extraction stands. --- ## Discovery & naming -A plugin is identified by a language identifier ``. Per identifier, -`codesight` looks for a single self-describing module — no sidecar manifest: +Plugin binaries must match the template (anything else on the path is ignored): ``` -codesight--ast.wasm # the module (required) +codesight--ast.wasm # `` ∈ [a-z0-9_-]+ ; the `-ast` capability namespace is reserved ``` -The `-ast` segment is the capability namespace (reserved to allow the same plugin -mechanism to host other capabilities later if/when it becomes beneficial or desirable -to do so). The module declares its own contract version and capabilities through -its exports (see below), so there is nothing else to ship or keep in sync. +The module is **fully self-describing** — no sidecar manifest. Its `describe()` +export (below) declares the authoritative `languageId` and the file `extensions` +it parses; the `` in the filename is only a discovery key + fallback id (if +`describe().languageId` is set and differs, the declared id wins, with a warning). +For the built-in language ids (`rust`/`go`/`python`) a default extension map +(`.rs`/`.go`/`.py`) applies when `describe()` is omitted; **any other language must +declare `extensions`** or it can't be routed. -Directories are searched in this order; the **first** directory containing a -matching `.wasm` binary wins (\*nix `PATH`-style waterfall): +Directories are searched in PATH-style waterfall order: 1. `--plugin-dir ` / `CODESIGHT_PLUGIN_DIR` (relative paths resolve against the project root) 2. `~/.codesight/plugins` 3. `${XDG_DATA_HOME:-~/.local/share}/codesight/plugins` 4. `/plugins` +**Precedence:** per language id, the first dir wins (lower dirs shadowed). If two +*different* languages claim the same extension, an explicitly-named language beats +an `all`-discovered one, else first-registered wins — with a warning either way. + --- ## Enabling native parsing Native parsing is off unless explicitly enabled. Precedence is `CLI` > `env` > `config file`. -| Mechanism | Example | -|-----------|---------------------------------------------------------------------------------------------------------------------------------| -| CLI | `codesight --native-ast` · `codesight --native-ast ` · `codesight --native-ast-strict` · `codesight --plugin-dir ./wasm` | -| Env | `CODESIGHT_NATIVE_AST=1` (or `strict`, or a comma list of language identifiers) · `CODESIGHT_PLUGIN_DIR=/path` | -| Config | `codesight.config.{json,js,ts}` → `{ "nativeAst": { "enabled": true, "languages": [""], "pluginDir": "./wasm" } }` | - -`enabled` may be `true`, `false`, or `"strict"`. Omitting `languages` (or passing -an empty list) enables every language identifier `codesight` recognizes; supplying a -list restricts native parsing to those identifiers. +| Mechanism | Example | +|-----------|------------------------------------------------------------------------------------------------------------------------------------| +| CLI | `codesight --native-ast` (= all) · `codesight --native-ast=rust,go` · `codesight --native-ast=none` · `codesight --native-ast-strict` · `codesight --plugin-dir ./wasm` | +| Env | `CODESIGHT_NATIVE_AST=all` (or `1`/`true`/`strict`/`none`, or a comma list of ids) · `CODESIGHT_PLUGIN_DIR=/path` | +| Config | `codesight.config.{json,js,ts}` → `{ "nativeAst": { "enabled": true, "languages": ["rust"], "pluginDir": "./wasm" } }` | + +`enabled` may be `true`, `false`, or `"strict"`. `none` forces off (overriding the +config file). **Dispatch mode depends on how you enable it:** + +- **`all` / bare flag / empty `languages`** → *additive*: every discovered plugin is + consulted, and results are **unioned** with the built-in extractors (native wins on + a `method:path` / model-name conflict). +- **An explicit language list** (`--native-ast=rust,go`) → *authoritative*: for files + a named plugin handles, its results **replace** the built-in routes from that file + (dropping regex over-matches). If the plugin returns nothing for a file the built-in + did extract, the built-in stands and a warning is emitted. (Authoritative replacement + is route-only — `SchemaModel` carries no file provenance, so schemas are always + native-preferred dedup-by-name.) --- @@ -90,15 +104,13 @@ list restricts native parsing to those identifiers. The host instantiates every module with a **minimal WASI import object** (Node's built-in `node:wasi` — clock/random/exit/stderr only; **no filesystem, no network, -no env/args**). Pure-compute modules with no imports (Rust/AssemblyScript on -`wasm32-unknown-unknown`) ignore it; modules whose runtime needs WASI (e.g. Go -`//go:wasmexport` reactors) get exactly those minimal capabilities. A module that -exports `_initialize` is initialized before any other export is called. The module -must not require JS-binding glue or host functions beyond WASI. `node:wasi` is built -in, so codesight keeps its zero-dependency guarantee. +no env/args**). Pure-compute modules (Rust/AssemblyScript, no imports) ignore it; +modules whose runtime needs WASI (e.g. Go `//go:wasmexport` reactors) get exactly +those minimal capabilities. A reactor that exports `_initialize` is initialized +before its other exports are called. "Plugins are pure compute" — if a kind ever +needs project context, it flows through the ABI, not the filesystem. -A conforming module exports a small fixed core plus one optional `parse*` function -per capability it provides: +A conforming module exports a small fixed core plus optional capability functions: | Export | Signature (wasm types) | Purpose | |-------------------|---------------------------------------|-------------------------------------------------| @@ -106,6 +118,7 @@ per capability it provides: | `alloc` | `(len: i32) -> i32` | reserve `len` bytes, return a pointer | | `dealloc` | `(ptr: i32, len: i32) -> ()` | release a prior allocation | | `contractVersion` | `() -> i32` | the contract version this plugin implements | +| `describe` | `() -> i64` | *optional* — packed JSON metadata (see below) | | `parseRoutes` | `(srcPtr: i32, srcLen: i32) -> i64` | *optional* — extract routes | | `parseSchemas` | `(srcPtr: i32, srcLen: i32) -> i64` | *optional* — extract schema models | | `parseImports` | `(srcPtr: i32, srcLen: i32) -> i64` | *optional* — extract imports (defined, not yet dispatched — see below) | @@ -116,6 +129,20 @@ also makes `contractVersion` a "this is a codesight plugin" marker. **Capability detected by export presence:** a plugin supports a kind iff it exports the matching `parse*` function. There are no kind codes and no manifest. +### `describe()` — self-description + +Optional. Returns packed `(outPtr << 32) | outLen` pointing at UTF-8 JSON (same +convention as `parse*`), e.g.: + +```jsonc +{ "languageId": "ruby", "extensions": [".rb", ".rake"] } +``` + +The host reads `languageId` (authoritative over the filename) and `extensions` (how +files are routed to this plugin); other fields are carried but unused for now. A +plugin for a non-built-in language **must** declare `extensions` here, or it has no +files to receive. + As WebAssembly text: ```wat @@ -286,10 +313,10 @@ from built-in `ast`/`regex` results in the scan summary. Implement the module in any toolchain that targets `wasm32` and imports **at most `wasi_snapshot_preview1`** (no JS-binding glue, no other host functions). Pure-compute languages (Rust/AssemblyScript via `wasm32-unknown-unknown`) need no imports at all; -full-runtime languages (Go via `GOOS=wasip1 -buildmode=c-shared`) import WASI, which -the host supplies minimally. The allocator and the per-kind marshalling are -boilerplate; the only part that changes is your extraction logic. Export only the -`parse*` functions for the kinds you support. +full-runtime languages (Go via `GOOS=wasip1 -buildmode=c-shared` + `//go:wasmexport`) +import WASI, which the host supplies minimally. The allocator and per-kind marshalling +are boilerplate; only your extraction logic changes. Export `describe()` (for routing) +plus the `parse*` functions for the kinds you support. Required exports and their behavior, in pseudocode: