refactor(content): replace content-scanner + content-manager with content-pipeline

.changeset/content-pipeline-redesign.md

@@ -0,0 +1,15 @@
+---
+"@statewalker/content-pipeline": minor
+"@statewalker/content-cli": patch
+---
+
+**BREAKING:** removed `@statewalker/content-scanner` and `@statewalker/content-manager`, replaced by a single new package `@statewalker/content-pipeline`.
+
+The new package implements the same cascade (files → extract → split → embed → fts/vec index) as a set of layered Trackers — each with a persisted cursor, monotonic integer stamps, batched pacing, runtime cascade via `onStampUpdate`, and tombstone propagation — wiring ~2450 LOC of scanner + manager infrastructure down to ~585 LOC. Three interchangeable `Store<E>` backends (`JsonManifestStore`, `BlobStore` with pluggable codecs including a raw Float32 fast-path for embeddings, and an optional day-2 `SqlStore`) let each layer pick the right persistence for its payload profile.
+
+**Migration:**
+- Replace `@statewalker/content-manager` / `@statewalker/content-scanner` imports with `@statewalker/content-pipeline`.
+- `createContentManager` options change: drop `registry: FilesScanRegistry`, add `statePrefix: string` (directory under which the pipeline stores its state). Everything else (`indexer`, `files`, `extractors`, `chunkOptions`, `embed`, `root`, `filter`) is unchanged. The `sync` / `search` / `status` / `clear` / `close` public surface is preserved.
+- First run after upgrading rebuilds state from scratch; the on-disk store layout is not compatible with the old one.
+
+See `openspec/changes/content-pipeline-redesign/` in the umbrella for the full proposal, design notes, and spec deltas.

README.md

@@ -10,8 +10,7 @@ Content pipeline: blocks, extractors, scanners, managers, plus the content-cli.
 | --- | --- |
 | [@statewalker/content-blocks](packages/content-blocks) | Block types shared across the content pipeline. |
 | [@statewalker/content-extractors](packages/content-extractors) | PDF/DOCX/XLSX/Markdown/HTML extractors. |
-| [@statewalker/content-scanner](packages/content-scanner) | Scans a file tree and streams blocks into indexers. |
-| [@statewalker/content-manager](packages/content-manager) | High-level scan + index orchestration. |
+| [@statewalker/content-pipeline](packages/content-pipeline) | Layered trackers that cascade file-system changes through extract, split, embed, and index stages. |
 
 ## Apps
 

apps/content-cli/package.json

@@ -32,14 +32,14 @@
   },
   "dependencies": {
     "@statewalker/content-extractors": "workspace:*",
-    "@statewalker/content-manager": "workspace:*",
-    "@statewalker/content-scanner": "workspace:*",
+    "@statewalker/content-pipeline": "workspace:*",
     "@statewalker/indexer-api": "catalog:",
     "@statewalker/indexer-mem-flexsearch": "catalog:",
     "@statewalker/webrun-files": "catalog:",
     "@statewalker/webrun-files-node": "catalog:"
   },
   "devDependencies": {
+    "@types/node": "catalog:",
     "rimraf": "catalog:",
     "tsdown": "catalog:",
     "tsx": "catalog:",

apps/content-cli/src/cli.ts

@@ -2,8 +2,7 @@
 
 import { resolve } from "node:path";
 import { createDefaultRegistry } from "@statewalker/content-extractors/extractors";
-import { createContentManager } from "@statewalker/content-manager";
-import { FilesScanRegistry } from "@statewalker/content-scanner";
+import { createContentManager } from "@statewalker/content-pipeline";
 import type { IndexerPersistence, PersistenceEntry } from "@statewalker/indexer-api";
 import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch";
 import type { FilesApi } from "@statewalker/webrun-files";
@@ -112,18 +111,17 @@ async function main() {
   const rootDir = resolve(rootPath);
   const files = new NodeFilesApi({ rootDir });
   const indexDir = `/${systemFolder}/indexer`;
-  const scanDir = `/${systemFolder}/scan`;
+  const statePrefix = `/${systemFolder}/content`;
 
   const persistence = createFilePersistence(files, indexDir);
   const indexer = createFlexSearchIndexer({ persistence });
-  const registry = new FilesScanRegistry({ files, prefix: scanDir });
   const extractors = createDefaultRegistry();
 
   const manager = createContentManager({
-    registry,
     indexer,
     files,
     extractors,
+    statePrefix,
     root: "/",
     filter: (path: string) => !path.startsWith(`/${systemFolder}/`),
   });

apps/content-cli/tsconfig.json

@@ -3,7 +3,8 @@
     "target": "ES2022",
     "module": "Preserve",
     "moduleResolution": "Bundler",
-    "lib": ["ESNext"],
+    "lib": ["ESNext", "DOM"],
+    "types": ["node"],
     "strict": true,
     "skipLibCheck": true,
     "verbatimModuleSyntax": true,

packages/content-blocks/README.md

@@ -23,4 +23,4 @@ For stable block ID generation use `@statewalker/shared-ids` directly.
 
 ## Related
 
-- `@statewalker/content-extractors`, `@statewalker/content-scanner`, `@statewalker/content-manager`.
+- `@statewalker/content-extractors`, `@statewalker/content-pipeline`.