chore: add load options

README.md

@@ -61,8 +61,8 @@ await ctx.close();
 | `basePath` | `string` | `process.cwd()` | Base path for resolving document IDs. Set for cross-machine consistent IDs. |
 | `onProgress` | `(phase, detail) => void` | — | Progress callback for `load()` phases: `'load'` → `'embed'` → `'insert'`. |
 | `queryExpansion` | `QueryExpansionOptions` | `false`  | Query expansion with user-provided synonym map. false disables. Without synonyms, expansion is a no-op. |
-| `ftsFields` | `string[]` | `['content']` | Fields to index for Full Text Search in hybrid mode |
-| `ftsFieldWeights` | `Record<string, number>` | `{ content: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. |
+| `ftsFields` | `string[]` | `['indexContent']` | Fields queried by Full Text Search in hybrid mode |
+| `ftsFieldWeights` | `Record<string, number>` | `{ indexContent: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. |
 | `rankConstant` | `number` | `60` | RRF rank constant for hybrid search fusion. Lower = "winner-takes-all", higher = more even. |
 
 #### Weight Configuration Example
@@ -114,7 +114,7 @@ const results = await ctx.query('How to configure a line chart', { library: 'g2'
 
 In read-only mode, `Context` opens existing `${library}.zvec` files with `ZVecOpen`. It does not create missing stores, and `load()` will throw because it would mutate the zvec file.
 
-### `ctx.load(library, pattern)`
+### `ctx.load(library, pattern, options?)`
 
 Load files into a specified library with automatic batch vectorization. Documents are embedded in batches and inserted into the vector store. A content-hash change detection mechanism re-embeds files whose content has changed since the last load.
 
@@ -124,12 +124,26 @@ Document IDs are derived from file paths relative to `basePath` for cross-machin
 |-----------|------|-------------|
 | `library` | `string` | Library name for organizing documents |
 | `pattern` | `string \| string[]` | Glob pattern(s) matching files to load |
+| `options.buildIndexContent` | `(document, context) => string \| Promise<string>` | Build derived text used for embedding and the `indexContent` FTS field while preserving the original returned content |
 
 ```typescript
 await ctx.load('g2', './docs/**/*.md');
 await ctx.load('g2', ['./docs/**/*.md', './docs/**/*.json']);
+
+await ctx.load('g2', './docs/**/*.md', {
+  buildIndexContent: (doc) => [
+    doc.meta?.title,
+    doc.meta?.description,
+    ...(Array.isArray(doc.meta?.tags) ? doc.meta.tags : []),
+    doc.content.slice(0, 500),
+  ].filter(Boolean).join('\n'),
+});
 ```
 
+When `buildIndexContent` is provided, its result is used for embedding and the
+default `indexContent` FTS query. The original `document.content` is stored
+unchanged and returned by `query()`.
+
 Load phases emit progress via the `onProgress` callback:
 
 ```typescript

package.json

@@ -1,11 +1,11 @@
 {
   "name": "@antv/context",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. ",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "build": "tsc",
+    "build": "rm -rf dist && tsc",
     "test": "HF_ENDPOINT=https://hf-mirror.com vitest run --coverage",
     "lint": "eslint src/ --ext .ts",
     "lint:fix": "eslint src/ --ext .ts --fix",

src/context.ts

@@ -6,6 +6,7 @@ import {
   QueryOptions,
   QueryResult,
   LoadedDoc,
+  LoadOptions,
 } from './types';
 import { Embedder } from './embedder';
 
@@ -54,7 +55,7 @@ export class Context {
     return new Context(options, embedder, { dimensions: embedder.dimensions });
   }
 
-  async load(library: string, pattern: string | string[]): Promise<void> {
+  async load(library: string, pattern: string | string[], options: LoadOptions = {}): Promise<void> {
     const patterns = Array.isArray(pattern) ? pattern : [pattern];
     const files = await glob(patterns, { absolute: true });
 
@@ -69,10 +70,14 @@ export class Context {
 
         const doc = await loader.load(filePath);
         const relativePath = path.relative(this.options.basePath!, filePath);
+        const indexContent = options.buildIndexContent
+          ? await options.buildIndexContent(doc, { library, filePath, relativePath })
+          : doc.content;
         return {
           ...doc,
+          indexContent,
           id: pathToId(relativePath),
-          contentHash: computeContentHash(doc.content),
+          contentHash: computeContentHash(`${doc.content}\n\0${indexContent}`),
           path: relativePath,
         };
       }),
@@ -91,7 +96,7 @@ export class Context {
 
     if (docsToEmbed.length === 0) return;
 
-    const contents = docsToEmbed.map((doc) => doc.content);
+    const contents = docsToEmbed.map((doc) => doc.indexContent);
     const vectors = await this.embedder.embedBatch(contents);
 
     if (this.options.onProgress) {
@@ -103,6 +108,7 @@ export class Context {
       vector: vectors[index],
       fields: {
         content: doc.content,
+        indexContent: doc.indexContent,
         meta: doc.meta ? JSON.stringify(doc.meta) : '',
         path: doc.path,
         contentHash: doc.contentHash,

src/storage/schema.ts

@@ -19,9 +19,23 @@ const INDEX_TYPES = {
   HNSW: ZVecIndexType.HNSW,
 };
 
-export function buildZvecSchema(dims: number, tokenizerName: string = 'jieba'): ZVecCollectionSchema {
+export function buildZvecSchema(
+  dims: number,
+  tokenizerName: string = 'jieba',
+  ftsFields: string[] = ['indexContent'],
+): ZVecCollectionSchema {
+  const fts = (name: string): Pick<ZvecFieldSchema, 'indexType' | 'indexOptions'> =>
+    ftsFields.includes(name)
+      ? { indexType: 'FTS', indexOptions: { tokenizerName } }
+      : {};
   const fields: ZvecFieldSchema[] = [
-    { name: 'content', dataType: 'STRING', indexType: 'FTS', indexOptions: { tokenizerName } },
+    { name: 'content', dataType: 'STRING', ...fts('content') },
+    {
+      name: 'indexContent',
+      dataType: 'STRING',
+      indexType: 'FTS',
+      indexOptions: { tokenizerName },
+    },
     { name: 'meta', dataType: 'STRING' },
     { name: 'path', dataType: 'STRING' },
     { name: 'contentHash', dataType: 'STRING' },
@@ -48,4 +62,4 @@ export function buildZvecSchema(dims: number, tokenizerName: string = 'jieba'):
         : {}),
     })) as never,
   }) as ZVecCollectionSchema;
-}
+}

src/storage/store.ts

@@ -8,7 +8,7 @@ import { Embedder } from '../embedder';
 import type { ContextOptions } from '../types';
 
 const VECTOR_FIELD = 'embedding';
-const FTS_FIELDS = ['content'];
+const FTS_FIELDS = ['indexContent'];
 
 /**
  * Store - zvec storage for multiple libraries.
@@ -46,7 +46,11 @@ export class Store {
     if (fs.existsSync(filePath)) {
       collection = ZVecOpen(filePath, { readOnly: this.options.readOnly });
     } else {
-      const schema = buildZvecSchema(this.embedder.dimensions, tokenizerName);
+      const schema = buildZvecSchema(
+        this.embedder.dimensions,
+        tokenizerName,
+        this.options.ftsFields,
+      );
       collection = ZVecCreateAndOpen(filePath, schema, { readOnly: this.options.readOnly });
     }
 
@@ -63,7 +67,10 @@ export class Store {
     const records = docs.map((d) => ({
       id: d.id,
       vectors: { [opts.vectorField]: d.vector },
-      fields: d.fields,
+      fields: {
+        ...d.fields,
+        indexContent: d.fields.indexContent ?? d.fields.content ?? '',
+      },
     }));
     collection.upsertSync(records);
   }

src/types.ts

@@ -14,6 +14,8 @@ export interface Document {
  * Document with computed fields for loading.
  */
 export interface LoadedDoc extends Document {
+  /** Derived text used for embedding and optional FTS indexing. */
+  indexContent: string;
   /** Document ID */
   id: string;
   /** Hash of document content */
@@ -22,6 +24,17 @@ export interface LoadedDoc extends Document {
   path: string;
 }
 
+export interface LoadOptions {
+  /**
+   * Build focused text for embedding and FTS while preserving the original
+   * document content returned by query(). Defaults to document.content.
+   */
+  buildIndexContent?: (
+    document: Document,
+    context: { library: string; filePath: string; relativePath: string },
+  ) => string | Promise<string>;
+}
+
 /** Configuration for query expansion. */
 export interface QueryExpansionOptions {
   /**
@@ -106,8 +119,8 @@ export interface ContextOptions {
   /**
    * Fields to index for Full Text Search (FTS).
    *
-   * By default, `content` is indexed. Add more fields (e.g. `meta.title`)
-   * to include metadata in text-based recall.
+   * By default, hybrid search queries `indexContent`, which falls back to
+   * the original `content` when no buildIndexContent callback is provided.
    *
    * Only affects newly created stores — existing stores keep their schema.
    */
@@ -120,7 +133,7 @@ export interface ContextOptions {
    * Example: `{ content: 1, title: 3 }` makes title matches 3× more
    * influential than content matches.
    *
-   * Defaults to `{ content: 1 }` when `ftsFields` includes `content`.
+   * Defaults to `{ indexContent: 1 }`.
    */
   ftsFieldWeights?: Record<string, number>;
 

test/context.test.ts

@@ -90,6 +90,34 @@ describe('Context', () => {
       expect(results[0].meta).toHaveProperty('title');
     });
 
+    it('should index derived content while returning original content', async () => {
+      const indexDir = TEST_DIR + '-index-content';
+      const indexContext = await Context.create({
+        vectorsDir: indexDir,
+        ftsFields: ['indexContent'],
+      });
+
+      await indexContext.load(
+        'index-content',
+        path.join(FIXTURES_DIR, 'getting-started.md'),
+        {
+          buildIndexContent: (doc) => `unique-index-marker ${String(doc.meta?.title ?? '')}`,
+        },
+      );
+
+      const results = await indexContext.query('unique-index-marker', {
+        library: 'index-content',
+        topK: 1,
+        rerank: false,
+      });
+      expect(results).toHaveLength(1);
+      expect(results[0].content).toContain('npm');
+      expect(results[0].content).not.toContain('unique-index-marker');
+
+      await indexContext.close();
+      fs.rmSync(indexDir, { recursive: true, force: true });
+    });
+
     it('should skip already loaded documents (deduplication)', async () => {
       await ctx.load('md', path.join(FIXTURES_DIR, 'getting-started.md'));
 
@@ -398,4 +426,4 @@ describe('Context two-phase separation', () => {
 
     await ctx3.close();
   });
-});
+});

test/storage/schema.test.ts

@@ -17,15 +17,23 @@ describe('buildZvecSchema', () => {
     expect(schemaStr).toContain('metric:COSINE');
   });
 
-  it('should include content field with FTS index and jieba tokenizer (default)', () => {
+  it('should include indexContent with FTS and jieba tokenizer by default', () => {
     const schema = buildZvecSchema(128, 'jieba');
     const schemaStr = String(schema);
 
-    expect(schemaStr).toContain("name: 'content'");
+    expect(schemaStr).toContain("name: 'indexContent'");
     expect(schemaStr).toContain('FtsIndexParams');
     expect(schemaStr).toContain('tokenizer_name:jieba');
   });
 
+  it('should allow content to be additionally indexed', () => {
+    const schema = buildZvecSchema(128, 'jieba', ['content']);
+    const schemaStr = String(schema);
+
+    expect(schemaStr).toContain("name: 'content'");
+    expect(schemaStr).toContain('FtsIndexParams');
+  });
+
   it('should accept custom tokenizer (standard)', () => {
     const schema = buildZvecSchema(128, 'standard');
     const schemaStr = String(schema);
@@ -48,4 +56,4 @@ describe('buildZvecSchema', () => {
 
     expect(schemaStr).toContain('data_type: VECTOR_FP32');
   });
-});
+});

test/storage/store.test.ts

@@ -98,8 +98,9 @@ describe('Store', () => {
       ];
       store.addDoc('test-lib', docs);
 
-      const result = store.fetchDocs('test-lib', [docId], ['content']);
+      const result = store.fetchDocs('test-lib', [docId], ['content', 'indexContent']);
       expect(result).toHaveProperty(docId);
+      expect(result[docId].fields.indexContent).toBe('test content');
     });
   });