From 23f20e9626f76dc9a1958ef6c213c0c2d7a6d043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A6=8F=E6=99=8B?= Date: Thu, 30 Jul 2026 15:19:28 +0800 Subject: [PATCH 1/4] chore: add load options --- README.md | 16 +++++++++++++++- package.json | 4 ++-- src/context.ts | 12 +++++++++--- src/storage/schema.ts | 15 ++++++++++++--- src/storage/store.ts | 6 +++++- src/types.ts | 13 +++++++++++++ test/context.test.ts | 30 +++++++++++++++++++++++++++++- test/storage/schema.test.ts | 10 +++++++++- 8 files changed, 94 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7d72e54..ddae587 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ const results = await ctx.query('How to configure a line chart', { library: 'g2' In read-only mode, `Context` opens existing `${library}.zvec` files with `ZVecOpen`. It does not create missing stores, and `load()` will throw because it would mutate the zvec file. -### `ctx.load(library, pattern)` +### `ctx.load(library, pattern, options?)` Load files into a specified library with automatic batch vectorization. Documents are embedded in batches and inserted into the vector store. A content-hash change detection mechanism re-embeds files whose content has changed since the last load. @@ -124,12 +124,26 @@ Document IDs are derived from file paths relative to `basePath` for cross-machin |-----------|------|-------------| | `library` | `string` | Library name for organizing documents | | `pattern` | `string \| string[]` | Glob pattern(s) matching files to load | +| `options.buildIndexContent` | `(document, context) => string \| Promise` | Build derived text used for embedding and the `indexContent` FTS field while preserving the original returned content | ```typescript await ctx.load('g2', './docs/**/*.md'); await ctx.load('g2', ['./docs/**/*.md', './docs/**/*.json']); + +await ctx.load('g2', './docs/**/*.md', { + buildIndexContent: (doc) => [ + doc.meta?.title, + doc.meta?.description, + ...(Array.isArray(doc.meta?.tags) ? doc.meta.tags : []), + doc.content.slice(0, 500), + ].filter(Boolean).join('\n'), +}); ``` +When `buildIndexContent` is provided, its result is used for embedding and can +be queried by configuring `ftsFields: ['indexContent']`. The original +`document.content` is stored unchanged and returned by `query()`. + Load phases emit progress via the `onProgress` callback: ```typescript diff --git a/package.json b/package.json index 5660ac2..bb34dc2 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { "name": "@antv/context", - "version": "0.1.1", + "version": "0.1.2", "description": "A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. ", "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { - "build": "tsc", + "build": "rm -rf dist && tsc", "test": "HF_ENDPOINT=https://hf-mirror.com vitest run --coverage", "lint": "eslint src/ --ext .ts", "lint:fix": "eslint src/ --ext .ts --fix", diff --git a/src/context.ts b/src/context.ts index ad059c9..1dffd33 100644 --- a/src/context.ts +++ b/src/context.ts @@ -6,6 +6,7 @@ import { QueryOptions, QueryResult, LoadedDoc, + LoadOptions, } from './types'; import { Embedder } from './embedder'; @@ -54,7 +55,7 @@ export class Context { return new Context(options, embedder, { dimensions: embedder.dimensions }); } - async load(library: string, pattern: string | string[]): Promise { + async load(library: string, pattern: string | string[], options: LoadOptions = {}): Promise { const patterns = Array.isArray(pattern) ? pattern : [pattern]; const files = await glob(patterns, { absolute: true }); @@ -69,10 +70,14 @@ export class Context { const doc = await loader.load(filePath); const relativePath = path.relative(this.options.basePath!, filePath); + const indexContent = options.buildIndexContent + ? await options.buildIndexContent(doc, { library, filePath, relativePath }) + : doc.content; return { ...doc, + indexContent, id: pathToId(relativePath), - contentHash: computeContentHash(doc.content), + contentHash: computeContentHash(`${doc.content}\n\0${indexContent}`), path: relativePath, }; }), @@ -91,7 +96,7 @@ export class Context { if (docsToEmbed.length === 0) return; - const contents = docsToEmbed.map((doc) => doc.content); + const contents = docsToEmbed.map((doc) => doc.indexContent); const vectors = await this.embedder.embedBatch(contents); if (this.options.onProgress) { @@ -103,6 +108,7 @@ export class Context { vector: vectors[index], fields: { content: doc.content, + indexContent: doc.indexContent, meta: doc.meta ? JSON.stringify(doc.meta) : '', path: doc.path, contentHash: doc.contentHash, diff --git a/src/storage/schema.ts b/src/storage/schema.ts index cb3f816..462cda7 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -19,9 +19,18 @@ const INDEX_TYPES = { HNSW: ZVecIndexType.HNSW, }; -export function buildZvecSchema(dims: number, tokenizerName: string = 'jieba'): ZVecCollectionSchema { +export function buildZvecSchema( + dims: number, + tokenizerName: string = 'jieba', + ftsFields: string[] = ['content'], +): ZVecCollectionSchema { + const fts = (name: string): Pick => + ftsFields.includes(name) + ? { indexType: 'FTS', indexOptions: { tokenizerName } } + : {}; const fields: ZvecFieldSchema[] = [ - { name: 'content', dataType: 'STRING', indexType: 'FTS', indexOptions: { tokenizerName } }, + { name: 'content', dataType: 'STRING', ...fts('content') }, + { name: 'indexContent', dataType: 'STRING', ...fts('indexContent') }, { name: 'meta', dataType: 'STRING' }, { name: 'path', dataType: 'STRING' }, { name: 'contentHash', dataType: 'STRING' }, @@ -48,4 +57,4 @@ export function buildZvecSchema(dims: number, tokenizerName: string = 'jieba'): : {}), })) as never, }) as ZVecCollectionSchema; -} \ No newline at end of file +} diff --git a/src/storage/store.ts b/src/storage/store.ts index dbadcab..5896324 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -46,7 +46,11 @@ export class Store { if (fs.existsSync(filePath)) { collection = ZVecOpen(filePath, { readOnly: this.options.readOnly }); } else { - const schema = buildZvecSchema(this.embedder.dimensions, tokenizerName); + const schema = buildZvecSchema( + this.embedder.dimensions, + tokenizerName, + this.options.ftsFields, + ); collection = ZVecCreateAndOpen(filePath, schema, { readOnly: this.options.readOnly }); } diff --git a/src/types.ts b/src/types.ts index 1c70207..a8d1404 100644 --- a/src/types.ts +++ b/src/types.ts @@ -14,6 +14,8 @@ export interface Document { * Document with computed fields for loading. */ export interface LoadedDoc extends Document { + /** Derived text used for embedding and optional FTS indexing. */ + indexContent: string; /** Document ID */ id: string; /** Hash of document content */ @@ -22,6 +24,17 @@ export interface LoadedDoc extends Document { path: string; } +export interface LoadOptions { + /** + * Build focused text for embedding and FTS while preserving the original + * document content returned by query(). Defaults to document.content. + */ + buildIndexContent?: ( + document: Document, + context: { library: string; filePath: string; relativePath: string }, + ) => string | Promise; +} + /** Configuration for query expansion. */ export interface QueryExpansionOptions { /** diff --git a/test/context.test.ts b/test/context.test.ts index a46e8f8..ec2970e 100644 --- a/test/context.test.ts +++ b/test/context.test.ts @@ -90,6 +90,34 @@ describe('Context', () => { expect(results[0].meta).toHaveProperty('title'); }); + it('should index derived content while returning original content', async () => { + const indexDir = TEST_DIR + '-index-content'; + const indexContext = await Context.create({ + vectorsDir: indexDir, + ftsFields: ['indexContent'], + }); + + await indexContext.load( + 'index-content', + path.join(FIXTURES_DIR, 'getting-started.md'), + { + buildIndexContent: (doc) => `unique-index-marker ${String(doc.meta?.title ?? '')}`, + }, + ); + + const results = await indexContext.query('unique-index-marker', { + library: 'index-content', + topK: 1, + rerank: false, + }); + expect(results).toHaveLength(1); + expect(results[0].content).toContain('npm'); + expect(results[0].content).not.toContain('unique-index-marker'); + + await indexContext.close(); + fs.rmSync(indexDir, { recursive: true, force: true }); + }); + it('should skip already loaded documents (deduplication)', async () => { await ctx.load('md', path.join(FIXTURES_DIR, 'getting-started.md')); @@ -398,4 +426,4 @@ describe('Context two-phase separation', () => { await ctx3.close(); }); -}); \ No newline at end of file +}); diff --git a/test/storage/schema.test.ts b/test/storage/schema.test.ts index 0088f87..284c9b7 100644 --- a/test/storage/schema.test.ts +++ b/test/storage/schema.test.ts @@ -26,6 +26,14 @@ describe('buildZvecSchema', () => { expect(schemaStr).toContain('tokenizer_name:jieba'); }); + it('should include indexContent with an FTS index', () => { + const schema = buildZvecSchema(128, 'jieba', ['indexContent']); + const schemaStr = String(schema); + + expect(schemaStr).toContain("name: 'indexContent'"); + expect(schemaStr).toContain('FtsIndexParams'); + }); + it('should accept custom tokenizer (standard)', () => { const schema = buildZvecSchema(128, 'standard'); const schemaStr = String(schema); @@ -48,4 +56,4 @@ describe('buildZvecSchema', () => { expect(schemaStr).toContain('data_type: VECTOR_FP32'); }); -}); \ No newline at end of file +}); From f2c6354dd621d981e87e926372538d3734f6ec5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A6=8F=E6=99=8B?= Date: Thu, 30 Jul 2026 17:53:12 +0800 Subject: [PATCH 2/4] fix: test --- src/storage/store.ts | 5 ++++- test/storage/store.test.ts | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index 5896324..916bb80 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -67,7 +67,10 @@ export class Store { const records = docs.map((d) => ({ id: d.id, vectors: { [opts.vectorField]: d.vector }, - fields: d.fields, + fields: { + ...d.fields, + indexContent: d.fields.indexContent ?? d.fields.content ?? '', + }, })); collection.upsertSync(records); } diff --git a/test/storage/store.test.ts b/test/storage/store.test.ts index 5840d7d..73ac671 100644 --- a/test/storage/store.test.ts +++ b/test/storage/store.test.ts @@ -98,8 +98,9 @@ describe('Store', () => { ]; store.addDoc('test-lib', docs); - const result = store.fetchDocs('test-lib', [docId], ['content']); + const result = store.fetchDocs('test-lib', [docId], ['content', 'indexContent']); expect(result).toHaveProperty(docId); + expect(result[docId].fields.indexContent).toBe('test content'); }); }); From f846ecf621e766e75f422f101d4a19707b3f664e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A6=8F=E6=99=8B?= Date: Thu, 30 Jul 2026 18:01:10 +0800 Subject: [PATCH 3/4] chore: update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb34dc2..087fe14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@antv/context", - "version": "0.1.2", + "version": "0.1.3", "description": "A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. ", "main": "dist/index.js", "types": "dist/index.d.ts", From 356370400b425db131e0f5a12d8f50337282049d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A6=8F=E6=99=8B?= Date: Thu, 30 Jul 2026 19:22:37 +0800 Subject: [PATCH 4/4] chore: remove dynamic fts fields --- README.md | 10 +++++----- src/storage/schema.ts | 9 +++++++-- src/storage/store.ts | 2 +- src/types.ts | 6 +++--- test/storage/schema.test.ts | 10 +++++----- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ddae587..d809cc0 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ await ctx.close(); | `basePath` | `string` | `process.cwd()` | Base path for resolving document IDs. Set for cross-machine consistent IDs. | | `onProgress` | `(phase, detail) => void` | — | Progress callback for `load()` phases: `'load'` → `'embed'` → `'insert'`. | | `queryExpansion` | `QueryExpansionOptions` | `false` | Query expansion with user-provided synonym map. false disables. Without synonyms, expansion is a no-op. | -| `ftsFields` | `string[]` | `['content']` | Fields to index for Full Text Search in hybrid mode | -| `ftsFieldWeights` | `Record` | `{ content: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. | +| `ftsFields` | `string[]` | `['indexContent']` | Fields queried by Full Text Search in hybrid mode | +| `ftsFieldWeights` | `Record` | `{ indexContent: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. | | `rankConstant` | `number` | `60` | RRF rank constant for hybrid search fusion. Lower = "winner-takes-all", higher = more even. | #### Weight Configuration Example @@ -140,9 +140,9 @@ await ctx.load('g2', './docs/**/*.md', { }); ``` -When `buildIndexContent` is provided, its result is used for embedding and can -be queried by configuring `ftsFields: ['indexContent']`. The original -`document.content` is stored unchanged and returned by `query()`. +When `buildIndexContent` is provided, its result is used for embedding and the +default `indexContent` FTS query. The original `document.content` is stored +unchanged and returned by `query()`. Load phases emit progress via the `onProgress` callback: diff --git a/src/storage/schema.ts b/src/storage/schema.ts index 462cda7..76d7947 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -22,7 +22,7 @@ const INDEX_TYPES = { export function buildZvecSchema( dims: number, tokenizerName: string = 'jieba', - ftsFields: string[] = ['content'], + ftsFields: string[] = ['indexContent'], ): ZVecCollectionSchema { const fts = (name: string): Pick => ftsFields.includes(name) @@ -30,7 +30,12 @@ export function buildZvecSchema( : {}; const fields: ZvecFieldSchema[] = [ { name: 'content', dataType: 'STRING', ...fts('content') }, - { name: 'indexContent', dataType: 'STRING', ...fts('indexContent') }, + { + name: 'indexContent', + dataType: 'STRING', + indexType: 'FTS', + indexOptions: { tokenizerName }, + }, { name: 'meta', dataType: 'STRING' }, { name: 'path', dataType: 'STRING' }, { name: 'contentHash', dataType: 'STRING' }, diff --git a/src/storage/store.ts b/src/storage/store.ts index 916bb80..8448fb9 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -8,7 +8,7 @@ import { Embedder } from '../embedder'; import type { ContextOptions } from '../types'; const VECTOR_FIELD = 'embedding'; -const FTS_FIELDS = ['content']; +const FTS_FIELDS = ['indexContent']; /** * Store - zvec storage for multiple libraries. diff --git a/src/types.ts b/src/types.ts index a8d1404..ce556b6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -119,8 +119,8 @@ export interface ContextOptions { /** * Fields to index for Full Text Search (FTS). * - * By default, `content` is indexed. Add more fields (e.g. `meta.title`) - * to include metadata in text-based recall. + * By default, hybrid search queries `indexContent`, which falls back to + * the original `content` when no buildIndexContent callback is provided. * * Only affects newly created stores — existing stores keep their schema. */ @@ -133,7 +133,7 @@ export interface ContextOptions { * Example: `{ content: 1, title: 3 }` makes title matches 3× more * influential than content matches. * - * Defaults to `{ content: 1 }` when `ftsFields` includes `content`. + * Defaults to `{ indexContent: 1 }`. */ ftsFieldWeights?: Record; diff --git a/test/storage/schema.test.ts b/test/storage/schema.test.ts index 284c9b7..f0a4110 100644 --- a/test/storage/schema.test.ts +++ b/test/storage/schema.test.ts @@ -17,20 +17,20 @@ describe('buildZvecSchema', () => { expect(schemaStr).toContain('metric:COSINE'); }); - it('should include content field with FTS index and jieba tokenizer (default)', () => { + it('should include indexContent with FTS and jieba tokenizer by default', () => { const schema = buildZvecSchema(128, 'jieba'); const schemaStr = String(schema); - expect(schemaStr).toContain("name: 'content'"); + expect(schemaStr).toContain("name: 'indexContent'"); expect(schemaStr).toContain('FtsIndexParams'); expect(schemaStr).toContain('tokenizer_name:jieba'); }); - it('should include indexContent with an FTS index', () => { - const schema = buildZvecSchema(128, 'jieba', ['indexContent']); + it('should allow content to be additionally indexed', () => { + const schema = buildZvecSchema(128, 'jieba', ['content']); const schemaStr = String(schema); - expect(schemaStr).toContain("name: 'indexContent'"); + expect(schemaStr).toContain("name: 'content'"); expect(schemaStr).toContain('FtsIndexParams'); });