diff --git a/src/isaac_rag_enhanced.ts b/src/isaac_rag_enhanced.ts new file mode 100644 index 0000000..8e5a7b0 --- /dev/null +++ b/src/isaac_rag_enhanced.ts @@ -0,0 +1,36 @@ +import { + Document, + VectorStoreIndex, + HierarchicalNodeParser, + get_response_synthesizer, + RetrieverQueryEngine, + SimilarityPostprocessor +} from "llamaindex"; + +export async function createEnhancedRagPipeline(texts: string[]) { + // 1. Create Documents + const documents = texts.map(t => new Document({ text: t })); + + // 2. Hierarchical Parsing (Better for Scientific Papers) + const nodeParser = new HierarchicalNodeParser({ + chunkSizes: [2048, 512, 128] + }); + const nodes = nodeParser.getNodesFromDocuments(documents); + + // 3. Build Index + const index = await VectorStoreIndex.fromNodes(nodes); + + // 4. Advanced Query Engine with Re-ranking + const retriever = index.asRetriever(); + retriever.similarityTopK = 5; + + const responseSynthesizer = get_response_synthesizer("compact"); + + const queryEngine = new RetrieverQueryEngine( + retriever, + responseSynthesizer, + [new SimilarityPostprocessor({ similarityCutoff: 0.7 })] + ); + + return queryEngine; +}