rag.py

import os

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import TextLoader
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings

currt_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(currt_dir, "RAGFiles", "Risk Management Rules and Standards.txt")
persist_directory = os.path.join(currt_dir, "db","chroma_GRisk_db")


    

loader = TextLoader(file_path, encoding="utf-8")
documents = loader.load()

text_splitter = RecursiveCharacterTextSplitter(chunk_size=2350, chunk_overlap=1175)
doc= text_splitter.split_documents(documents)

print(f"Number of documents: {len(doc)}")
    #print(f"sample document:\n{doc[0].page_content}\n")
if not os.path.exists(persist_directory):
    print("Creating vector store...")

    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

    print("done creating embeddings")

    db = Chroma.from_documents(doc, embeddings, persist_directory=persist_directory)
    print("done creating vector store")

else: 
    print("Directory already exists.")
    print(f"sample document:\n{doc[0].page_content}\n")