vLLM support for LangChain MCP export

Author: corradodebari

.gitignore

@@ -67,4 +67,4 @@ spring_ai/drop.sql
 src/client/spring_ai/target/classes/*
 api_server_key
 .env
-testEmbeddings.py
+

src/client/content/config/tabs/settings.py

@@ -161,7 +161,9 @@ def spring_ai_conf_check(ll_model: dict, embed_model: dict) -> str:
 
     ll_provider = ll_model.get("provider", "")
     embed_provider = embed_model.get("provider", "")
-
+    logger.info(f"llm chat:{ll_provider} - embeddings:{embed_provider}")
+    if all("openai_compatible" in p for p in (ll_provider, embed_provider)):
+        return "openai_compatible"
     if all("openai" in p for p in (ll_provider, embed_provider)):
         return "openai"
     if all("ollama" in p for p in (ll_provider, embed_provider)):
@@ -343,6 +345,8 @@ def display_settings():
         embed_config = {}
     spring_ai_conf = spring_ai_conf_check(ll_config, embed_config)
 
+    logger.info(f"config found:{spring_ai_conf}")
+    
     if spring_ai_conf == "hybrid":
         st.markdown(f"""
             The current configuration combination of embedding and language models
@@ -353,21 +357,23 @@ def display_settings():
     else:
         col_left, col_centre, _ = st.columns([3, 4, 3])
         with col_left:
-            st.download_button(
-                label="Download SpringAI",
-                data=spring_ai_zip(spring_ai_conf, ll_config, embed_config),  # Generate zip on the fly
-                file_name="spring_ai.zip",  # Zip file name
-                mime="application/zip",  # Mime type for zip file
-                disabled=spring_ai_conf == "hybrid",
-            )
-        with col_centre:
             st.download_button(
                 label="Download LangchainMCP",
                 data=langchain_mcp_zip(settings),  # Generate zip on the fly
                 file_name="langchain_mcp.zip",  # Zip file name
                 mime="application/zip",  # Mime type for zip file
                 disabled=spring_ai_conf == "hybrid",
             )
+        with col_centre:
+            if (spring_ai_conf != "openai_compatible"):
+                st.download_button(
+                    label="Download SpringAI",
+                    data=spring_ai_zip(spring_ai_conf, ll_config, embed_config),  # Generate zip on the fly
+                    file_name="spring_ai.zip",  # Zip file name
+                    mime="application/zip",  # Mime type for zip file
+                    disabled=spring_ai_conf == "hybrid",
+                )
+            
 
 
 if __name__ == "__main__":

src/client/mcp/rag/optimizer_utils/config.py

@@ -39,8 +39,11 @@ def get_llm(data):
         llm = OllamaLLM(model=model, base_url=url)
         logger.info("Ollama LLM created")
     elif provider == "openai":
-        llm = llm = ChatOpenAI(model=model, api_key=api_key)
+        llm = ChatOpenAI(model=model, api_key=api_key)
         logger.info("OpenAI LLM created")
+    elif provider =="openai_compatible":
+        llm = ChatOpenAI(model=model, api_key=api_key,base_url=url)
+        logger.info("OpenAI compatible LLM created")
     return llm
 
 
@@ -60,9 +63,13 @@ def get_embeddings(data):
     if provider == "ollama":
         embeddings = OllamaEmbeddings(model=model, base_url=url)
         logger.info("Ollama Embeddings connection successful")
-    elif (provider == "openai") or (provider == "openai_compatible"):
+    elif (provider == "openai"):
         embeddings = OpenAIEmbeddings(model=model, api_key=api_key)
         logger.info("OpenAI embeddings connection successful")
+    elif (provider == "openai_compatible"):
+        embeddings = OpenAIEmbeddings(model=model, api_key=api_key,base_url=url,check_embedding_ctx_length=False)
+        logger.info("OpenAI compatible embeddings connection successful")
+
     return embeddings
 
 
@@ -80,7 +87,7 @@ def get_vectorstore(data, embeddings):
     distance_metric=data["client_settings"]["vector_search"]["distance_metric"]
     index_type=data["client_settings"]["vector_search"]["index_type"]
 
-    db_table=(table_alias+"_"+model+"_"+chunk_size+"_"+chunk_overlap+"_"+distance_metric+"_"+index_type).upper().replace("-", "_")
+    db_table=(table_alias+"_"+model+"_"+chunk_size+"_"+chunk_overlap+"_"+distance_metric+"_"+index_type).upper().replace("-", "_").replace("/", "_")
     logger.info(f"db_table:{db_table}")
 
 

src/client/mcp/rag/optimizer_utils/rag.py

@@ -54,7 +54,7 @@ def rag_tool_base(question: str) -> str:
 
             logger.info("rag_prompt:")
             logger.info(rag_prompt)
-            template = """DOCUMENTS: {context} \n"""+rag_prompt+"""\nQuestion: {question} """
+            template = rag_prompt+"""\n# DOCUMENTS :\n {context} \n"""+"""\n # Question: {question} """
             logger.info(template)
             logger.info(f"user_question: {user_question}")
             prompt = PromptTemplate.from_template(template)

src/server/bootstrap/models.py

@@ -88,6 +88,20 @@ def main() -> list[Model]:
             "max_completion_tokens": 2048,
             "frequency_penalty": 0.0,
         },
+        {
+            # This is intentionally last to line up with docos
+            "id": "meta-llama/Llama-3.2-1B-Instruct",
+            "enabled": os.getenv("ON_PREM_VLLM_URL") is not None,
+            "type": "ll",
+            "provider": "openai_compatible",
+            "api_key": "",
+            "openai_compat": True,
+            "url": os.environ.get("ON_PREM_VLLM_URL", default="http://gpu:8000/v1"),
+            "context_length": 131072,
+            "temperature": 1.0,
+            "max_completion_tokens": 2048,
+            "frequency_penalty": 0.0,
+        },
         {
             # This is intentionally last to line up with docos
             "id": "llama3.1",
@@ -133,7 +147,7 @@ def main() -> list[Model]:
             "max_chunk_size": 512,
         },
         {
-            "id": "text-embedding-nomic-embed-text-v1.5",
+            "id": "nomic-ai/nomic-embed-text-v1",
             "enabled": False,
             "type": "embed",
             "provider": "openai_compatible",