New LLMs

LLama3.2/HF TGI support, SpringAI fix

Author: corradodebari

app/src/modules/chatbot.py

@@ -99,9 +99,10 @@ def langchain_rag(self, rag_params, chat_instr, context_instr, input, chat_histo
 
         retrieved_documents = retriever.invoke(input)
         logger.debug("Retrieved %i documents", len(retrieved_documents))
+        logger.info("Retrieved %i documents", len(retrieved_documents))
         # Retrieve documents for inspection (Use for debugging)
-        # for i, doc in enumerate(retrieved_documents):
-        #     logger.debug("Document %i %s", i + 1, doc)
+        for i, doc in enumerate(retrieved_documents):
+             logger.info("Document %i %s", i + 1, doc)
 
         # QA Chain
         context_messages = [("system", context_instr)]

app/src/modules/metadata.py

@@ -139,6 +139,19 @@ def ll_models():
             "frequency_penalty": [0.0, 0.0, -1.0, 1.0],
             "presence_penalty": [0.0, 0.0, -2.0, 2.0],
         },
+        "tgi": {
+            "enabled": False,
+            "api": "OpenAI",
+            "url": "http://127.0.0.1:8080",
+            "api_key": "",
+            "openai_compat": True,
+            "context_length": 127072,
+            "temperature": [1.0, 1.0, 0.0, 2.0],
+            "top_p": [0.99, .99, 0.0, 0.99],
+            "max_tokens": [256, 256, 1, 8191],
+            "frequency_penalty": [0.0, 0.0, -1.0, 1.0],
+            "presence_penalty": [0.0, 0.0, -2.0, 2.0],
+        },
         "gpt-4o": {
             "enabled": os.getenv("OPENAI_API_KEY") is not None,
             "api": "OpenAI",
@@ -192,6 +205,20 @@ def ll_models():
             "frequency_penalty": [0.0, 0.0, -2.0, 2.0],
             "presence_penalty": [0.0, 0.0, -2.0, 2.0],
         },
+        # llama3.2-3b
+        "llama3.2": {
+            "enabled": os.getenv("ON_PREM_OLLAMA_URL") is not None,
+            "api": "ChatOllama",
+            "url": os.environ.get("ON_PREM_OLLAMA_URL", default="http://127.0.0.1:11434"),
+            "api_key": "",
+            "openai_compat": True,
+            "context_length": 131072,
+            "temperature": [1.0, 1.0, 0.0, 2.0],
+            "top_p": [1.0, 1.0, 0.0, 1.0],
+            "max_tokens": [256, 256, 1, 2048],
+            "frequency_penalty": [0.0, 0.0, -2.0, 2.0],
+            "presence_penalty": [0.0, 0.0, -2.0, 2.0],
+        },
     }
     return ll_models_dict
 

app/src/modules/utilities.py

@@ -110,7 +110,7 @@ def get_ll_model(model, ll_models_config=None, giskarded=False):
         _client = OpenAI(api_key=giskard_key, base_url=f"{llm_url}/v1/")
         client = OpenAIClient(model=model, client=_client)
     elif llm_api == "OpenAI":
-        client = ChatOpenAI(api_key=lm_params["api_key"], **common_params)
+        client = ChatOpenAI(api_key=lm_params["api_key"],base_url=f"{llm_url}/v1/", **common_params)
     elif llm_api == "Cohere":
         client = ChatCohere(cohere_api_key=lm_params["api_key"], **common_params)
     elif llm_api == "ChatPerplexity":

spring_ai/README.md

@@ -122,7 +122,9 @@ ollama:
     number: 1
   models:
     - llama3.1
+    - llama3.2
     - mxbai-embed-large
+    - nomic-embed-text
 nodeSelector:
   node.kubernetes.io/instance-type: VM.GPU.A10.1
 ```
@@ -138,9 +140,10 @@ kubectl -n ollama exec svc/ollama -- ollama ls
 ```
 it should be:
 ```
-NAME                        ID              SIZE      MODIFIED           
-llama3.1:latest             42182419e950    4.7 GB    About a minute ago    
-mxbai-embed-large:latest    468836162de7    669 MB    About a minute ago 
+NAME                        ID              SIZE      MODIFIED      
+nomic-embed-text:latest     0a109f422b47    274 MB    3 minutes ago    
+mxbai-embed-large:latest    468836162de7    669 MB    3 minutes ago    
+llama3.1:latest             a80c4f17acd5    2.0 GB    3 minutes ago 
 ```
 * test a single LLM:
 ```

spring_ai/ollama-values.yaml

@@ -0,0 +1,13 @@
+ollama:
+  gpu:
+    enabled: true
+    type: 'nvidia'
+    number: 1
+  models:
+    - llama3.1
+    - llama3.2
+    - mxbai-embed-large
+    - nomic-embed-text
+nodeSelector:
+  node.kubernetes.io/instance-type: VM.GPU.A10.1
+  

spring_ai/pom.xml

@@ -75,6 +75,7 @@
 		<dependency>
 			<groupId>org.springframework.ai</groupId>
 			<artifactId>spring-ai-oracle-store-spring-boot-starter</artifactId>
+			<version>1.0.0-SNAPSHOT</version>
 		</dependency>
 		<!--<dependency>
 			<groupId>org.springframework.ai</groupId>

spring_ai/src/main/java/org/springframework/ai/openai/samples/helloworld/AIController.java

@@ -11,7 +11,6 @@
 import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
 import org.springframework.ai.vectorstore.SearchRequest;
-import org.springframework.ai.vectorstore.SimpleVectorStore.Similarity;
 import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;