feat(memory): add knowledgebase and memory management from LocalRecall

README.md

@@ -33,7 +33,7 @@ LocalAGI ensures your data stays exactly where you want it—on your hardware. N
 - 🤖 **Advanced Agent Teaming**: Instantly create cooperative agent teams from a single prompt.
 - 📡 **Connectors**: Built-in integrations with Discord, Slack, Telegram, GitHub Issues, and IRC.
 - 🛠 **Comprehensive REST API**: Seamless integration into your workflows. Every agent created will support OpenAI Responses API out of the box.
-- 📚 **Short & Long-Term Memory**: Powered by [LocalRecall](https://github.com/mudler/LocalRecall).
+- 📚 **Short & Long-Term Memory**: Built-in knowledge base (RAG) for collections, file uploads, and semantic search. Manage collections in the Web UI under **Knowledge base**; agents with "Knowledge base" enabled use it automatically (implementation uses [LocalRecall](https://github.com/mudler/LocalRecall) libraries).
 - 🧠 **Planning & Reasoning**: Agents intelligently plan, reason, and adapt.
 - 🔄 **Periodic Tasks**: Schedule tasks with cron-like syntax.
 - 💾 **Memory Management**: Control memory usage with options for long-term and summary memory.
@@ -108,7 +108,7 @@ Still having issues? see this Youtube video: https://youtu.be/HtVwIxW3ePg
     </td>
     <td width="50%" valign="top">
       <h3><a href="https://github.com/mudler/LocalRecall">LocalRecall</a></h3>
-      <p>A REST-ful API and knowledge base management system that provides persistent memory and storage capabilities for AI agents.</p>
+      <p>A REST-ful API and knowledge base management system. LocalAGI embeds this functionality: the Web UI includes a <strong>Knowledge base</strong> section and the same collections API, so you no longer need to run LocalRecall separately.</p>
     </td>
   </tr>
 </table>
@@ -239,11 +239,13 @@ LocalAGI supports environment configurations. Note that these environment variab
 | `LOCALAGI_LLM_API_KEY` | API authentication |
 | `LOCALAGI_TIMEOUT` | Request timeout settings |
 | `LOCALAGI_STATE_DIR` | Where state gets stored |
-| `LOCALAGI_LOCALRAG_URL` | LocalRecall connection |
+| `LOCALAGI_BASE_URL` | Optional base URL for the app (only relevant when using an external LocalRAG URL; not used for built-in knowledge base) |
 | `LOCALAGI_ENABLE_CONVERSATIONS_LOGGING` | Toggle conversation logs |
 | `LOCALAGI_API_KEYS` | A comma separated list of api keys used for authentication |
 | `LOCALAGI_CUSTOM_ACTIONS_DIR` | Directory containing custom Go action files to be automatically loaded |
 
+For the built-in knowledge base, optional env (defaults use `LOCALAGI_STATE_DIR`): `COLLECTION_DB_PATH`, `FILE_ASSETS`, `VECTOR_ENGINE` (e.g. `chromem`, `postgres`), `EMBEDDING_MODEL`, `DATABASE_URL` (when `VECTOR_ENGINE=postgres`).
+
 Skills are stored in a fixed `skills` subdirectory under `LOCALAGI_STATE_DIR` (e.g. `/pool/skills` in Docker). Git repo config for skills lives in that directory. No extra environment variables are required.
 
 ## Installation Options
@@ -339,15 +341,16 @@ import (
     "github.com/mudler/LocalAGI/core/types"
 )
 
-// Create a new agent pool
+// Create a new agent pool (call pool.SetRAGProvider(...) for knowledge base; see main.go)
 pool, err := state.NewAgentPool(
     "default-model",           // default model name
     "default-multimodal-model", // default multimodal model
-    "image-model",            // image generation model
+    "transcription-model",     // default transcription model
+    "en",                     // default transcription language
+    "tts-model",              // default TTS model
     "http://localhost:8080",  // API URL
-    "your-api-key",          // API key
-    "./state",               // state directory
-    "http://localhost:8081", // LocalRAG API URL
+    "your-api-key",           // API key
+    "./state",                // state directory
     func(config *AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action {
         // Define available actions for agents
         return func(ctx context.Context, pool *AgentPool) []types.Action {
@@ -374,8 +377,9 @@ pool, err := state.NewAgentPool(
             // Add your custom filters here
         }
     },
-    "10m", // timeout
-    true,  // enable conversation logs
+    "10m",  // timeout
+    true,   // enable conversation logs
+    nil,    // skills service (optional)
 )
 
 // Create a new agent in the pool
@@ -741,7 +745,7 @@ export LOCALAGI_MODEL=gemma-3-4b-it-qat
 export LOCALAGI_MULTIMODAL_MODEL=moondream2-20250414
 export LOCALAGI_IMAGE_MODEL=sd-1.5-ggml
 export LOCALAGI_LLM_API_URL=http://localai:8080
-export LOCALAGI_LOCALRAG_URL=http://localrecall:8080
+# Knowledge base is built-in; no separate LocalRecall service needed
 export LOCALAGI_STATE_DIR=./pool
 export LOCALAGI_TIMEOUT=5m
 export LOCALAGI_ENABLE_CONVERSATIONS_LOGGING=false
@@ -1045,7 +1049,7 @@ LocalAGI supports environment configurations. Note that these environment variab
 | `LOCALAGI_LLM_API_KEY` | API authentication |
 | `LOCALAGI_TIMEOUT` | Request timeout settings |
 | `LOCALAGI_STATE_DIR` | Where state gets stored |
-| `LOCALAGI_LOCALRAG_URL` | LocalRecall connection |
+| `LOCALAGI_BASE_URL` | Optional base URL for built-in knowledge base (default `http://localhost:3000`) |
 | `LOCALAGI_SSHBOX_URL` | LocalAGI SSHBox URL, e.g. user:pass@ip:port |
 | `LOCALAGI_ENABLE_CONVERSATIONS_LOGGING` | Toggle conversation logs |
 | `LOCALAGI_API_KEYS` | A comma separated list of api keys used for authentication |

core/state/compaction.go

@@ -15,6 +15,33 @@ import (
 	"github.com/sashabaranov/go-openai"
 )
 
+// KBCompactionClient is the interface used by compaction. It can be implemented by the HTTP RAG client adapter or by the in-process collection adapter.
+type KBCompactionClient interface {
+	Collection() string
+	ListEntries() ([]string, error)
+	GetEntryContent(entry string) (content string, chunkCount int, err error)
+	Store(filePath string) error
+	DeleteEntry(entry string) error
+}
+
+// wrappedClientCompactionAdapter adapts *localrag.WrappedClient to KBCompactionClient.
+type wrappedClientCompactionAdapter struct {
+	*localrag.WrappedClient
+}
+
+func (a *wrappedClientCompactionAdapter) ListEntries() ([]string, error) {
+	return a.Client.ListEntries(a.Collection())
+}
+
+func (a *wrappedClientCompactionAdapter) Store(filePath string) error {
+	return a.Client.Store(a.Collection(), filePath)
+}
+
+func (a *wrappedClientCompactionAdapter) DeleteEntry(entry string) error {
+	_, err := a.Client.DeleteEntry(a.Collection(), entry)
+	return err
+}
+
 // datePrefixRegex matches YYYY-MM-DD at the start of a filename (e.g. 2006-01-02-15-04-05-hash.txt).
 var datePrefixRegex = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})`)
 
@@ -102,9 +129,9 @@ func (s *openAISummarizer) Summarize(ctx context.Context, content string) (strin
 }
 
 // RunCompaction runs one compaction pass: list entries, group by period, for each group fetch content, optionally summarize, store result, delete originals.
-func RunCompaction(ctx context.Context, client *localrag.WrappedClient, period string, summarize bool, apiURL, apiKey, model string) error {
+func RunCompaction(ctx context.Context, client KBCompactionClient, period string, summarize bool, apiURL, apiKey, model string) error {
 	collection := client.Collection()
-	entries, err := client.Client.ListEntries(collection)
+	entries, err := client.ListEntries()
 	if err != nil {
 		return fmt.Errorf("list entries: %w", err)
 	}
@@ -164,15 +191,15 @@ func RunCompaction(ctx context.Context, client *localrag.WrappedClient, period s
 			xlog.Warn("compaction: write temp file failed", "error", err)
 			continue
 		}
-		if err := client.Client.Store(collection, tmpPath); err != nil {
+		if err := client.Store(tmpPath); err != nil {
 			os.RemoveAll(tmpDir)
 			xlog.Warn("compaction: store failed", "key", key, "error", err)
 			continue
 		}
 		os.RemoveAll(tmpDir)
 
 		for _, entry := range groupEntries {
-			if _, err := client.Client.DeleteEntry(collection, entry); err != nil {
+			if err := client.DeleteEntry(entry); err != nil {
 				xlog.Warn("compaction: delete entry failed", "entry", entry, "error", err)
 			}
 		}
@@ -182,7 +209,7 @@ func RunCompaction(ctx context.Context, client *localrag.WrappedClient, period s
 }
 
 // runCompactionTicker runs compaction on a schedule (daily/weekly/monthly). It stops when ctx is done.
-func runCompactionTicker(ctx context.Context, client *localrag.WrappedClient, config *AgentConfig, apiURL, apiKey, model string) {
+func runCompactionTicker(ctx context.Context, client KBCompactionClient, config *AgentConfig, apiURL, apiKey, model string) {
 	// Run first compaction immediately on startup
 	if err := RunCompaction(ctx, client, config.KBCompactionInterval, config.KBCompactionSummarize, apiURL, apiKey, model); err != nil {
 		xlog.Warn("compaction ticker initial run failed", "collection", client.Collection(), "error", err)

core/state/pool.go

@@ -27,6 +27,26 @@ type SkillsProvider interface {
 	GetMCPSession(ctx context.Context) (*mcp.ClientSession, error)
 }
 
+// RAGProvider returns a RAGDB and optional compaction client for a collection (e.g. agent name).
+// effectiveRAGURL/Key are pool/agent defaults; implementation may use them (HTTP) or ignore them (embedded).
+type RAGProvider func(collectionName, effectiveRAGURL, effectiveRAGKey string) (RAGDB, KBCompactionClient, bool)
+
+// NewHTTPRAGProvider returns a RAGProvider that uses the LocalRAG HTTP API. When effective URL/key are empty, baseURL/baseKey are used.
+func NewHTTPRAGProvider(baseURL, baseKey string) RAGProvider {
+	return func(collectionName, effectiveURL, effectiveKey string) (RAGDB, KBCompactionClient, bool) {
+		url := effectiveURL
+		if url == "" {
+			url = baseURL
+		}
+		key := effectiveKey
+		if key == "" {
+			key = baseKey
+		}
+		wc := localrag.NewWrappedClient(url, key, collectionName)
+		return wc, &wrappedClientCompactionAdapter{WrappedClient: wc}, true
+	}
+}
+
 type AgentPool struct {
 	sync.Mutex
 	file                                                          string
@@ -37,7 +57,8 @@ type AgentPool struct {
 	agentStatus                                                   map[string]*Status
 	apiURL, defaultModel, defaultMultimodalModel, defaultTTSModel string
 	defaultTranscriptionModel, defaultTranscriptionLanguage       string
-	localRAGAPI, localRAGKey, apiKey                              string
+	apiKey                                                        string
+	ragProvider                                                   RAGProvider
 	availableActions                                              func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action
 	connectors                                                    func(*AgentConfig) []Connector
 	dynamicPrompt                                                 func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []DynamicPrompt
@@ -47,6 +68,13 @@ type AgentPool struct {
 	skillsService                                                 SkillsProvider
 }
 
+// SetRAGProvider sets the single RAG provider (HTTP or embedded). Must be called after pool creation.
+func (a *AgentPool) SetRAGProvider(fn RAGProvider) {
+	a.Lock()
+	defer a.Unlock()
+	a.ragProvider = fn
+}
+
 type Status struct {
 	ActionResults []types.ActionState
 }
@@ -79,7 +107,6 @@ func loadPoolFromFile(path string) (*AgentPoolData, error) {
 
 func NewAgentPool(
 	defaultModel, defaultMultimodalModel, defaultTranscriptionModel, defaultTranscriptionLanguage, defaultTTSModel, apiURL, apiKey, directory string,
-	LocalRAGAPI string,
 	availableActions func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action,
 	connectors func(*AgentConfig) []Connector,
 	promptBlocks func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []DynamicPrompt,
@@ -108,7 +135,6 @@ func NewAgentPool(
 			defaultTranscriptionModel:    defaultTranscriptionModel,
 			defaultTranscriptionLanguage: defaultTranscriptionLanguage,
 			defaultTTSModel:              defaultTTSModel,
-			localRAGAPI:                  LocalRAGAPI,
 			apiKey:                       apiKey,
 			agents:                       make(map[string]*Agent),
 			pool:                         make(map[string]AgentConfig),
@@ -143,7 +169,6 @@ func NewAgentPool(
 		agentStatus:                  map[string]*Status{},
 		pool:                         *poolData,
 		connectors:                   connectors,
-		localRAGAPI:                  LocalRAGAPI,
 		dynamicPrompt:                promptBlocks,
 		filters:                      filters,
 		availableActions:             availableActions,
@@ -303,14 +328,8 @@ func (a *AgentPool) startAgentWithConfig(name, pooldir string, config *AgentConf
 	} else {
 		config.APIKey = a.apiKey
 	}
-	effectiveLocalRAGAPI := a.localRAGAPI
-	if config.LocalRAGURL != "" {
-		effectiveLocalRAGAPI = config.LocalRAGURL
-	}
-	effectiveLocalRAGKey := a.localRAGKey
-	if config.LocalRAGAPIKey != "" {
-		effectiveLocalRAGKey = config.LocalRAGAPIKey
-	}
+	effectiveLocalRAGAPI := config.LocalRAGURL
+	effectiveLocalRAGKey := config.LocalRAGAPIKey
 
 	connectors := a.connectors(config)
 	promptBlocks := a.dynamicPrompt(config)(ctx, a)
@@ -510,26 +529,27 @@ func (a *AgentPool) startAgentWithConfig(name, pooldir string, config *AgentConf
 		}
 	}
 
-	var ragClient *localrag.WrappedClient
-	if config.EnableKnowledgeBase {
-		ragClient = localrag.NewWrappedClient(effectiveLocalRAGAPI, effectiveLocalRAGKey, name)
-		opts = append(opts, WithRAGDB(ragClient), EnableKnowledgeBase)
-		// Set KB auto search option (defaults to true for backward compatibility)
-		// For backward compatibility: if both new KB fields are false (zero values),
-		// assume this is an old config and default KBAutoSearch to true
+	var ragDB RAGDB
+	var compactionClient KBCompactionClient
+	if config.EnableKnowledgeBase && a.ragProvider != nil {
+		if db, comp, ok := a.ragProvider(name, effectiveLocalRAGAPI, effectiveLocalRAGKey); ok && db != nil {
+			ragDB = db
+			compactionClient = comp
+		}
+	}
+	if ragDB != nil {
+		opts = append(opts, WithRAGDB(ragDB), EnableKnowledgeBase)
 		kbAutoSearch := config.KBAutoSearch
 		if !config.KBAutoSearch && !config.KBAsTools {
-			// Both new fields are false, likely an old config - default to true for backward compatibility
 			kbAutoSearch = true
 		}
 		opts = append(opts, WithKBAutoSearch(kbAutoSearch))
-		// Inject KB wrapper actions if enabled
-		if config.KBAsTools && ragClient != nil {
+		if config.KBAsTools {
 			kbResults := config.KnowledgeBaseResults
 			if kbResults <= 0 {
-				kbResults = 5 // Default
+				kbResults = 5
 			}
-			searchAction, addAction := NewKBWrapperActions(ragClient, kbResults)
+			searchAction, addAction := NewKBWrapperActions(ragDB, kbResults)
 			opts = append(opts, WithActions(searchAction, addAction))
 		}
 	}
@@ -582,8 +602,8 @@ func (a *AgentPool) startAgentWithConfig(name, pooldir string, config *AgentConf
 		}
 	}()
 
-	if config.EnableKnowledgeBase && config.EnableKBCompaction && ragClient != nil {
-		go runCompactionTicker(ctx, ragClient, config, effectiveAPIURL, effectiveAPIKey, model)
+	if config.EnableKnowledgeBase && config.EnableKBCompaction && compactionClient != nil {
+		go runCompactionTicker(ctx, compactionClient, config, effectiveAPIURL, effectiveAPIKey, model)
 	}
 
 	xlog.Info("Starting connectors", "name", name, "config", config)

docker-compose.amd.yaml

@@ -11,25 +11,15 @@ services:
       - /dev/dri
       - /dev/kfd
 
-  dind:
-    extends:
-      file: docker-compose.yaml
-      service: dind
-
-  localrecall-postgres:
+  postgres:
     extends:
       file: docker-compose.yaml
-      service: localrecall-postgres
+      service: postgres
 
-  localrecall:
-    extends:
-      file: docker-compose.yaml
-      service: localrecall
-
-  localrecall-healthcheck:
+  dind:
     extends:
       file: docker-compose.yaml
-      service: localrecall-healthcheck
+      service: dind
 
   localagi:
     extends:

docker-compose.intel.yaml

@@ -12,25 +12,15 @@ services:
       - /dev/dri/card1
       - /dev/dri/renderD129
 
-  dind:
-    extends:
-      file: docker-compose.yaml
-      service: dind
-
-  localrecall-postgres:
+  postgres:
     extends:
       file: docker-compose.yaml
-      service: localrecall-postgres
+      service: postgres
 
-  localrecall:
-    extends:
-      file: docker-compose.yaml
-      service: localrecall
-
-  localrecall-healthcheck:
+  dind:
     extends:
       file: docker-compose.yaml
-      service: localrecall-healthcheck
+      service: dind
 
   localagi:
     extends:

docker-compose.nvidia.yaml

@@ -17,25 +17,15 @@ services:
               count: 1
               capabilities: [gpu]
 
-  dind:
-    extends:
-      file: docker-compose.yaml
-      service: dind
-
-  localrecall-postgres:
+  postgres:
     extends:
       file: docker-compose.yaml
-      service: localrecall-postgres
+      service: postgres
 
-  localrecall:
-    extends:
-      file: docker-compose.yaml
-      service: localrecall
-
-  localrecall-healthcheck:
+  dind:
     extends:
       file: docker-compose.yaml
-      service: localrecall-healthcheck
+      service: dind
 
   localagi:
     extends: