diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 7fc6af9d69..41d4a686dc 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -7002,8 +7002,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'مطلوب مفتاح API', 'userErrors.apiKeyMissing.body': 'لا يوجد مفتاح API لمزوّد الذكاء الاصطناعي. أضِفه في إعدادات المزوّد للمتابعة.', + 'userErrors.localModelUnavailable.title': 'النموذج المحلي غير متاح', + 'userErrors.localModelUnavailable.body': + 'لا يمكن الوصول إلى Ollama على النقطة الطرفية المُهيأة، أو أن النموذج المطلوب غير مثبّت عليها. شغّل Ollama ونزّل النموذج على تلك النقطة الطرفية، أو حوّل هذا العمل إلى مزوّد سحابي.', 'userErrors.scope.chat': 'الدردشة', 'userErrors.scope.cron': 'مهمة مجدوَلة', + 'userErrors.scope.memory': 'الذاكرة', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'المبلغ', 'agentWorld.trading.networkLabel': 'الشبكة', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 9d9736729c..3ff6720bb9 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -7161,8 +7161,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API কী প্রয়োজন', 'userErrors.apiKeyMissing.body': 'আপনার AI প্রদানকারীর কোনো API কী সেট নেই। চালিয়ে যেতে প্রদানকারী সেটিংসে একটি যোগ করুন।', + 'userErrors.localModelUnavailable.title': 'লোকাল মডেল অনুপলব্ধ', + 'userErrors.localModelUnavailable.body': + 'কনফিগার করা এন্ডপয়েন্টে Ollama-তে পৌঁছানো যাচ্ছে না, অথবা সেখানে প্রয়োজনীয় মডেলটি ইনস্টল করা নেই। Ollama চালু করে সেই এন্ডপয়েন্টে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', 'userErrors.scope.chat': 'চ্যাট', 'userErrors.scope.cron': 'নির্ধারিত কাজ', + 'userErrors.scope.memory': 'মেমরি', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'পরিমাণ', 'agentWorld.trading.networkLabel': 'নেটওয়ার্ক', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index e723971f2a..0df7caf585 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -7362,8 +7362,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API-Schlüssel erforderlich', 'userErrors.apiKeyMissing.body': 'Für deinen KI-Anbieter ist kein API-Schlüssel hinterlegt. Füge in den Anbietereinstellungen einen hinzu, um fortzufahren.', + 'userErrors.localModelUnavailable.title': 'Lokales Modell nicht verfügbar', + 'userErrors.localModelUnavailable.body': + 'Ollama ist unter dem konfigurierten Endpunkt nicht erreichbar, oder das benötigte Modell ist dort nicht installiert. Starte Ollama und lade das Modell auf diesem Endpunkt, oder verlagere diese Arbeit auf einen Cloud-Anbieter.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Geplante Aufgabe', + 'userErrors.scope.memory': 'Speicher', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Betrag', 'agentWorld.trading.networkLabel': 'Netzwerk', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 47d702debd..62f01d45a4 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -7558,8 +7558,12 @@ const en: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API key required', 'userErrors.apiKeyMissing.body': 'Your AI provider has no API key set. Add one in provider settings to continue.', + 'userErrors.localModelUnavailable.title': 'Local model unavailable', + 'userErrors.localModelUnavailable.body': + 'Ollama is not reachable at the configured endpoint, or the required model is not installed there. Start Ollama and pull the model at that endpoint, or switch this workload to a cloud provider.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Scheduled job', + 'userErrors.scope.memory': 'Memory', 'memorySources.codingSessions.title': 'Coding-agent sessions', 'memorySources.codingSessions.description': 'Turn your Codex and Claude Code decisions and corrections into private persona memory.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 68a2efedca..7d0485bbbc 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -7309,8 +7309,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Se requiere clave de API', 'userErrors.apiKeyMissing.body': 'Tu proveedor de IA no tiene una clave de API configurada. Añade una en los ajustes del proveedor para continuar.', + 'userErrors.localModelUnavailable.title': 'Modelo local no disponible', + 'userErrors.localModelUnavailable.body': + 'No se puede acceder a Ollama en el punto de conexión configurado, o el modelo necesario no está instalado allí. Inicia Ollama y descarga el modelo en ese punto de conexión, o cambia este trabajo a un proveedor en la nube.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarea programada', + 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importe', 'agentWorld.trading.networkLabel': 'Red', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 4d6bf9653a..e8d6d81e5c 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -7341,8 +7341,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Clé API requise', 'userErrors.apiKeyMissing.body': "Aucune clé API n'est définie pour votre fournisseur d'IA. Ajoutez-en une dans les paramètres du fournisseur pour continuer.", + 'userErrors.localModelUnavailable.title': 'Modèle local indisponible', + 'userErrors.localModelUnavailable.body': + "Ollama n'est pas joignable sur le point de terminaison configuré, ou le modèle requis n'y est pas installé. Lancez Ollama et téléchargez le modèle sur ce point de terminaison, ou basculez cette charge de travail vers un fournisseur cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tâche planifiée', + 'userErrors.scope.memory': 'Mémoire', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Montant', 'agentWorld.trading.networkLabel': 'Réseau', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index a1775f3c6b..e6b9949fcb 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -7159,8 +7159,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API कुंजी आवश्यक', 'userErrors.apiKeyMissing.body': 'आपके AI प्रदाता के लिए कोई API कुंजी सेट नहीं है। जारी रखने के लिए प्रदाता सेटिंग्स में एक जोड़ें।', + 'userErrors.localModelUnavailable.title': 'लोकल मॉडल उपलब्ध नहीं है', + 'userErrors.localModelUnavailable.body': + 'कॉन्फ़िगर किए गए एंडपॉइंट पर Ollama तक पहुँच नहीं है, या ज़रूरी मॉडल वहाँ इंस्टॉल नहीं है। Ollama शुरू करके उसी एंडपॉइंट पर मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', 'userErrors.scope.chat': 'चैट', 'userErrors.scope.cron': 'निर्धारित कार्य', + 'userErrors.scope.memory': 'मेमोरी', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'राशि', 'agentWorld.trading.networkLabel': 'नेटवर्क', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 1b628350d6..fe672a6ac2 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -7197,8 +7197,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Kunci API diperlukan', 'userErrors.apiKeyMissing.body': 'Penyedia AI Anda belum memiliki kunci API. Tambahkan satu di pengaturan penyedia untuk melanjutkan.', + 'userErrors.localModelUnavailable.title': 'Model lokal tidak tersedia', + 'userErrors.localModelUnavailable.body': + 'Ollama tidak dapat dijangkau di endpoint yang dikonfigurasi, atau model yang dibutuhkan belum terpasang di sana. Jalankan Ollama dan unduh modelnya di endpoint tersebut, atau alihkan pekerjaan ini ke penyedia cloud.', 'userErrors.scope.chat': 'Obrolan', 'userErrors.scope.cron': 'Tugas terjadwal', + 'userErrors.scope.memory': 'Memori', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Jumlah', 'agentWorld.trading.networkLabel': 'Jaringan', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 0f1f611011..067596b64f 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -7294,8 +7294,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Chiave API richiesta', 'userErrors.apiKeyMissing.body': 'Il tuo provider IA non ha una chiave API impostata. Aggiungine una nelle impostazioni del provider per continuare.', + 'userErrors.localModelUnavailable.title': 'Modello locale non disponibile', + 'userErrors.localModelUnavailable.body': + "Ollama non è raggiungibile sull'endpoint configurato, oppure il modello necessario non è installato lì. Avvia Ollama e scarica il modello su quell'endpoint, oppure sposta questo lavoro su un provider cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Attività pianificata', + 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importo', 'agentWorld.trading.networkLabel': 'Rete', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 07f8c92e2d..baff9ef2db 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -7080,8 +7080,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API 키 필요', 'userErrors.apiKeyMissing.body': 'AI 제공업체에 API 키가 설정되지 않았습니다. 제공업체 설정에서 추가하세요.', + 'userErrors.localModelUnavailable.title': '로컬 모델을 사용할 수 없음', + 'userErrors.localModelUnavailable.body': + '구성된 엔드포인트에서 Ollama에 연결할 수 없거나 필요한 모델이 그곳에 설치되어 있지 않습니다. Ollama를 실행하고 해당 엔드포인트에 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', 'userErrors.scope.chat': '채팅', 'userErrors.scope.cron': '예약된 작업', + 'userErrors.scope.memory': '메모리', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '금액', 'agentWorld.trading.networkLabel': '네트워크', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index e49c8e81a0..e564acfc6b 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -7266,8 +7266,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Wymagany klucz API', 'userErrors.apiKeyMissing.body': 'Twój dostawca AI nie ma ustawionego klucza API. Dodaj go w ustawieniach dostawcy, aby kontynuować.', + 'userErrors.localModelUnavailable.title': 'Model lokalny niedostępny', + 'userErrors.localModelUnavailable.body': + 'Ollama jest nieosiągalna pod skonfigurowanym punktem końcowym albo wymagany model nie jest tam zainstalowany. Uruchom Ollamę i pobierz model w tym punkcie końcowym lub przenieś tę pracę do dostawcy w chmurze.', 'userErrors.scope.chat': 'Czat', 'userErrors.scope.cron': 'Zaplanowane zadanie', + 'userErrors.scope.memory': 'Pamięć', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Kwota', 'agentWorld.trading.networkLabel': 'Sieć', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 925a375360..c6aecb0882 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -7277,8 +7277,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Chave de API necessária', 'userErrors.apiKeyMissing.body': 'Seu provedor de IA não tem uma chave de API definida. Adicione uma nas configurações do provedor para continuar.', + 'userErrors.localModelUnavailable.title': 'Modelo local indisponível', + 'userErrors.localModelUnavailable.body': + 'O Ollama não está acessível no endpoint configurado, ou o modelo necessário não está instalado nele. Inicie o Ollama e baixe o modelo nesse endpoint, ou mude este trabalho para um provedor na nuvem.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarefa agendada', + 'userErrors.scope.memory': 'Memória', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Valor', 'agentWorld.trading.networkLabel': 'Rede', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index e17c8258ff..17323f2747 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -7239,8 +7239,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Требуется ключ API', 'userErrors.apiKeyMissing.body': 'У провайдера ИИ не задан ключ API. Добавьте его в настройках провайдера.', + 'userErrors.localModelUnavailable.title': 'Локальная модель недоступна', + 'userErrors.localModelUnavailable.body': + 'Ollama недоступен по настроенному адресу, либо нужная модель там не установлена. Запустите Ollama и загрузите модель по этому адресу или переведите эту работу на облачного провайдера.', 'userErrors.scope.chat': 'Чат', 'userErrors.scope.cron': 'Запланированная задача', + 'userErrors.scope.memory': 'Память', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Сумма', 'agentWorld.trading.networkLabel': 'Сеть', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 43498261fe..edcd1357c6 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -6775,8 +6775,12 @@ const messages: TranslationMap = { 'userErrors.insufficientCredits.body': '提供商额度已用完,请充值或更新 API 密钥。', 'userErrors.apiKeyMissing.title': '需要 API 密钥', 'userErrors.apiKeyMissing.body': '您的 AI 提供商未设置 API 密钥,请在提供商设置中添加以继续。', + 'userErrors.localModelUnavailable.title': '本地模型不可用', + 'userErrors.localModelUnavailable.body': + '无法在配置的端点连接 Ollama,或所需模型未安装在该端点。请启动 Ollama 并在该端点拉取模型,或将此工作切换到云端提供商。', 'userErrors.scope.chat': '聊天', 'userErrors.scope.cron': '定时任务', + 'userErrors.scope.memory': '记忆', // Agent World:Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '金额', 'agentWorld.trading.networkLabel': '网络', diff --git a/app/src/lib/userErrors/__tests__/classify.test.ts b/app/src/lib/userErrors/__tests__/classify.test.ts index 7718409858..ca225456e3 100644 --- a/app/src/lib/userErrors/__tests__/classify.test.ts +++ b/app/src/lib/userErrors/__tests__/classify.test.ts @@ -60,6 +60,50 @@ describe('classifyUserActionableError', () => { expect(classifyUserActionableError({ message: 'Incorrect API key provided' })).toBeNull(); }); + it('classifies an unusable local model runtime (memory user_error kind token)', () => { + // Core's memory embedder health gate emits the stable kind token with + // error_source=memory (#5354); socketService maps that to the memory scope. + const a = classifyUserActionableError({ + errorType: 'local_model_unavailable', + scope: 'memory', + sourceDomain: 'memory', + }); + expect(a?.kind).toBe('local_model_unavailable'); + expect(a?.scope).toBe('memory'); + expect(a?.action).toBe('open_provider_settings'); + expect(a?.titleKey).toBe('userErrors.localModelUnavailable.title'); + expect(a?.bodyKey).toBe('userErrors.localModelUnavailable.body'); + expect(a?.id).toBe(userErrorId('local_model_unavailable', 'memory', undefined)); + + // …and every prose shape the local embedder / health gate / doctor + // produce. Both Rust-side shapes are covered so the two classifiers stay + // symmetric — daemon-not-listening AND model-never-pulled. + for (const msg of [ + 'ollama embed request failed (is Ollama running at http://localhost:11434?)', + 'Ollama embedding model `bge-m3` is not installed at http://localhost:11434. Run `ollama pull bge-m3`', + 'Ollama daemon unreachable at http://localhost:11434', + 'ollama embeddings opted-in but daemon unreachable at http://localhost:11434', + ]) { + expect(classifyUserActionableError({ message: msg })?.kind).toBe('local_model_unavailable'); + } + }); + + it('does NOT promote a bare "daemon unreachable" from another domain', () => { + // Backend connection-health logs use this phrase too. Matching it loosely + // would tell a user with a flaky backend link to install Ollama. The Rust + // matcher anchors on the full producer wording for the same reason. + expect( + classifyUserActionableError({ message: 'backend daemon unreachable at api.tinyhumans.ai' }) + ).toBeNull(); + }); + + it('keeps billing remediation for a credits error that also names Ollama', () => { + // The local-runtime rule is last on purpose: an out-of-credits provider + // must not be told to install Ollama. + const a = classifyUserActionableError({ message: 'ollama proxy requires more credits' }); + expect(a?.kind).toBe('insufficient_credits'); + }); + it('returns null for generic / non-actionable errors and empty input', () => { expect(classifyUserActionableError({ message: GENERIC_MSG })).toBeNull(); expect(classifyUserActionableError({ message: '', errorType: 'inference' })).toBeNull(); diff --git a/app/src/lib/userErrors/classify.ts b/app/src/lib/userErrors/classify.ts index 8f7b6f95c6..5629d11b80 100644 --- a/app/src/lib/userErrors/classify.ts +++ b/app/src/lib/userErrors/classify.ts @@ -126,5 +126,45 @@ export function classifyUserActionableError( }; } + // The local model runtime a workload depends on is unusable — Ollama is not + // running, or the configured model was never pulled (#5354). Emitted by core + // as the stable `local_model_unavailable` kind token (memory embedder health + // gate); the prose variants match the wording the local embedder itself + // produces, so a signal that arrives with a message instead of a token still + // classifies. Deliberately last: it is the narrowest rule, and a provider + // that is out of credits should keep its billing remediation even when the + // message happens to name Ollama. + // Each prose matcher is anchored on the FULL producer wording, never a bare + // `daemon unreachable at`: backend connection-health logs in other domains + // emit that phrase too, and promoting one of those into an "install Ollama" + // panel entry would be worse than showing nothing. Mirrors the same + // deliberate anchoring in the Rust matcher `is_ollama_user_config_rejection`. + const isLocalModelUnavailable = + text.includes('local_model_unavailable') || + // tinyagents embedder, daemon not listening. + text.includes('is ollama running') || + // tinyagents embedder, model never pulled — the second shape the Rust + // classifier recognises, so the two sides stay symmetric. + (text.includes('ollama embedding model') && text.includes('is not installed at')) || + // platform doctor report. + text.includes('ollama daemon unreachable') || + // memory embedder health gate. + text.includes('ollama embeddings opted-in but daemon unreachable at'); + if (isLocalModelUnavailable) { + return { + id: userErrorId('local_model_unavailable', scope, signal.provider), + kind: 'local_model_unavailable', + severity: 'warning', + scope, + sourceDomain: signal.sourceDomain, + provider: signal.provider, + titleKey: 'userErrors.localModelUnavailable.title', + bodyKey: 'userErrors.localModelUnavailable.body', + // Local AI + embedding provider selection both live behind + // `/settings/llm`, which redirects to Connections → API keys. + action: 'open_provider_settings', + }; + } + return null; } diff --git a/app/src/services/__tests__/socketService.events.test.ts b/app/src/services/__tests__/socketService.events.test.ts index c61553be3f..ad64504d12 100644 --- a/app/src/services/__tests__/socketService.events.test.ts +++ b/app/src/services/__tests__/socketService.events.test.ts @@ -417,6 +417,33 @@ describe('socketService — agent_meetings event handlers (lines 428-480)', () = expect(signal.message).toBeUndefined(); }); + it('scopes a memory-sourced "user_error" to memory rather than cron (#5354)', async () => { + const { handlers, mockSocket } = buildMockSocket(); + + vi.doMock('socket.io-client', () => ({ io: vi.fn(() => mockSocket) })); + getCoreRpcUrlMock.mockResolvedValue('http://127.0.0.1:7788/rpc'); + ingestRuntimeErrorSignalMock.mockClear(); + + const { socketService } = await import('../socketService'); + socketService.connect('jwt-test-user-error-memory'); + + await pollUntil(() => expect(handlers['user_error']).toBeDefined()); + + // The memory embedder health gate is the second producer of this event. + // Scope is part of the panel entry's dedupe identity, so it must follow the + // producing domain instead of staying pinned to the cron default. + handlers['user_error']!({ error_type: 'local_model_unavailable', error_source: 'memory' }); + + expect(ingestRuntimeErrorSignalMock).toHaveBeenCalledTimes(1); + const signal = ingestRuntimeErrorSignalMock.mock.calls[0]?.[1] as Record; + expect(signal).toMatchObject({ + errorType: 'local_model_unavailable', + scope: 'memory', + sourceDomain: 'memory', + }); + expect(signal.message).toBeUndefined(); + }); + it('defaults "user_error" sourceDomain to cron when error_source is absent (#4165)', async () => { const { handlers, mockSocket } = buildMockSocket(); diff --git a/app/src/services/socketService.ts b/app/src/services/socketService.ts index 84ea20f39b..832ff83f25 100644 --- a/app/src/services/socketService.ts +++ b/app/src/services/socketService.ts @@ -18,6 +18,7 @@ import { upsertChannelConnection } from '../store/channelConnectionsSlice'; import { setBackend } from '../store/connectivitySlice'; import { resetForUser, setSocketIdForUser, setStatusForUser } from '../store/socketSlice'; import type { ChannelAuthMode, ChannelConnectionStatus, ChannelType } from '../types/channels'; +import type { UserErrorScope } from '../types/userError'; import { IS_DEV } from '../utils/config'; import { createSafeLogData, sanitizeError } from '../utils/sanitize'; import { getCoreRpcToken, getCoreRpcUrl } from './coreRpcClient'; @@ -408,17 +409,18 @@ class SocketService { const provider = typeof obj.error_provider === 'string' ? obj.error_provider : undefined; const sourceDomain = typeof obj.error_source === 'string' ? obj.error_source : 'cron'; socketLog('user_error kind=%s source=%s', errorType ?? 'none', sourceDomain); + // Scope groups the entry in the panel and is part of its dedupe identity, + // so it must follow the producing domain. It was pinned to `cron` while + // the scheduler was the only producer; the memory embedder health gate + // (#5354) is the second. Unknown domains keep the historical `cron` + // default rather than widening the scope union from wire data. + const scope: UserErrorScope = sourceDomain === 'memory' ? 'memory' : 'cron'; // Metadata-only ingest: forward the stable kind token + scope ONLY, never // a raw `message` body. The cron producer already omits it, but we drop // any `obj.message` here too so a future/buggy broadcast can't leak raw // provider text into the UI — classify() keys on `errorType` for this // path. Locks the no-leak contract FE-side (CodeRabbit #4169). - ingestRuntimeErrorSignal(store.dispatch, { - errorType, - scope: 'cron', - sourceDomain, - provider, - }); + ingestRuntimeErrorSignal(store.dispatch, { errorType, scope, sourceDomain, provider }); }); // Backend Meet bot events — forwarded from core's DomainEvent bus diff --git a/app/src/types/userError.ts b/app/src/types/userError.ts index 87e4f9c6c7..1ae50ecabc 100644 --- a/app/src/types/userError.ts +++ b/app/src/types/userError.ts @@ -14,10 +14,26 @@ */ /** Stable discriminator the UI branches on. Extend as new states are added. */ -export type UserErrorKind = 'insufficient_credits' | 'budget_exceeded' | 'api_key_missing'; +export type UserErrorKind = + | 'insufficient_credits' + | 'budget_exceeded' + | 'api_key_missing' + /** + * The local model runtime a workload depends on is not usable — Ollama is + * not running, or the configured model was never pulled (#5354). Mirrors the + * core-side `LOCAL_MODEL_UNAVAILABLE_KIND` token. + */ + | 'local_model_unavailable'; /** Where the failure originated, for grouping/labelling (privacy-safe). */ -export type UserErrorScope = 'chat' | 'cron' | 'provider' | 'integration' | 'workspace'; +export type UserErrorScope = + | 'chat' + | 'cron' + | 'provider' + | 'integration' + | 'workspace' + /** Memory ingestion / embedding pipeline. */ + | 'memory'; /** Primary next-step the user can take. `dismiss` is always available too. */ export type UserErrorAction = 'open_billing' | 'open_provider_settings' | 'dismiss'; diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory/store/factories.rs index 75e221cb35..ca8fee44de 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory/store/factories.rs @@ -34,11 +34,30 @@ static OLLAMA_HEALTH_REPORTED: AtomicBool = AtomicBool::new(false); /// Reports the Ollama-unreachable fallback to Sentry at most once per /// process and publishes an [`EmbeddingModelUnhealthy`] domain event. /// +/// The "once" applies to the Sentry report and the domain event only. The +/// client-facing `user_error` broadcast fires on **every** call, deliberately +/// — see the comment on the first statement. +/// /// Returns `true` on the firing call, `false` afterwards — callers use the /// return value only for logging context. /// /// [`EmbeddingModelUnhealthy`]: crate::core::event_bus::events::DomainEvent::EmbeddingModelUnhealthy fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { + // Deliberately ABOVE the Sentry latch (#5354). `publish_web_channel_event` + // is a `broadcast::send`: with no socket client attached yet it returns Err + // and the event is dropped, with no buffering and no redelivery. Memory is + // constructed early (once per agent in the harness), so the very first + // failed probe usually lands before the renderer's socket is up — under the + // latch that one dropped send would be the only attempt ever made, and the + // UserErrorCenter would stay empty for the whole outage. + // + // Re-broadcasting per failed probe is safe and intended: the panel store + // dedupes on the descriptor's `kind:scope:provider` identity and bumps + // `count`, so repeats collapse into one entry rather than stacking. Only + // the Sentry report below stays once-per-process, which is what the latch + // was introduced for. + surface_local_model_unavailable_to_clients(); + if OLLAMA_HEALTH_REPORTED .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_err() @@ -92,6 +111,20 @@ fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { true } +/// Surface the Ollama-unreachable fallback in every connected client's +/// UserErrorCenter (#5354). +/// +/// `DomainEvent::EmbeddingModelUnhealthy` is published above, but nothing +/// bridges the domain bus to the product UI — `/events/domain` is consumed only +/// by the developer Event Log panel — so that event alone reaches no user. The +/// payload and publisher live in `memory::tree::health::user_error` so this +/// producer and the embed-failure classifier emit one identical, tested shape. +fn surface_local_model_unavailable_to_clients() { + crate::openhuman::memory::tree::health::publish_local_model_unavailable_user_error( + "health_gate", + ); +} + /// Resets the once-per-process Sentry latch. Test-only — any test that /// exercises a fallback path should call this first so it can't be flaked by /// suite ordering (an earlier test that already tripped the latch). @@ -551,6 +584,8 @@ pub fn create_memory_for_migration( #[cfg(test)] mod tests { use super::*; + use crate::openhuman::memory::tree::health::user_error::LOCAL_MODEL_UNAVAILABLE_KIND; + use axum::{routing::get, Json, Router}; use std::ffi::OsString; use std::net::SocketAddr; @@ -837,6 +872,45 @@ mod tests { assert_eq!(redact_ollama_host(""), "unknown"); } + /// #5354 — the client broadcast must NOT ride the once-per-process Sentry + /// latch. + /// + /// `publish_web_channel_event` is a `broadcast::send` with no buffering: if + /// no socket client is attached the event is dropped outright. Memory is + /// built early (once per agent), so the first failed probe typically fires + /// before the renderer connects. Latched, that single dropped send would be + /// the only attempt ever made and the UserErrorCenter would stay empty for + /// the entire outage. Subscribing here proves a second gate call still + /// broadcasts even though its Sentry half is suppressed. + #[test] + fn user_error_broadcast_is_not_suppressed_by_the_sentry_latch() { + let _lock = crate::openhuman::inference::local::inference_test_guard(); + reset_health_gate_for_test(); + + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + assert!( + report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "first call must fire the Sentry report" + ); + assert!( + !report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "second call must suppress the Sentry report" + ); + + // Both calls must still have reached connected clients. + for attempt in 1..=2 { + let event = rx + .try_recv() + .unwrap_or_else(|e| panic!("broadcast {attempt} missing: {e}")); + assert_eq!(event.event, "user_error"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + } + } + /// First call to `report_ollama_health_gate_once` fires the report; /// subsequent calls in the same process must be suppressed. We can't /// observe the Sentry side effect directly here, but the boolean return diff --git a/src/openhuman/memory/tinycortex/queue_driver.rs b/src/openhuman/memory/tinycortex/queue_driver.rs index 33ba4d6b5a..182cf4c2d9 100644 --- a/src/openhuman/memory/tinycortex/queue_driver.rs +++ b/src/openhuman/memory/tinycortex/queue_driver.rs @@ -182,6 +182,17 @@ async fn reembed_collect( } Err(e) => { let failure = health::classify_embed_error(&e); + // Correlation is the re-embed operation identity + typed + // outcome only — never the raw provider error or row content. + log::debug!( + "[tinycortex::queue_driver] action=classify_embed_failure op=reembed \ + label={label} id={id} sig={active_sig} code={} class={}", + failure.code.as_str(), + failure.class.as_str() + ); + // #5354: name the local-runtime fix on the status panel now + // rather than after the retry budget drains. + health::mark_local_model_unavailable_if_applicable(&failure); if matches!(failure.code, health::FailureCode::AuthMissing) { return Err(anyhow::Error::new(failure).context(format!( "reembed: {label} {id} cloud auth missing (sig={active_sig}): {e:#}" diff --git a/src/openhuman/memory/tinycortex/seal.rs b/src/openhuman/memory/tinycortex/seal.rs index 58fcb95d9a..bd3a2178b1 100644 --- a/src/openhuman/memory/tinycortex/seal.rs +++ b/src/openhuman/memory/tinycortex/seal.rs @@ -26,6 +26,19 @@ impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { async fn embed(&self, text: &str) -> Result> { let vector = self.0.embed(text).await.map_err(|error| { let failure = crate::openhuman::memory::tree::health::classify_embed_error(&error); + // Correlation is the embedder identity + typed outcome only — never + // the raw provider error, endpoint, or the text being embedded. + log::debug!( + "[memory_tree::seal] action=classify_embed_failure embedder={} code={} class={}", + self.0.name(), + failure.code.as_str(), + failure.class.as_str() + ); + // #5354: name the local-runtime fix on the status panel now rather + // than after the retry budget drains. + crate::openhuman::memory::tree::health::mark_local_model_unavailable_if_applicable( + &failure, + ); anyhow::Error::new(failure).context(format!("seal embedding failed: {error:#}")) })?; crate::openhuman::memory::tree::score::embed::pack_checked(&vector) diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index 1406ceaed4..dd99ac8429 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -27,6 +27,9 @@ use std::fmt; pub mod doctor; pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, StageHealth}; +pub(crate) mod user_error; +pub(crate) use user_error::publish_local_model_unavailable_user_error; + /// Whether a failure should be retried (`Transient`) or fail fast /// (`Unrecoverable`). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -130,9 +133,19 @@ impl FailureCode { } /// Retry policy for this cause. + /// + /// [`LocalModelUnavailable`](Self::LocalModelUnavailable) is deliberately + /// **transient** even though the user has to act: the condition (Ollama + /// daemon stopped, model not pulled) clears from outside the app, and only + /// transient rows are picked up by `requeue_transient_failed` — the + /// automatic self-healing requeue. Classifying it unrecoverable would park + /// every affected job until someone clicks "Retry failed" by hand, so a + /// user who simply restarts Ollama would never see ingestion resume. pub fn class(self) -> FailureClass { match self { - Self::Transient | Self::ExtractionTimeout => FailureClass::Transient, + Self::Transient | Self::ExtractionTimeout | Self::LocalModelUnavailable => { + FailureClass::Transient + } _ => FailureClass::Unrecoverable, } } @@ -213,6 +226,8 @@ impl PipelineFailure { /// `budget_exhausted` (the managed Voyage route is out of budget; the /// user must bring their own key or top up — retrying won't help). /// - dimension-mismatch text → `embedding_dim_mismatch`. +/// - Ollama daemon-unreachable / model-not-pulled text → +/// `local_model_unavailable`, so the panel names the local-runtime fix. /// - everything else (5xx, timeouts, transport, unparseable) → `transient`, /// so the worker's existing retry-with-backoff still applies. /// @@ -256,6 +271,32 @@ pub fn classify_embed_error_str(msg: &str) -> PipelineFailure { return PipelineFailure::new(FailureCode::AuthMissing).with_detail(truncate_detail(msg)); } + // #5354 — the local Ollama runtime is not usable: the daemon is not + // listening, or the configured embedding model was never pulled. Both are + // emitted by `tinyagents::harness::embeddings::ollama` with the fix already + // in the text: + // + // "ollama embed request failed (is Ollama running at ?): …" + // "Ollama embedding model `` is not installed at . Run `ollama pull ` …" + // + // Neither carries an `Embedding API error ()` shape — the first is a + // transport bail, the second a rewritten 404 — so both used to fall through + // to `Transient` and surface as "a temporary error … will retry + // automatically". That is the wrong remediation: retrying cannot start a + // daemon or pull a model, and the user was never told what to do. Match the + // two shapes explicitly so the status panel renders the + // `local_model_unavailable` remediation instead. The class stays transient + // (see `FailureCode::class`) so jobs auto-resume once Ollama is back. + // + // Anchored on Ollama-specific wording so a generic cloud-embedder transport + // failure ("error sending request for url …") keeps its `Transient` code. + if lower.contains("is ollama running at") + || (lower.contains("ollama embedding model") && lower.contains("is not installed at")) + { + return PipelineFailure::new(FailureCode::LocalModelUnavailable) + .with_detail(truncate_detail(msg)); + } + // Dimension mismatch — the trait validator / CloudEmbedder rejects a // vector whose length isn't EMBEDDING_DIM. Check before status parsing: // it's a 2xx-but-wrong-shape case with no HTTP status to match. @@ -392,6 +433,15 @@ impl DegradedState { use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; static SEMANTIC_RECALL_DEGRADED: AtomicBool = AtomicBool::new(false); +/// Whether the clients have already been told about the *current* local-runtime +/// outage. Separate from [`SEMANTIC_RECALL_DEGRADED`] because "is recall +/// degraded" and "have we announced it" are different questions, and the +/// announcement must be claimed by exactly one caller: the embed path runs +/// concurrently across worker tasks, and a plain read-then-write of the +/// degraded flag lets two of them both decide they are the first +/// (CodeRabbit, #5398). Claimed with `compare_exchange`, released by +/// [`clear_semantic_recall_degraded`] so a later outage announces again. +static LOCAL_MODEL_USER_ERROR_SURFACED: AtomicBool = AtomicBool::new(false); static STRUCTURE_DEGRADED: AtomicBool = AtomicBool::new(false); /// The host filesystem can't service the memory_tree path (EIO/ENOSPC/EROFS). /// Set by the queue worker's host-I/O arm; cleared on the next successful @@ -441,9 +491,74 @@ fn u8_to_code(v: u8) -> Option { /// Record that semantic recall is degraded (embeddings were skipped because no /// usable provider is available). `cause` names why so the status surface can /// lead the user to the fix. Idempotent / cheap; safe to call per embed-stage. +/// +/// The cause is published **before** the flag, and the flag with `Release`, so +/// a concurrent [`current_degraded_state`] that observes the flag set cannot +/// still read the previous degradation's cause and render the wrong +/// remediation (CodeRabbit, #5398). Same ordering in every `mark_*` below. pub fn mark_semantic_recall_degraded(cause: FailureCode) { - SEMANTIC_RECALL_DEGRADED.store(true, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + SEMANTIC_RECALL_DEGRADED.store(true, Ordering::Release); +} + +/// Surface a local-runtime embed failure on the status panel immediately +/// (#5354). No-op for every other cause. +/// +/// The typed `failure_reason` a job persists is only read back once that job +/// settles *terminally* — for a transient class that means after the whole +/// retry budget has drained. The local-runtime causes (Ollama daemon stopped, +/// model never pulled) are user-fixable right now, so waiting out the backoff +/// before naming the fix is exactly the silent window this issue is about. +/// Setting the degraded flag at classification time puts the remediation on +/// the panel from the first failure; the flag self-clears on the next +/// successful embed, so a user who starts Ollama sees it disappear. +/// +/// This is also the **only** producer of the durable UserErrorCenter entry for +/// the "model was never pulled" half of the cause. The embedder health gate in +/// `memory::store::factories` probes `GET /api/tags`, which succeeds whenever +/// the daemon is up — so a running daemon with a missing model never trips that +/// gate and never publishes its `user_error` (codex, #5398). Publishing here +/// covers both halves from the one place that has actually classified the +/// failure. +/// +/// The broadcast fires only on the **transition** into the state, not on every +/// failed embed: the re-embed path calls this per row, and while the panel +/// store dedupes on the descriptor identity, emitting one socket event per +/// chunk would be pointless traffic. +/// +/// The announcement is claimed with a `compare_exchange` on a dedicated latch +/// rather than by reading the degraded flag, so exactly one of several +/// concurrent embed tasks publishes (CodeRabbit, #5398). +/// +/// The latch is released by [`clear_semantic_recall_degraded`], which every +/// write-embedder build calls once per seal / re-embed operation. That is +/// deliberate: it makes the announcement **re-emit once per failing operation +/// until recovery**, so a client that was not yet connected when the outage +/// began still receives it on the next operation. `publish_web_channel_event` +/// is an unbuffered broadcast with no replay, so bounded re-emission is what +/// stands in for one. +pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { + if failure.code != FailureCode::LocalModelUnavailable { + return; + } + // Claim the announcement before mutating anything else. `compare_exchange` + // makes the check-and-claim one indivisible step, so concurrent callers + // cannot all conclude they are first. + let claimed_announcement = LOCAL_MODEL_USER_ERROR_SURFACED + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(); + + log::warn!( + "[memory_tree::health] action=mark_degraded surface=semantic_recall \ + cause=local_model_unavailable class={} announced={}", + failure.class.as_str(), + claimed_announcement + ); + mark_semantic_recall_degraded(FailureCode::LocalModelUnavailable); + + if claimed_announcement { + publish_local_model_unavailable_user_error("embed_classify"); + } } /// Clear the semantic-recall degraded flag — call when an embed succeeds, so @@ -452,13 +567,18 @@ pub fn mark_semantic_recall_degraded(cause: FailureCode) { pub fn clear_semantic_recall_degraded() { SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); + // Release the announcement claim so a later local-runtime failure tells the + // clients again. Called once per write-embedder build, which is what turns + // the single announcement into bounded re-emission until recovery — the + // reason a client connecting mid-outage still gets told. + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Release); } /// Record that wiki structure is degraded (extraction yielded nothing across /// the board). `cause` is typically [`FailureCode::ExtractionTimeout`]. pub fn mark_structure_degraded(cause: FailureCode) { - STRUCTURE_DEGRADED.store(true, Ordering::Relaxed); STRUCTURE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STRUCTURE_DEGRADED.store(true, Ordering::Release); } /// Clear the structure degraded flag — call when extraction yields entities. @@ -474,8 +594,8 @@ pub fn clear_structure_degraded() { /// worker's host-I/O arm so the status surface tells the user to check their /// disk; idempotent / cheap. pub fn mark_storage_degraded(cause: FailureCode) { - STORAGE_DEGRADED.store(true, Ordering::Relaxed); STORAGE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STORAGE_DEGRADED.store(true, Ordering::Release); } /// Clear the storage degraded flag — call when a claim succeeds (the DB opened, @@ -501,6 +621,7 @@ pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { .lock() .unwrap_or_else(|p| p.into_inner()); SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Relaxed); STRUCTURE_DEGRADED.store(false, Ordering::Relaxed); STORAGE_DEGRADED.store(false, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); @@ -513,9 +634,13 @@ pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { /// doctor surface. The `cause` is populated from the last recorded /// [`FailureCode`] when either flag is set. pub fn current_degraded_state() -> DegradedState { - let semantic_recall = SEMANTIC_RECALL_DEGRADED.load(Ordering::Relaxed); - let structure = STRUCTURE_DEGRADED.load(Ordering::Relaxed); - let storage = STORAGE_DEGRADED.load(Ordering::Relaxed); + // Acquire pairs with the Release store on each flag in `mark_*_degraded`, + // which publishes the cause FIRST. A reader that observes a set flag is + // therefore guaranteed to observe the cause that was stored with it, never + // a stale one from a previous degradation (CodeRabbit, #5398). + let semantic_recall = SEMANTIC_RECALL_DEGRADED.load(Ordering::Acquire); + let structure = STRUCTURE_DEGRADED.load(Ordering::Acquire); + let storage = STORAGE_DEGRADED.load(Ordering::Acquire); // Each flag carries its own cause; pick the most actionable one to surface. // Storage degradation is reported first — the host FS can't open the DB, so // it's the foundational failure beneath both recall and structure (no point @@ -542,6 +667,7 @@ pub fn current_degraded_state() -> DegradedState { #[cfg(test)] mod tests { use super::*; + use user_error::LOCAL_MODEL_UNAVAILABLE_KIND; const ALL_CODES: [FailureCode; 11] = [ FailureCode::BudgetExhausted, @@ -571,11 +697,14 @@ mod tests { "{} remediation key has unexpected prefix: {key}", code.as_str() ); - // class() must be total (no panic); Transient + ExtractionTimeout - // are retryable, everything else is unrecoverable. + // class() must be total (no panic); Transient, ExtractionTimeout + // and LocalModelUnavailable are retryable, everything else is + // unrecoverable. let class = code.class(); match code { - FailureCode::Transient | FailureCode::ExtractionTimeout => { + FailureCode::Transient + | FailureCode::ExtractionTimeout + | FailureCode::LocalModelUnavailable => { assert_eq!( class, FailureClass::Transient, @@ -789,6 +918,69 @@ mod tests { assert!(!f.is_unrecoverable()); } + /// #5354 — the Ollama daemon is not listening. Verbatim wording from + /// `tinyagents::harness::embeddings::ollama::OllamaEmbeddingModel::request`. + /// Note the parenthesised hint: `parse_http_status` reads the first `(`, so + /// without an explicit match this fell through to `Transient` and the panel + /// told the user to wait for a retry that can never start their daemon. + #[test] + fn classify_ollama_daemon_down_as_local_model_unavailable() { + let f = classify_embed_error_str( + "ollama embed request failed (is Ollama running at http://localhost:11434?): \ + error sending request for url (http://localhost:11434/api/embed)", + ); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + assert_eq!( + f.remediation_key, + "memory.health.remediation.local_model_unavailable" + ); + // Transient so `requeue_transient_failed` resumes ingestion by itself + // once the user starts Ollama again. + assert!(!f.is_unrecoverable()); + } + + /// #5354 — the model was never pulled. `ollama_http_error` rewrites the + /// 404 into remediation prose, so the `Embedding API error ()` + /// shape the status parser looks for is gone. + #[test] + fn classify_ollama_model_not_pulled_as_local_model_unavailable() { + let f = classify_embed_error_str( + "Ollama embedding model `bge-m3` is not installed at http://localhost:11434. \ + Run `ollama pull bge-m3` or choose an installed embedding model", + ); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + assert!(!f.is_unrecoverable()); + } + + /// The real call path wraps the provider error twice (`ProviderEmbedder` + /// adds "ollama embeddings failed", then the seal/reembed site adds its + /// own context), so the matcher must survive the flattened chain. + #[test] + fn classify_ollama_daemon_down_through_anyhow_context_chain() { + let base = anyhow::anyhow!( + "ollama embed request failed (is Ollama running at http://127.0.0.1:11434?): \ + tcp connect error: Connection refused (os error 61)" + ); + let wrapped = base + .context("ollama embeddings failed") + .context("seal embedding failed"); + let f = classify_embed_error(&wrapped); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + } + + /// Regression guard for the matcher's blast radius: a cloud-embedder + /// transport failure carries no Ollama wording and must keep its generic + /// `Transient` code, or every network blip would start telling users to + /// install Ollama. + #[test] + fn classify_non_ollama_transport_error_stays_transient() { + let f = classify_embed_error_str( + "cloud embeddings failed: error sending request for url \ + (https://api.tinyhumans.ai/openai/v1/embeddings): connection reset", + ); + assert_eq!(f.code, FailureCode::Transient); + } + #[test] fn classify_through_anyhow_context_chain() { // The embed error is commonly `.context()`-wrapped on the way up; @@ -819,6 +1011,173 @@ mod tests { assert!(out.ends_with('…')); } + /// #5354 — a classified local-runtime failure flips the recall flag with + /// its own cause, so the panel names the Ollama fix from the first failed + /// embed instead of waiting out the retry budget. + #[test] + fn local_model_unavailable_marks_recall_degraded_with_its_cause() { + let _g = test_guard(); + + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + let s = current_degraded_state(); + assert!(s.semantic_recall, "recall must be flagged degraded"); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::LocalModelUnavailable) + ); + assert_eq!( + s.cause.as_ref().map(|c| c.remediation_key.as_str()), + Some("memory.health.remediation.local_model_unavailable") + ); + } + + /// #5398 (codex) — the classifier is the ONLY producer of the durable + /// UserErrorCenter entry when Ollama is running but the model was never + /// pulled: the factory health gate probes `GET /api/tags`, which succeeds + /// in that case, so it never fires. It must broadcast on the transition + /// into the state, and must not re-broadcast per failed row afterwards. + #[test] + fn local_model_unavailable_broadcasts_once_per_transition() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // First failure of the outage → clients are told. + mark_local_model_unavailable_if_applicable(&failure); + let event = rx.try_recv().expect("transition must broadcast"); + assert_eq!(event.event, "user_error"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + + // Subsequent failures in the same outage must stay quiet — the re-embed + // path calls this per row. + mark_local_model_unavailable_if_applicable(&failure); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + rx.try_recv().is_err(), + "must not re-broadcast while already degraded for this cause" + ); + + // A successful embed clears the flag; the next outage is a new + // transition and must tell the clients again. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + rx.try_recv().is_ok(), + "a fresh outage after recovery must broadcast again" + ); + } + + /// #5398 (CodeRabbit) — concurrent embed tasks must not all decide they are + /// the first to announce. The claim is a `compare_exchange`, so exactly one + /// of N racing callers publishes. Deterministic: the assertion is on the + /// count of claims, which the atomic makes exact regardless of scheduling. + #[test] + fn concurrent_failures_announce_exactly_once() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + const THREADS: usize = 8; + std::thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|| { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + }); + } + }); + + let mut published = 0; + while rx.try_recv().is_ok() { + published += 1; + } + assert_eq!( + published, 1, + "{THREADS} concurrent failures must yield exactly one announcement" + ); + } + + /// #5398 (CodeRabbit) — `publish_web_channel_event` is an unbuffered + /// broadcast: an announcement made before any client subscribed is dropped + /// with no replay. Bounded re-emission is what covers that, so a client + /// connecting mid-outage must still be told on the next failing operation. + #[test] + fn announcement_reaches_a_client_that_connects_mid_outage() { + let _g = test_guard(); + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // Outage starts with nobody listening — this send goes nowhere. + mark_local_model_unavailable_if_applicable(&failure); + + // The client connects now, after the first failure. + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + assert!( + rx.try_recv().is_err(), + "the pre-subscription announcement is genuinely gone, not buffered" + ); + + // Next seal / re-embed operation builds its write embedder, which + // clears the degraded state, then fails again against the same dead + // runtime. The late subscriber must receive that one. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + + let event = rx + .try_recv() + .expect("a client connecting mid-outage must still be told"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + } + + /// A different active cause must not be mistaken for "already surfaced" — + /// recall degraded for an unrelated reason still needs the local-runtime + /// entry when Ollama then goes away. + #[test] + fn local_model_unavailable_broadcasts_over_a_different_active_cause() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + assert!( + rx.try_recv().is_ok(), + "a cause change into local_model_unavailable is a transition" + ); + } + + /// The helper must stay a no-op for every other cause — a cloud budget or + /// transport failure has nothing to do with the local runtime, and marking + /// recall degraded there would show the wrong remediation. + #[test] + fn other_failure_codes_do_not_mark_recall_degraded() { + let _g = test_guard(); + + for code in [ + FailureCode::Transient, + FailureCode::BudgetExhausted, + FailureCode::AuthMissing, + ] { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new(code)); + assert!( + !current_degraded_state().semantic_recall, + "{} must not flip the recall flag", + code.as_str() + ); + } + } + /// Regression (CodeRabbit): per-flag causes. Mark recall, then structure, /// then clear structure — recall must still report its OWN cause, not the /// (now-cleared) structure cause. With the old single shared slot this diff --git a/src/openhuman/memory/tree/health/user_error.rs b/src/openhuman/memory/tree/health/user_error.rs new file mode 100644 index 0000000000..e22bcd14e2 --- /dev/null +++ b/src/openhuman/memory/tree/health/user_error.rs @@ -0,0 +1,100 @@ +//! Client-facing `user_error` surfacing for memory-pipeline health causes. +//! +//! The memory pipeline already records typed causes for the status panel, but +//! the panel only exists while the user is looking at it. A cause the user must +//! act on outside the app — the local Ollama runtime being unusable — also +//! belongs in the durable UserErrorCenter, which is fed by the metadata-only +//! `user_error` web-channel event the cron scheduler introduced. +//! +//! This module owns that payload and its publisher so the two producers (the +//! embedder health gate in `memory::store::factories` and the failure +//! classifier in the parent module) emit one identical, tested shape. + +use crate::core::socketio::WebChannelEvent; + +/// Stable `error_type` token for the local-embedding-runtime user error. +/// +/// Mirrors the frontend `UserErrorKind` discriminator of the same name; the +/// classifier keys on this exact string, so a drift on either side drops the +/// signal silently. Kept as a constant so the FE-parity test names one symbol. +pub(crate) const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; + +/// `error_source` for everything published here. Drives the panel's scope +/// grouping (`socketService` maps it to the `memory` `UserErrorScope`). +const MEMORY_SOURCE: &str = "memory"; + +/// The metadata-only `user_error` payload for an unusable local embedding +/// runtime. Built separately from the publish so the no-leak contract is +/// unit-testable without a live socket. +/// +/// Metadata only, exactly like the cron producer: a stable `kind` token in +/// `error_type` plus `error_source`, and never the raw provider text, the model +/// id, or the configured endpoint (which can carry a private host). +pub(crate) fn local_model_unavailable_user_error() -> WebChannelEvent { + WebChannelEvent { + event: "user_error".to_string(), + // Every socket auto-joins the "system" room, so this reaches all + // connected clients rather than one chat session. + client_id: "system".to_string(), + error_type: Some(LOCAL_MODEL_UNAVAILABLE_KIND.to_string()), + error_source: Some(MEMORY_SOURCE.to_string()), + ..Default::default() + } +} + +/// Broadcast the local-runtime user error to every connected client. +/// +/// `origin` is a short, non-sensitive tag naming which producer fired +/// (`health_gate` / `embed_classify`) so the two paths stay distinguishable in +/// the log without threading a correlation id through the health API. +pub(crate) fn publish_local_model_unavailable_user_error(origin: &str) { + log::debug!( + "[memory_tree::health] action=surface_user_error kind={LOCAL_MODEL_UNAVAILABLE_KIND} \ + source={MEMORY_SOURCE} origin={origin}" + ); + crate::openhuman::web_chat::publish_web_channel_event(local_model_unavailable_user_error()); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Pins the wire shape the frontend `socketService` handler reads, plus the + /// metadata-only no-leak contract. + #[test] + fn payload_is_metadata_only() { + let event = local_model_unavailable_user_error(); + + assert_eq!(event.event, "user_error"); + // The "system" room is the one every socket auto-joins. + assert_eq!(event.client_id, "system"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + assert_eq!(event.error_source.as_deref(), Some(MEMORY_SOURCE)); + + // Nothing that could carry the base URL, a model id, or raw provider + // prose may ride along. + assert!(event.message.is_none(), "must not carry raw error prose"); + assert!(event.full_response.is_none()); + assert!(event.thread_id.is_empty()); + } + + /// The kind token is a cross-language contract: `app/src/types/userError.ts` + /// declares this exact `UserErrorKind` discriminator and `classify.ts` keys + /// on it. A rename on either side drops the signal with no compile error on + /// either side, so pin the wire string. + #[test] + fn kind_matches_frontend_discriminator() { + assert_eq!(LOCAL_MODEL_UNAVAILABLE_KIND, "local_model_unavailable"); + } + + /// `socketService` only maps `error_source == "memory"` onto the `memory` + /// scope; anything else falls back to the historical `cron` default, which + /// would file this entry under the wrong heading. + #[test] + fn source_matches_frontend_scope_mapping() { + assert_eq!(MEMORY_SOURCE, "memory"); + } +}