From 135339ba3ea9f1f181c2ce0af24f9ee87af68960 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 4 Aug 2026 19:29:37 +0530 Subject: [PATCH 1/4] fix(memory): classify Ollama-unavailable embed failures actionably The memory embedder reported an unusable local Ollama runtime as a generic transient fault. Both failure shapes the embedder produces -- "is Ollama running at ?" (daemon not listening) and "Ollama embedding model `` is not installed at " (model never pulled) -- carry no `Embedding API error ()` envelope, so `classify_embed_error_str` fell through to `Transient` and the memory status panel told the user "a temporary error interrupted memory processing, it will retry automatically". Retrying cannot start a daemon or pull a model, and the actual fix was never named. `FailureCode::LocalModelUnavailable` and its translated remediation already existed but had no producer anywhere in the tree. Match the two shapes explicitly so that code is emitted, and mark the semantic-recall surface degraded at classification time so the remediation appears on the first failed embed rather than after the retry budget drains. Keep the code in the transient retry class. Only transient rows are picked up by `requeue_transient_failed`, the automatic self-healing requeue; classifying it unrecoverable would park every affected job until someone pressed "Retry failed" by hand, so a user who simply restarted Ollama would never see ingestion resume. Also bridge the existing health-gate signal to the UI. The gate published `DomainEvent::EmbeddingModelUnhealthy`, but nothing carries the domain bus to the product UI -- `/events/domain` is read only by the developer Event Log panel -- so that event reached no user. Broadcast the condition over the metadata-only `user_error` web-channel path the cron scheduler already uses, which lands a durable UserErrorCenter entry with a deep link to provider settings. The broadcast deliberately sits above the once-per-process Sentry latch: `publish_web_channel_event` is an unbuffered broadcast send, memory is constructed before the renderer socket attaches, and a single dropped send under the latch would be the only attempt ever made. The panel store dedupes on the descriptor identity, so repeats collapse into one entry. The reported root cause -- a deprecated `/api/embeddings` call -- was not present: the embedder has used `POST /api/embed` with `input` since the provider port, and the deprecated path appears nowhere in the tree. Closes #5354 --- app/src/lib/i18n/ar.ts | 4 + app/src/lib/i18n/bn.ts | 4 + app/src/lib/i18n/de.ts | 4 + app/src/lib/i18n/en.ts | 4 + app/src/lib/i18n/es.ts | 4 + app/src/lib/i18n/fr.ts | 4 + app/src/lib/i18n/hi.ts | 4 + app/src/lib/i18n/id.ts | 4 + app/src/lib/i18n/it.ts | 4 + app/src/lib/i18n/ko.ts | 4 + app/src/lib/i18n/pl.ts | 4 + app/src/lib/i18n/pt.ts | 4 + app/src/lib/i18n/ru.ts | 4 + app/src/lib/i18n/zh-CN.ts | 4 + .../lib/userErrors/__tests__/classify.test.ts | 40 ++++ app/src/lib/userErrors/classify.ts | 37 ++++ .../__tests__/socketService.events.test.ts | 27 +++ app/src/services/socketService.ts | 14 +- app/src/types/userError.ts | 20 +- src/openhuman/memory/store/factories.rs | 136 +++++++++++++ .../memory/tinycortex/queue_driver.rs | 3 + src/openhuman/memory/tinycortex/seal.rs | 5 + src/openhuman/memory/tree/health/mod.rs | 179 +++++++++++++++++- 23 files changed, 505 insertions(+), 12 deletions(-) diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 08cc8320e4..33b95162c2 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -6985,8 +6985,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'مطلوب مفتاح API', 'userErrors.apiKeyMissing.body': 'لا يوجد مفتاح API لمزوّد الذكاء الاصطناعي. أضِفه في إعدادات المزوّد للمتابعة.', + 'userErrors.localModelUnavailable.title': 'النموذج المحلي غير متاح', + 'userErrors.localModelUnavailable.body': + 'إما أن Ollama لا يعمل أو أن النموذج المطلوب لم يُنزَّل. شغّل Ollama ونزّل النموذج، أو حوّل هذه المهمة إلى مزوّد سحابي.', 'userErrors.scope.chat': 'الدردشة', 'userErrors.scope.cron': 'مهمة مجدوَلة', + 'userErrors.scope.memory': 'الذاكرة', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'المبلغ', 'agentWorld.trading.networkLabel': 'الشبكة', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index acb487b620..6629f46a2c 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -7144,8 +7144,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API কী প্রয়োজন', 'userErrors.apiKeyMissing.body': 'আপনার AI প্রদানকারীর কোনো API কী সেট নেই। চালিয়ে যেতে প্রদানকারী সেটিংসে একটি যোগ করুন।', + 'userErrors.localModelUnavailable.title': 'লোকাল মডেল অনুপলব্ধ', + 'userErrors.localModelUnavailable.body': + 'Ollama চলছে না, অথবা প্রয়োজনীয় মডেলটি কখনও পুল করা হয়নি। Ollama চালু করে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', 'userErrors.scope.chat': 'চ্যাট', 'userErrors.scope.cron': 'নির্ধারিত কাজ', + 'userErrors.scope.memory': 'মেমরি', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'পরিমাণ', 'agentWorld.trading.networkLabel': 'নেটওয়ার্ক', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index f2ee98763a..619a55f3d6 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -7345,8 +7345,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API-Schlüssel erforderlich', 'userErrors.apiKeyMissing.body': 'Für deinen KI-Anbieter ist kein API-Schlüssel hinterlegt. Füge in den Anbietereinstellungen einen hinzu, um fortzufahren.', + 'userErrors.localModelUnavailable.title': 'Lokales Modell nicht verfügbar', + 'userErrors.localModelUnavailable.body': + 'Ollama läuft nicht, oder das benötigte Modell wurde nie geladen. Starte Ollama und lade das Modell, oder stelle diese Aufgabe auf einen Cloud-Anbieter um.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Geplante Aufgabe', + 'userErrors.scope.memory': 'Speicher', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Betrag', 'agentWorld.trading.networkLabel': 'Netzwerk', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 5d512e60b1..1cf70c64bb 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -7541,8 +7541,12 @@ const en: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API key required', 'userErrors.apiKeyMissing.body': 'Your AI provider has no API key set. Add one in provider settings to continue.', + 'userErrors.localModelUnavailable.title': 'Local model unavailable', + 'userErrors.localModelUnavailable.body': + 'Ollama is not running, or the model it needs was never pulled. Start Ollama and pull the model, or switch this workload to a cloud provider.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Scheduled job', + 'userErrors.scope.memory': 'Memory', 'memorySources.codingSessions.title': 'Coding-agent sessions', 'memorySources.codingSessions.description': 'Turn your Codex and Claude Code decisions and corrections into private persona memory.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 029a396d34..31d7f4c225 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -7292,8 +7292,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Se requiere clave de API', 'userErrors.apiKeyMissing.body': 'Tu proveedor de IA no tiene una clave de API configurada. Añade una en los ajustes del proveedor para continuar.', + 'userErrors.localModelUnavailable.title': 'Modelo local no disponible', + 'userErrors.localModelUnavailable.body': + 'Ollama no se está ejecutando, o el modelo que necesita nunca se descargó. Inicia Ollama y descarga el modelo, o cambia esta tarea a un proveedor en la nube.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarea programada', + 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importe', 'agentWorld.trading.networkLabel': 'Red', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index aac520de19..fdb4c3e13d 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -7324,8 +7324,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Clé API requise', 'userErrors.apiKeyMissing.body': "Aucune clé API n'est définie pour votre fournisseur d'IA. Ajoutez-en une dans les paramètres du fournisseur pour continuer.", + 'userErrors.localModelUnavailable.title': 'Modèle local indisponible', + 'userErrors.localModelUnavailable.body': + "Ollama n'est pas en cours d'exécution, ou le modèle requis n'a jamais été téléchargé. Lancez Ollama et téléchargez le modèle, ou basculez cette tâche vers un fournisseur cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tâche planifiée', + 'userErrors.scope.memory': 'Mémoire', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Montant', 'agentWorld.trading.networkLabel': 'Réseau', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index edb514179c..985e081d71 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -7142,8 +7142,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API कुंजी आवश्यक', 'userErrors.apiKeyMissing.body': 'आपके AI प्रदाता के लिए कोई API कुंजी सेट नहीं है। जारी रखने के लिए प्रदाता सेटिंग्स में एक जोड़ें।', + 'userErrors.localModelUnavailable.title': 'लोकल मॉडल उपलब्ध नहीं है', + 'userErrors.localModelUnavailable.body': + 'या तो Ollama चल नहीं रहा है, या ज़रूरी मॉडल कभी पुल नहीं किया गया। Ollama शुरू करके मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', 'userErrors.scope.chat': 'चैट', 'userErrors.scope.cron': 'निर्धारित कार्य', + 'userErrors.scope.memory': 'मेमोरी', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'राशि', 'agentWorld.trading.networkLabel': 'नेटवर्क', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 61d7e71ae4..8b484f777b 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -7180,8 +7180,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Kunci API diperlukan', 'userErrors.apiKeyMissing.body': 'Penyedia AI Anda belum memiliki kunci API. Tambahkan satu di pengaturan penyedia untuk melanjutkan.', + 'userErrors.localModelUnavailable.title': 'Model lokal tidak tersedia', + 'userErrors.localModelUnavailable.body': + 'Ollama tidak berjalan, atau model yang dibutuhkan belum pernah diunduh. Jalankan Ollama dan unduh modelnya, atau alihkan tugas ini ke penyedia cloud.', 'userErrors.scope.chat': 'Obrolan', 'userErrors.scope.cron': 'Tugas terjadwal', + 'userErrors.scope.memory': 'Memori', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Jumlah', 'agentWorld.trading.networkLabel': 'Jaringan', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 83fdb97e36..7d4dbfbe3a 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -7277,8 +7277,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Chiave API richiesta', 'userErrors.apiKeyMissing.body': 'Il tuo provider IA non ha una chiave API impostata. Aggiungine una nelle impostazioni del provider per continuare.', + 'userErrors.localModelUnavailable.title': 'Modello locale non disponibile', + 'userErrors.localModelUnavailable.body': + 'Ollama non è in esecuzione, oppure il modello necessario non è mai stato scaricato. Avvia Ollama e scarica il modello, oppure sposta questa attività su un provider cloud.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Attività pianificata', + 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importo', 'agentWorld.trading.networkLabel': 'Rete', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 73ea4dd03b..d79373b7cd 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -7063,8 +7063,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API 키 필요', 'userErrors.apiKeyMissing.body': 'AI 제공업체에 API 키가 설정되지 않았습니다. 제공업체 설정에서 추가하세요.', + 'userErrors.localModelUnavailable.title': '로컬 모델을 사용할 수 없음', + 'userErrors.localModelUnavailable.body': + 'Ollama가 실행 중이 아니거나 필요한 모델을 내려받지 않았습니다. Ollama를 실행하고 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', 'userErrors.scope.chat': '채팅', 'userErrors.scope.cron': '예약된 작업', + 'userErrors.scope.memory': '메모리', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '금액', 'agentWorld.trading.networkLabel': '네트워크', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 35461557d5..c345c9c96f 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -7249,8 +7249,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Wymagany klucz API', 'userErrors.apiKeyMissing.body': 'Twój dostawca AI nie ma ustawionego klucza API. Dodaj go w ustawieniach dostawcy, aby kontynuować.', + 'userErrors.localModelUnavailable.title': 'Model lokalny niedostępny', + 'userErrors.localModelUnavailable.body': + 'Ollama nie działa albo potrzebny model nigdy nie został pobrany. Uruchom Ollamę i pobierz model lub przenieś to zadanie do dostawcy w chmurze.', 'userErrors.scope.chat': 'Czat', 'userErrors.scope.cron': 'Zaplanowane zadanie', + 'userErrors.scope.memory': 'Pamięć', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Kwota', 'agentWorld.trading.networkLabel': 'Sieć', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index b581b7c871..fbee32ed20 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -7260,8 +7260,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Chave de API necessária', 'userErrors.apiKeyMissing.body': 'Seu provedor de IA não tem uma chave de API definida. Adicione uma nas configurações do provedor para continuar.', + 'userErrors.localModelUnavailable.title': 'Modelo local indisponível', + 'userErrors.localModelUnavailable.body': + 'O Ollama não está em execução, ou o modelo necessário nunca foi baixado. Inicie o Ollama e baixe o modelo, ou mude esta tarefa para um provedor na nuvem.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarefa agendada', + 'userErrors.scope.memory': 'Memória', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Valor', 'agentWorld.trading.networkLabel': 'Rede', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index 85585bc64d..47fd167555 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -7222,8 +7222,12 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.title': 'Требуется ключ API', 'userErrors.apiKeyMissing.body': 'У провайдера ИИ не задан ключ API. Добавьте его в настройках провайдера.', + 'userErrors.localModelUnavailable.title': 'Локальная модель недоступна', + 'userErrors.localModelUnavailable.body': + 'Ollama не запущен либо нужная модель не была загружена. Запустите Ollama и загрузите модель или переведите эту задачу на облачного провайдера.', 'userErrors.scope.chat': 'Чат', 'userErrors.scope.cron': 'Запланированная задача', + 'userErrors.scope.memory': 'Память', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Сумма', 'agentWorld.trading.networkLabel': 'Сеть', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index c3b81afc15..3ff556e81c 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -6759,8 +6759,12 @@ const messages: TranslationMap = { 'userErrors.insufficientCredits.body': '提供商额度已用完，请充值或更新 API 密钥。', 'userErrors.apiKeyMissing.title': '需要 API 密钥', 'userErrors.apiKeyMissing.body': '您的 AI 提供商未设置 API 密钥，请在提供商设置中添加以继续。', + 'userErrors.localModelUnavailable.title': '本地模型不可用', + 'userErrors.localModelUnavailable.body': + 'Ollama 未运行，或所需模型从未拉取。请启动 Ollama 并拉取模型，或将此任务切换到云端提供商。', 'userErrors.scope.chat': '聊天', 'userErrors.scope.cron': '定时任务', + 'userErrors.scope.memory': '记忆', // Agent World：Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '金额', 'agentWorld.trading.networkLabel': '网络', diff --git a/app/src/lib/userErrors/__tests__/classify.test.ts b/app/src/lib/userErrors/__tests__/classify.test.ts index 7718409858..fd063673ad 100644 --- a/app/src/lib/userErrors/__tests__/classify.test.ts +++ b/app/src/lib/userErrors/__tests__/classify.test.ts @@ -60,6 +60,46 @@ describe('classifyUserActionableError', () => { expect(classifyUserActionableError({ message: 'Incorrect API key provided' })).toBeNull(); }); + it('classifies an unusable local model runtime (memory user_error kind token)', () => { + // Core's memory embedder health gate emits the stable kind token with + // error_source=memory (#5354); socketService maps that to the memory scope. + const a = classifyUserActionableError({ + errorType: 'local_model_unavailable', + scope: 'memory', + sourceDomain: 'memory', + }); + expect(a?.kind).toBe('local_model_unavailable'); + expect(a?.scope).toBe('memory'); + expect(a?.action).toBe('open_provider_settings'); + expect(a?.titleKey).toBe('userErrors.localModelUnavailable.title'); + expect(a?.bodyKey).toBe('userErrors.localModelUnavailable.body'); + expect(a?.id).toBe(userErrorId('local_model_unavailable', 'memory', undefined)); + + // …and the prose the local embedder / health gate produce. + for (const msg of [ + 'ollama embed request failed (is Ollama running at http://localhost:11434?)', + 'ollama embeddings opted-in but daemon unreachable at http://localhost:11434', + ]) { + expect(classifyUserActionableError({ message: msg })?.kind).toBe('local_model_unavailable'); + } + }); + + it('does NOT promote a bare "daemon unreachable" from another domain', () => { + // Backend connection-health logs use this phrase too. Matching it loosely + // would tell a user with a flaky backend link to install Ollama. The Rust + // matcher anchors on the full producer wording for the same reason. + expect( + classifyUserActionableError({ message: 'backend daemon unreachable at api.tinyhumans.ai' }) + ).toBeNull(); + }); + + it('keeps billing remediation for a credits error that also names Ollama', () => { + // The local-runtime rule is last on purpose: an out-of-credits provider + // must not be told to install Ollama. + const a = classifyUserActionableError({ message: 'ollama proxy requires more credits' }); + expect(a?.kind).toBe('insufficient_credits'); + }); + it('returns null for generic / non-actionable errors and empty input', () => { expect(classifyUserActionableError({ message: GENERIC_MSG })).toBeNull(); expect(classifyUserActionableError({ message: '', errorType: 'inference' })).toBeNull(); diff --git a/app/src/lib/userErrors/classify.ts b/app/src/lib/userErrors/classify.ts index 8f7b6f95c6..4918d2ebb8 100644 --- a/app/src/lib/userErrors/classify.ts +++ b/app/src/lib/userErrors/classify.ts @@ -126,5 +126,42 @@ export function classifyUserActionableError( }; } + // The local model runtime a workload depends on is unusable — Ollama is not + // running, or the configured model was never pulled (#5354). Emitted by core + // as the stable `local_model_unavailable` kind token (memory embedder health + // gate); the prose variants match the wording the local embedder itself + // produces, so a signal that arrives with a message instead of a token still + // classifies. Deliberately last: it is the narrowest rule, and a provider + // that is out of credits should keep its billing remediation even when the + // message happens to name Ollama. + // Each prose matcher is anchored on the FULL producer wording, never a bare + // `daemon unreachable at`: backend connection-health logs in other domains + // emit that phrase too, and promoting one of those into an "install Ollama" + // panel entry would be worse than showing nothing. Mirrors the same + // deliberate anchoring in the Rust matcher `is_ollama_user_config_rejection`. + const isLocalModelUnavailable = + text.includes('local_model_unavailable') || + // tinyagents embedder, daemon not listening. + text.includes('is ollama running') || + // platform doctor report. + text.includes('ollama daemon unreachable') || + // memory embedder health gate. + text.includes('ollama embeddings opted-in but daemon unreachable at'); + if (isLocalModelUnavailable) { + return { + id: userErrorId('local_model_unavailable', scope, signal.provider), + kind: 'local_model_unavailable', + severity: 'warning', + scope, + sourceDomain: signal.sourceDomain, + provider: signal.provider, + titleKey: 'userErrors.localModelUnavailable.title', + bodyKey: 'userErrors.localModelUnavailable.body', + // Local AI + embedding provider selection both live behind + // `/settings/llm`, which redirects to Connections → API keys. + action: 'open_provider_settings', + }; + } + return null; } diff --git a/app/src/services/__tests__/socketService.events.test.ts b/app/src/services/__tests__/socketService.events.test.ts index c61553be3f..ad64504d12 100644 --- a/app/src/services/__tests__/socketService.events.test.ts +++ b/app/src/services/__tests__/socketService.events.test.ts @@ -417,6 +417,33 @@ describe('socketService — agent_meetings event handlers (lines 428-480)', () = expect(signal.message).toBeUndefined(); }); + it('scopes a memory-sourced "user_error" to memory rather than cron (#5354)', async () => { + const { handlers, mockSocket } = buildMockSocket(); + + vi.doMock('socket.io-client', () => ({ io: vi.fn(() => mockSocket) })); + getCoreRpcUrlMock.mockResolvedValue('http://127.0.0.1:7788/rpc'); + ingestRuntimeErrorSignalMock.mockClear(); + + const { socketService } = await import('../socketService'); + socketService.connect('jwt-test-user-error-memory'); + + await pollUntil(() => expect(handlers['user_error']).toBeDefined()); + + // The memory embedder health gate is the second producer of this event. + // Scope is part of the panel entry's dedupe identity, so it must follow the + // producing domain instead of staying pinned to the cron default. + handlers['user_error']!({ error_type: 'local_model_unavailable', error_source: 'memory' }); + + expect(ingestRuntimeErrorSignalMock).toHaveBeenCalledTimes(1); + const signal = ingestRuntimeErrorSignalMock.mock.calls[0]?.[1] as Record; + expect(signal).toMatchObject({ + errorType: 'local_model_unavailable', + scope: 'memory', + sourceDomain: 'memory', + }); + expect(signal.message).toBeUndefined(); + }); + it('defaults "user_error" sourceDomain to cron when error_source is absent (#4165)', async () => { const { handlers, mockSocket } = buildMockSocket(); diff --git a/app/src/services/socketService.ts b/app/src/services/socketService.ts index 84ea20f39b..832ff83f25 100644 --- a/app/src/services/socketService.ts +++ b/app/src/services/socketService.ts @@ -18,6 +18,7 @@ import { upsertChannelConnection } from '../store/channelConnectionsSlice'; import { setBackend } from '../store/connectivitySlice'; import { resetForUser, setSocketIdForUser, setStatusForUser } from '../store/socketSlice'; import type { ChannelAuthMode, ChannelConnectionStatus, ChannelType } from '../types/channels'; +import type { UserErrorScope } from '../types/userError'; import { IS_DEV } from '../utils/config'; import { createSafeLogData, sanitizeError } from '../utils/sanitize'; import { getCoreRpcToken, getCoreRpcUrl } from './coreRpcClient'; @@ -408,17 +409,18 @@ class SocketService { const provider = typeof obj.error_provider === 'string' ? obj.error_provider : undefined; const sourceDomain = typeof obj.error_source === 'string' ? obj.error_source : 'cron'; socketLog('user_error kind=%s source=%s', errorType ?? 'none', sourceDomain); + // Scope groups the entry in the panel and is part of its dedupe identity, + // so it must follow the producing domain. It was pinned to `cron` while + // the scheduler was the only producer; the memory embedder health gate + // (#5354) is the second. Unknown domains keep the historical `cron` + // default rather than widening the scope union from wire data. + const scope: UserErrorScope = sourceDomain === 'memory' ? 'memory' : 'cron'; // Metadata-only ingest: forward the stable kind token + scope ONLY, never // a raw `message` body. The cron producer already omits it, but we drop // any `obj.message` here too so a future/buggy broadcast can't leak raw // provider text into the UI — classify() keys on `errorType` for this // path. Locks the no-leak contract FE-side (CodeRabbit #4169). - ingestRuntimeErrorSignal(store.dispatch, { - errorType, - scope: 'cron', - sourceDomain, - provider, - }); + ingestRuntimeErrorSignal(store.dispatch, { errorType, scope, sourceDomain, provider }); }); // Backend Meet bot events — forwarded from core's DomainEvent bus diff --git a/app/src/types/userError.ts b/app/src/types/userError.ts index 87e4f9c6c7..1ae50ecabc 100644 --- a/app/src/types/userError.ts +++ b/app/src/types/userError.ts @@ -14,10 +14,26 @@ */ /** Stable discriminator the UI branches on. Extend as new states are added. */ -export type UserErrorKind = 'insufficient_credits' | 'budget_exceeded' | 'api_key_missing'; +export type UserErrorKind = + | 'insufficient_credits' + | 'budget_exceeded' + | 'api_key_missing' + /** + * The local model runtime a workload depends on is not usable — Ollama is + * not running, or the configured model was never pulled (#5354). Mirrors the + * core-side `LOCAL_MODEL_UNAVAILABLE_KIND` token. + */ + | 'local_model_unavailable'; /** Where the failure originated, for grouping/labelling (privacy-safe). */ -export type UserErrorScope = 'chat' | 'cron' | 'provider' | 'integration' | 'workspace'; +export type UserErrorScope = + | 'chat' + | 'cron' + | 'provider' + | 'integration' + | 'workspace' + /** Memory ingestion / embedding pipeline. */ + | 'memory'; /** Primary next-step the user can take. `dismiss` is always available too. */ export type UserErrorAction = 'open_billing' | 'open_provider_settings' | 'dismiss'; diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory/store/factories.rs index 75e221cb35..bf6e42d0cf 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory/store/factories.rs @@ -34,11 +34,30 @@ static OLLAMA_HEALTH_REPORTED: AtomicBool = AtomicBool::new(false); /// Reports the Ollama-unreachable fallback to Sentry at most once per /// process and publishes an [`EmbeddingModelUnhealthy`] domain event. /// +/// The "once" applies to the Sentry report and the domain event only. The +/// client-facing `user_error` broadcast fires on **every** call, deliberately +/// — see the comment on the first statement. +/// /// Returns `true` on the firing call, `false` afterwards — callers use the /// return value only for logging context. /// /// [`EmbeddingModelUnhealthy`]: crate::core::event_bus::events::DomainEvent::EmbeddingModelUnhealthy fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { + // Deliberately ABOVE the Sentry latch (#5354). `publish_web_channel_event` + // is a `broadcast::send`: with no socket client attached yet it returns Err + // and the event is dropped, with no buffering and no redelivery. Memory is + // constructed early (once per agent in the harness), so the very first + // failed probe usually lands before the renderer's socket is up — under the + // latch that one dropped send would be the only attempt ever made, and the + // UserErrorCenter would stay empty for the whole outage. + // + // Re-broadcasting per failed probe is safe and intended: the panel store + // dedupes on the descriptor's `kind:scope:provider` identity and bumps + // `count`, so repeats collapse into one entry rather than stacking. Only + // the Sentry report below stays once-per-process, which is what the latch + // was introduced for. + surface_local_model_unavailable_to_clients(); + if OLLAMA_HEALTH_REPORTED .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_err() @@ -92,6 +111,48 @@ fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { true } +/// Stable `error_type` token for the local-embedding-runtime user error. +/// +/// Mirrors the frontend `UserErrorKind` discriminator of the same name; the +/// classifier keys on this exact string, so a drift on either side drops the +/// signal silently. Kept as a constant so the FE-parity test names one symbol. +pub(crate) const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; + +/// Surface the Ollama-unreachable fallback in every connected client's +/// UserErrorCenter (#5354). +/// +/// `DomainEvent::EmbeddingModelUnhealthy` is published above, but nothing +/// bridges the domain bus to the product UI — `/events/domain` is consumed only +/// by the developer Event Log panel — so that event alone reaches no user. This +/// broadcasts the same condition over the web-channel path the cron scheduler +/// already uses for permanent user-config halts (`publish_cron_user_error`), +/// which `socketService` routes into the durable UserErrorCenter entry. +/// +/// Metadata-only, exactly like the cron producer: a stable `kind` token in +/// `error_type` plus `error_source`, and never the raw provider text or the +/// configured endpoint (which can carry a private host). +fn surface_local_model_unavailable_to_clients() { + log::debug!( + "[memory::factory] action=surface_user_error kind={LOCAL_MODEL_UNAVAILABLE_KIND} source=memory" + ); + crate::openhuman::web_chat::publish_web_channel_event(local_model_unavailable_user_error()); +} + +/// The metadata-only `user_error` payload for the local-embedding-runtime +/// fallback. Split out from the publish so the no-leak contract is unit- +/// testable without a live socket. +fn local_model_unavailable_user_error() -> crate::core::socketio::WebChannelEvent { + crate::core::socketio::WebChannelEvent { + event: "user_error".to_string(), + // Every socket auto-joins the "system" room, so this reaches all + // connected clients rather than one chat session. + client_id: "system".to_string(), + error_type: Some(LOCAL_MODEL_UNAVAILABLE_KIND.to_string()), + error_source: Some("memory".to_string()), + ..Default::default() + } +} + /// Resets the once-per-process Sentry latch. Test-only — any test that /// exercises a fallback path should call this first so it can't be flaked by /// suite ordering (an earlier test that already tripped the latch). @@ -837,6 +898,81 @@ mod tests { assert_eq!(redact_ollama_host(""), "unknown"); } + /// #5354 — the `user_error` broadcast that actually reaches the UI. + /// + /// `DomainEvent::EmbeddingModelUnhealthy` is published beside it, but the + /// domain bus has no product-UI consumer, so this web-channel event is the + /// one that lands in the UserErrorCenter. Two things must hold: the wire + /// shape the frontend `socketService` handler reads, and the metadata-only + /// no-leak contract (no raw provider text, no configured endpoint). + #[test] + fn local_model_unavailable_user_error_is_metadata_only() { + let event = local_model_unavailable_user_error(); + + assert_eq!(event.event, "user_error"); + // The "system" room is the one every socket auto-joins. + assert_eq!(event.client_id, "system"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + assert_eq!(event.error_source.as_deref(), Some("memory")); + + // No-leak contract: nothing that could carry the base URL, a model id, + // or raw provider prose may ride along. + assert!(event.message.is_none(), "must not carry raw error prose"); + assert!(event.full_response.is_none()); + assert!(event.thread_id.is_empty()); + } + + /// #5354 — the client broadcast must NOT ride the once-per-process Sentry + /// latch. + /// + /// `publish_web_channel_event` is a `broadcast::send` with no buffering: if + /// no socket client is attached the event is dropped outright. Memory is + /// built early (once per agent), so the first failed probe typically fires + /// before the renderer connects. Latched, that single dropped send would be + /// the only attempt ever made and the UserErrorCenter would stay empty for + /// the entire outage. Subscribing here proves a second gate call still + /// broadcasts even though its Sentry half is suppressed. + #[test] + fn user_error_broadcast_is_not_suppressed_by_the_sentry_latch() { + let _lock = crate::openhuman::inference::local::inference_test_guard(); + reset_health_gate_for_test(); + + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + assert!( + report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "first call must fire the Sentry report" + ); + assert!( + !report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "second call must suppress the Sentry report" + ); + + // Both calls must still have reached connected clients. + for attempt in 1..=2 { + let event = rx + .try_recv() + .unwrap_or_else(|e| panic!("broadcast {attempt} missing: {e}")); + assert_eq!(event.event, "user_error"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + } + } + + /// The kind token is a cross-language contract: `app/src/types/userError.ts` + /// declares this exact `UserErrorKind` discriminator and `classify.ts` keys + /// on it. A rename on either side drops the signal with no compile error on + /// either side, so pin the wire string. + #[test] + fn local_model_unavailable_kind_matches_frontend_discriminator() { + assert_eq!(LOCAL_MODEL_UNAVAILABLE_KIND, "local_model_unavailable"); + } + /// First call to `report_ollama_health_gate_once` fires the report; /// subsequent calls in the same process must be suppressed. We can't /// observe the Sentry side effect directly here, but the boolean return diff --git a/src/openhuman/memory/tinycortex/queue_driver.rs b/src/openhuman/memory/tinycortex/queue_driver.rs index 33ba4d6b5a..e6febc8f24 100644 --- a/src/openhuman/memory/tinycortex/queue_driver.rs +++ b/src/openhuman/memory/tinycortex/queue_driver.rs @@ -182,6 +182,9 @@ async fn reembed_collect( } Err(e) => { let failure = health::classify_embed_error(&e); + // #5354: name the local-runtime fix on the status panel now + // rather than after the retry budget drains. + health::mark_local_model_unavailable_if_applicable(&failure); if matches!(failure.code, health::FailureCode::AuthMissing) { return Err(anyhow::Error::new(failure).context(format!( "reembed: {label} {id} cloud auth missing (sig={active_sig}): {e:#}" diff --git a/src/openhuman/memory/tinycortex/seal.rs b/src/openhuman/memory/tinycortex/seal.rs index 58fcb95d9a..704f2322c3 100644 --- a/src/openhuman/memory/tinycortex/seal.rs +++ b/src/openhuman/memory/tinycortex/seal.rs @@ -26,6 +26,11 @@ impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { async fn embed(&self, text: &str) -> Result> { let vector = self.0.embed(text).await.map_err(|error| { let failure = crate::openhuman::memory::tree::health::classify_embed_error(&error); + // #5354: name the local-runtime fix on the status panel now rather + // than after the retry budget drains. + crate::openhuman::memory::tree::health::mark_local_model_unavailable_if_applicable( + &failure, + ); anyhow::Error::new(failure).context(format!("seal embedding failed: {error:#}")) })?; crate::openhuman::memory::tree::score::embed::pack_checked(&vector) diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index 1406ceaed4..feaeb7e185 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -130,9 +130,19 @@ impl FailureCode { } /// Retry policy for this cause. + /// + /// [`LocalModelUnavailable`](Self::LocalModelUnavailable) is deliberately + /// **transient** even though the user has to act: the condition (Ollama + /// daemon stopped, model not pulled) clears from outside the app, and only + /// transient rows are picked up by `requeue_transient_failed` — the + /// automatic self-healing requeue. Classifying it unrecoverable would park + /// every affected job until someone clicks "Retry failed" by hand, so a + /// user who simply restarts Ollama would never see ingestion resume. pub fn class(self) -> FailureClass { match self { - Self::Transient | Self::ExtractionTimeout => FailureClass::Transient, + Self::Transient | Self::ExtractionTimeout | Self::LocalModelUnavailable => { + FailureClass::Transient + } _ => FailureClass::Unrecoverable, } } @@ -213,6 +223,8 @@ impl PipelineFailure { /// `budget_exhausted` (the managed Voyage route is out of budget; the /// user must bring their own key or top up — retrying won't help). /// - dimension-mismatch text → `embedding_dim_mismatch`. +/// - Ollama daemon-unreachable / model-not-pulled text → +/// `local_model_unavailable`, so the panel names the local-runtime fix. /// - everything else (5xx, timeouts, transport, unparseable) → `transient`, /// so the worker's existing retry-with-backoff still applies. /// @@ -256,6 +268,32 @@ pub fn classify_embed_error_str(msg: &str) -> PipelineFailure { return PipelineFailure::new(FailureCode::AuthMissing).with_detail(truncate_detail(msg)); } + // #5354 — the local Ollama runtime is not usable: the daemon is not + // listening, or the configured embedding model was never pulled. Both are + // emitted by `tinyagents::harness::embeddings::ollama` with the fix already + // in the text: + // + // "ollama embed request failed (is Ollama running at ?): …" + // "Ollama embedding model `` is not installed at . Run `ollama pull ` …" + // + // Neither carries an `Embedding API error ()` shape — the first is a + // transport bail, the second a rewritten 404 — so both used to fall through + // to `Transient` and surface as "a temporary error … will retry + // automatically". That is the wrong remediation: retrying cannot start a + // daemon or pull a model, and the user was never told what to do. Match the + // two shapes explicitly so the status panel renders the + // `local_model_unavailable` remediation instead. The class stays transient + // (see `FailureCode::class`) so jobs auto-resume once Ollama is back. + // + // Anchored on Ollama-specific wording so a generic cloud-embedder transport + // failure ("error sending request for url …") keeps its `Transient` code. + if lower.contains("is ollama running at") + || (lower.contains("ollama embedding model") && lower.contains("is not installed at")) + { + return PipelineFailure::new(FailureCode::LocalModelUnavailable) + .with_detail(truncate_detail(msg)); + } + // Dimension mismatch — the trait validator / CloudEmbedder rejects a // vector whose length isn't EMBEDDING_DIM. Check before status parsing: // it's a 2xx-but-wrong-shape case with no HTTP status to match. @@ -446,6 +484,29 @@ pub fn mark_semantic_recall_degraded(cause: FailureCode) { SEMANTIC_RECALL_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); } +/// Surface a local-runtime embed failure on the status panel immediately +/// (#5354). No-op for every other cause. +/// +/// The typed `failure_reason` a job persists is only read back once that job +/// settles *terminally* — for a transient class that means after the whole +/// retry budget has drained. The local-runtime causes (Ollama daemon stopped, +/// model never pulled) are user-fixable right now, so waiting out the backoff +/// before naming the fix is exactly the silent window this issue is about. +/// Setting the degraded flag at classification time puts the remediation on +/// the panel from the first failure; the flag self-clears on the next +/// successful embed, so a user who starts Ollama sees it disappear. +pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { + if failure.code != FailureCode::LocalModelUnavailable { + return; + } + log::warn!( + "[memory_tree::health] embed failed against the local runtime — marking semantic \ + recall degraded (cause=local_model_unavailable, class={})", + failure.class.as_str() + ); + mark_semantic_recall_degraded(FailureCode::LocalModelUnavailable); +} + /// Clear the semantic-recall degraded flag — call when an embed succeeds, so /// the surface recovers once the user fixes the provider. Clears only this /// flag's cause; a still-active structure degradation keeps its own. @@ -571,11 +632,14 @@ mod tests { "{} remediation key has unexpected prefix: {key}", code.as_str() ); - // class() must be total (no panic); Transient + ExtractionTimeout - // are retryable, everything else is unrecoverable. + // class() must be total (no panic); Transient, ExtractionTimeout + // and LocalModelUnavailable are retryable, everything else is + // unrecoverable. let class = code.class(); match code { - FailureCode::Transient | FailureCode::ExtractionTimeout => { + FailureCode::Transient + | FailureCode::ExtractionTimeout + | FailureCode::LocalModelUnavailable => { assert_eq!( class, FailureClass::Transient, @@ -789,6 +853,69 @@ mod tests { assert!(!f.is_unrecoverable()); } + /// #5354 — the Ollama daemon is not listening. Verbatim wording from + /// `tinyagents::harness::embeddings::ollama::OllamaEmbeddingModel::request`. + /// Note the parenthesised hint: `parse_http_status` reads the first `(`, so + /// without an explicit match this fell through to `Transient` and the panel + /// told the user to wait for a retry that can never start their daemon. + #[test] + fn classify_ollama_daemon_down_as_local_model_unavailable() { + let f = classify_embed_error_str( + "ollama embed request failed (is Ollama running at http://localhost:11434?): \ + error sending request for url (http://localhost:11434/api/embed)", + ); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + assert_eq!( + f.remediation_key, + "memory.health.remediation.local_model_unavailable" + ); + // Transient so `requeue_transient_failed` resumes ingestion by itself + // once the user starts Ollama again. + assert!(!f.is_unrecoverable()); + } + + /// #5354 — the model was never pulled. `ollama_http_error` rewrites the + /// 404 into remediation prose, so the `Embedding API error ()` + /// shape the status parser looks for is gone. + #[test] + fn classify_ollama_model_not_pulled_as_local_model_unavailable() { + let f = classify_embed_error_str( + "Ollama embedding model `bge-m3` is not installed at http://localhost:11434. \ + Run `ollama pull bge-m3` or choose an installed embedding model", + ); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + assert!(!f.is_unrecoverable()); + } + + /// The real call path wraps the provider error twice (`ProviderEmbedder` + /// adds "ollama embeddings failed", then the seal/reembed site adds its + /// own context), so the matcher must survive the flattened chain. + #[test] + fn classify_ollama_daemon_down_through_anyhow_context_chain() { + let base = anyhow::anyhow!( + "ollama embed request failed (is Ollama running at http://127.0.0.1:11434?): \ + tcp connect error: Connection refused (os error 61)" + ); + let wrapped = base + .context("ollama embeddings failed") + .context("seal embedding failed"); + let f = classify_embed_error(&wrapped); + assert_eq!(f.code, FailureCode::LocalModelUnavailable); + } + + /// Regression guard for the matcher's blast radius: a cloud-embedder + /// transport failure carries no Ollama wording and must keep its generic + /// `Transient` code, or every network blip would start telling users to + /// install Ollama. + #[test] + fn classify_non_ollama_transport_error_stays_transient() { + let f = classify_embed_error_str( + "cloud embeddings failed: error sending request for url \ + (https://api.tinyhumans.ai/openai/v1/embeddings): connection reset", + ); + assert_eq!(f.code, FailureCode::Transient); + } + #[test] fn classify_through_anyhow_context_chain() { // The embed error is commonly `.context()`-wrapped on the way up; @@ -819,6 +946,50 @@ mod tests { assert!(out.ends_with('…')); } + /// #5354 — a classified local-runtime failure flips the recall flag with + /// its own cause, so the panel names the Ollama fix from the first failed + /// embed instead of waiting out the retry budget. + #[test] + fn local_model_unavailable_marks_recall_degraded_with_its_cause() { + let _g = test_guard(); + + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + let s = current_degraded_state(); + assert!(s.semantic_recall, "recall must be flagged degraded"); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::LocalModelUnavailable) + ); + assert_eq!( + s.cause.as_ref().map(|c| c.remediation_key.as_str()), + Some("memory.health.remediation.local_model_unavailable") + ); + } + + /// The helper must stay a no-op for every other cause — a cloud budget or + /// transport failure has nothing to do with the local runtime, and marking + /// recall degraded there would show the wrong remediation. + #[test] + fn other_failure_codes_do_not_mark_recall_degraded() { + let _g = test_guard(); + + for code in [ + FailureCode::Transient, + FailureCode::BudgetExhausted, + FailureCode::AuthMissing, + ] { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new(code)); + assert!( + !current_degraded_state().semantic_recall, + "{} must not flip the recall flag", + code.as_str() + ); + } + } + /// Regression (CodeRabbit): per-flag causes. Mark recall, then structure, /// then clear structure — recall must still report its OWN cause, not the /// (now-cleared) structure cause. With the old single shared slot this From ae4b459b974748219162ecc1e29650cfbdac7660 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 4 Aug 2026 20:19:40 +0530 Subject: [PATCH 2/4] fix(memory): surface local-model errors from the classifier too Review follow-ups on #5354. The embedder health gate probes `GET /api/tags`, which succeeds whenever the daemon is up. A running daemon whose embedding model was never pulled therefore never tripped that gate, so the "model not installed" half of `LocalModelUnavailable` set the degraded flag but never raised the durable UserErrorCenter entry. The failure classifier now publishes it as well, covering both halves from the one place that has actually classified the failure. It fires on the transition into the state rather than per failed embed, since the re-embed path calls it per row. The payload and publisher move to a dedicated `health::user_error` module so both producers emit one identical, tested shape. Publish the degraded cause before its flag, and pair a Release store with an Acquire load, so a concurrent status read cannot observe a freshly-set flag alongside the previous degradation's cause and render the wrong remediation. Teach the frontend classifier the second Ollama prose shape ("embedding model ... is not installed at"). It only recognised the daemon-down shape, so a raw message carrying the model-not-pulled text fell through to null despite the comment claiming both were covered. Reword the remediation copy from "never pulled" to "not installed" across all fourteen locales, English included: the backend condition also covers a model that was removed or lives on a different endpoint. Add correlated debug traces at both classification sites, carrying only the embedder identity, operation, and typed outcome. --- app/src/lib/i18n/ar.ts | 2 +- app/src/lib/i18n/bn.ts | 2 +- app/src/lib/i18n/de.ts | 2 +- app/src/lib/i18n/en.ts | 2 +- app/src/lib/i18n/es.ts | 2 +- app/src/lib/i18n/fr.ts | 2 +- app/src/lib/i18n/hi.ts | 2 +- app/src/lib/i18n/id.ts | 2 +- app/src/lib/i18n/it.ts | 2 +- app/src/lib/i18n/ko.ts | 2 +- app/src/lib/i18n/pl.ts | 2 +- app/src/lib/i18n/pt.ts | 2 +- app/src/lib/i18n/ru.ts | 2 +- app/src/lib/i18n/zh-CN.ts | 2 +- .../lib/userErrors/__tests__/classify.test.ts | 6 +- app/src/lib/userErrors/classify.ts | 3 + src/openhuman/memory/store/factories.rs | 76 ++---------- .../memory/tinycortex/queue_driver.rs | 8 ++ src/openhuman/memory/tinycortex/seal.rs | 8 ++ src/openhuman/memory/tree/health/mod.rs | 115 ++++++++++++++++-- .../memory/tree/health/user_error.rs | 100 +++++++++++++++ 21 files changed, 251 insertions(+), 93 deletions(-) create mode 100644 src/openhuman/memory/tree/health/user_error.rs diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 0515bcc395..2f77913278 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -7004,7 +7004,7 @@ const messages: TranslationMap = { 'لا يوجد مفتاح API لمزوّد الذكاء الاصطناعي. أضِفه في إعدادات المزوّد للمتابعة.', 'userErrors.localModelUnavailable.title': 'النموذج المحلي غير متاح', 'userErrors.localModelUnavailable.body': - 'إما أن Ollama لا يعمل أو أن النموذج المطلوب لم يُنزَّل. شغّل Ollama ونزّل النموذج، أو حوّل هذه المهمة إلى مزوّد سحابي.', + 'إما أن Ollama لا يعمل أو أن النموذج المطلوب غير مثبّت. شغّل Ollama ونزّل النموذج، أو حوّل هذه المهمة إلى مزوّد سحابي.', 'userErrors.scope.chat': 'الدردشة', 'userErrors.scope.cron': 'مهمة مجدوَلة', 'userErrors.scope.memory': 'الذاكرة', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index e4ff09d486..f96d2a6ad6 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -7163,7 +7163,7 @@ const messages: TranslationMap = { 'আপনার AI প্রদানকারীর কোনো API কী সেট নেই। চালিয়ে যেতে প্রদানকারী সেটিংসে একটি যোগ করুন।', 'userErrors.localModelUnavailable.title': 'লোকাল মডেল অনুপলব্ধ', 'userErrors.localModelUnavailable.body': - 'Ollama চলছে না, অথবা প্রয়োজনীয় মডেলটি কখনও পুল করা হয়নি। Ollama চালু করে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', + 'Ollama চলছে না, অথবা প্রয়োজনীয় মডেলটি ইনস্টল করা নেই। Ollama চালু করে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', 'userErrors.scope.chat': 'চ্যাট', 'userErrors.scope.cron': 'নির্ধারিত কাজ', 'userErrors.scope.memory': 'মেমরি', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index fa080da764..78e1b12583 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -7364,7 +7364,7 @@ const messages: TranslationMap = { 'Für deinen KI-Anbieter ist kein API-Schlüssel hinterlegt. Füge in den Anbietereinstellungen einen hinzu, um fortzufahren.', 'userErrors.localModelUnavailable.title': 'Lokales Modell nicht verfügbar', 'userErrors.localModelUnavailable.body': - 'Ollama läuft nicht, oder das benötigte Modell wurde nie geladen. Starte Ollama und lade das Modell, oder stelle diese Aufgabe auf einen Cloud-Anbieter um.', + 'Ollama läuft nicht, oder das benötigte Modell ist nicht installiert. Starte Ollama und lade das Modell, oder stelle diese Aufgabe auf einen Cloud-Anbieter um.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Geplante Aufgabe', 'userErrors.scope.memory': 'Speicher', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index b87651cdc8..3a98c33e85 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -7560,7 +7560,7 @@ const en: TranslationMap = { 'Your AI provider has no API key set. Add one in provider settings to continue.', 'userErrors.localModelUnavailable.title': 'Local model unavailable', 'userErrors.localModelUnavailable.body': - 'Ollama is not running, or the model it needs was never pulled. Start Ollama and pull the model, or switch this workload to a cloud provider.', + 'Ollama is not running, or the model it needs is not installed. Start Ollama and pull the model, or switch this workload to a cloud provider.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Scheduled job', 'userErrors.scope.memory': 'Memory', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index aa47dab85d..abaaa4a2ac 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -7311,7 +7311,7 @@ const messages: TranslationMap = { 'Tu proveedor de IA no tiene una clave de API configurada. Añade una en los ajustes del proveedor para continuar.', 'userErrors.localModelUnavailable.title': 'Modelo local no disponible', 'userErrors.localModelUnavailable.body': - 'Ollama no se está ejecutando, o el modelo que necesita nunca se descargó. Inicia Ollama y descarga el modelo, o cambia esta tarea a un proveedor en la nube.', + 'Ollama no se está ejecutando, o el modelo que necesita no está instalado. Inicia Ollama y descarga el modelo, o cambia esta tarea a un proveedor en la nube.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarea programada', 'userErrors.scope.memory': 'Memoria', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index ef2f00775a..5f111a35eb 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -7343,7 +7343,7 @@ const messages: TranslationMap = { "Aucune clé API n'est définie pour votre fournisseur d'IA. Ajoutez-en une dans les paramètres du fournisseur pour continuer.", 'userErrors.localModelUnavailable.title': 'Modèle local indisponible', 'userErrors.localModelUnavailable.body': - "Ollama n'est pas en cours d'exécution, ou le modèle requis n'a jamais été téléchargé. Lancez Ollama et téléchargez le modèle, ou basculez cette tâche vers un fournisseur cloud.", + "Ollama n'est pas en cours d'exécution, ou le modèle requis n'est pas installé. Lancez Ollama et téléchargez le modèle, ou basculez cette tâche vers un fournisseur cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tâche planifiée', 'userErrors.scope.memory': 'Mémoire', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 6ccc5855bd..b342f07907 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -7161,7 +7161,7 @@ const messages: TranslationMap = { 'आपके AI प्रदाता के लिए कोई API कुंजी सेट नहीं है। जारी रखने के लिए प्रदाता सेटिंग्स में एक जोड़ें।', 'userErrors.localModelUnavailable.title': 'लोकल मॉडल उपलब्ध नहीं है', 'userErrors.localModelUnavailable.body': - 'या तो Ollama चल नहीं रहा है, या ज़रूरी मॉडल कभी पुल नहीं किया गया। Ollama शुरू करके मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', + 'या तो Ollama चल नहीं रहा है, या ज़रूरी मॉडल इंस्टॉल नहीं है। Ollama शुरू करके मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', 'userErrors.scope.chat': 'चैट', 'userErrors.scope.cron': 'निर्धारित कार्य', 'userErrors.scope.memory': 'मेमोरी', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 60b7875120..53775a8730 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -7199,7 +7199,7 @@ const messages: TranslationMap = { 'Penyedia AI Anda belum memiliki kunci API. Tambahkan satu di pengaturan penyedia untuk melanjutkan.', 'userErrors.localModelUnavailable.title': 'Model lokal tidak tersedia', 'userErrors.localModelUnavailable.body': - 'Ollama tidak berjalan, atau model yang dibutuhkan belum pernah diunduh. Jalankan Ollama dan unduh modelnya, atau alihkan tugas ini ke penyedia cloud.', + 'Ollama tidak berjalan, atau model yang dibutuhkan belum terpasang. Jalankan Ollama dan unduh modelnya, atau alihkan tugas ini ke penyedia cloud.', 'userErrors.scope.chat': 'Obrolan', 'userErrors.scope.cron': 'Tugas terjadwal', 'userErrors.scope.memory': 'Memori', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index beb1fab19e..1f36ea773c 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -7296,7 +7296,7 @@ const messages: TranslationMap = { 'Il tuo provider IA non ha una chiave API impostata. Aggiungine una nelle impostazioni del provider per continuare.', 'userErrors.localModelUnavailable.title': 'Modello locale non disponibile', 'userErrors.localModelUnavailable.body': - 'Ollama non è in esecuzione, oppure il modello necessario non è mai stato scaricato. Avvia Ollama e scarica il modello, oppure sposta questa attività su un provider cloud.', + 'Ollama non è in esecuzione, oppure il modello necessario non è installato. Avvia Ollama e scarica il modello, oppure sposta questa attività su un provider cloud.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Attività pianificata', 'userErrors.scope.memory': 'Memoria', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 449a06e486..89df54936c 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -7082,7 +7082,7 @@ const messages: TranslationMap = { 'AI 제공업체에 API 키가 설정되지 않았습니다. 제공업체 설정에서 추가하세요.', 'userErrors.localModelUnavailable.title': '로컬 모델을 사용할 수 없음', 'userErrors.localModelUnavailable.body': - 'Ollama가 실행 중이 아니거나 필요한 모델을 내려받지 않았습니다. Ollama를 실행하고 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', + 'Ollama가 실행 중이 아니거나 필요한 모델이 설치되어 있지 않습니다. Ollama를 실행하고 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', 'userErrors.scope.chat': '채팅', 'userErrors.scope.cron': '예약된 작업', 'userErrors.scope.memory': '메모리', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index d17e2bea5d..964197ece7 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -7268,7 +7268,7 @@ const messages: TranslationMap = { 'Twój dostawca AI nie ma ustawionego klucza API. Dodaj go w ustawieniach dostawcy, aby kontynuować.', 'userErrors.localModelUnavailable.title': 'Model lokalny niedostępny', 'userErrors.localModelUnavailable.body': - 'Ollama nie działa albo potrzebny model nigdy nie został pobrany. Uruchom Ollamę i pobierz model lub przenieś to zadanie do dostawcy w chmurze.', + 'Ollama nie działa albo wymagany model nie jest zainstalowany. Uruchom Ollamę i pobierz model lub przenieś to zadanie do dostawcy w chmurze.', 'userErrors.scope.chat': 'Czat', 'userErrors.scope.cron': 'Zaplanowane zadanie', 'userErrors.scope.memory': 'Pamięć', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 3a752ba149..10c1408639 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -7279,7 +7279,7 @@ const messages: TranslationMap = { 'Seu provedor de IA não tem uma chave de API definida. Adicione uma nas configurações do provedor para continuar.', 'userErrors.localModelUnavailable.title': 'Modelo local indisponível', 'userErrors.localModelUnavailable.body': - 'O Ollama não está em execução, ou o modelo necessário nunca foi baixado. Inicie o Ollama e baixe o modelo, ou mude esta tarefa para um provedor na nuvem.', + 'O Ollama não está em execução, ou o modelo necessário não está instalado. Inicie o Ollama e baixe o modelo, ou mude esta tarefa para um provedor na nuvem.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarefa agendada', 'userErrors.scope.memory': 'Memória', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index abb51c3c8e..c22700a068 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -7241,7 +7241,7 @@ const messages: TranslationMap = { 'У провайдера ИИ не задан ключ API. Добавьте его в настройках провайдера.', 'userErrors.localModelUnavailable.title': 'Локальная модель недоступна', 'userErrors.localModelUnavailable.body': - 'Ollama не запущен либо нужная модель не была загружена. Запустите Ollama и загрузите модель или переведите эту задачу на облачного провайдера.', + 'Ollama не запущен либо нужная модель не установлена. Запустите Ollama и загрузите модель или переведите эту задачу на облачного провайдера.', 'userErrors.scope.chat': 'Чат', 'userErrors.scope.cron': 'Запланированная задача', 'userErrors.scope.memory': 'Память', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 7558dc48db..10ebbb1e15 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -6777,7 +6777,7 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.body': '您的 AI 提供商未设置 API 密钥，请在提供商设置中添加以继续。', 'userErrors.localModelUnavailable.title': '本地模型不可用', 'userErrors.localModelUnavailable.body': - 'Ollama 未运行，或所需模型从未拉取。请启动 Ollama 并拉取模型，或将此任务切换到云端提供商。', + 'Ollama 未运行，或所需模型未安装。请启动 Ollama 并拉取模型，或将此任务切换到云端提供商。', 'userErrors.scope.chat': '聊天', 'userErrors.scope.cron': '定时任务', 'userErrors.scope.memory': '记忆', diff --git a/app/src/lib/userErrors/__tests__/classify.test.ts b/app/src/lib/userErrors/__tests__/classify.test.ts index fd063673ad..ca225456e3 100644 --- a/app/src/lib/userErrors/__tests__/classify.test.ts +++ b/app/src/lib/userErrors/__tests__/classify.test.ts @@ -75,9 +75,13 @@ describe('classifyUserActionableError', () => { expect(a?.bodyKey).toBe('userErrors.localModelUnavailable.body'); expect(a?.id).toBe(userErrorId('local_model_unavailable', 'memory', undefined)); - // …and the prose the local embedder / health gate produce. + // …and every prose shape the local embedder / health gate / doctor + // produce. Both Rust-side shapes are covered so the two classifiers stay + // symmetric — daemon-not-listening AND model-never-pulled. for (const msg of [ 'ollama embed request failed (is Ollama running at http://localhost:11434?)', + 'Ollama embedding model `bge-m3` is not installed at http://localhost:11434. Run `ollama pull bge-m3`', + 'Ollama daemon unreachable at http://localhost:11434', 'ollama embeddings opted-in but daemon unreachable at http://localhost:11434', ]) { expect(classifyUserActionableError({ message: msg })?.kind).toBe('local_model_unavailable'); diff --git a/app/src/lib/userErrors/classify.ts b/app/src/lib/userErrors/classify.ts index 4918d2ebb8..5629d11b80 100644 --- a/app/src/lib/userErrors/classify.ts +++ b/app/src/lib/userErrors/classify.ts @@ -143,6 +143,9 @@ export function classifyUserActionableError( text.includes('local_model_unavailable') || // tinyagents embedder, daemon not listening. text.includes('is ollama running') || + // tinyagents embedder, model never pulled — the second shape the Rust + // classifier recognises, so the two sides stay symmetric. + (text.includes('ollama embedding model') && text.includes('is not installed at')) || // platform doctor report. text.includes('ollama daemon unreachable') || // memory embedder health gate. diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory/store/factories.rs index bf6e42d0cf..b2e339d59c 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory/store/factories.rs @@ -111,46 +111,18 @@ fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { true } -/// Stable `error_type` token for the local-embedding-runtime user error. -/// -/// Mirrors the frontend `UserErrorKind` discriminator of the same name; the -/// classifier keys on this exact string, so a drift on either side drops the -/// signal silently. Kept as a constant so the FE-parity test names one symbol. -pub(crate) const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; - /// Surface the Ollama-unreachable fallback in every connected client's /// UserErrorCenter (#5354). /// /// `DomainEvent::EmbeddingModelUnhealthy` is published above, but nothing /// bridges the domain bus to the product UI — `/events/domain` is consumed only -/// by the developer Event Log panel — so that event alone reaches no user. This -/// broadcasts the same condition over the web-channel path the cron scheduler -/// already uses for permanent user-config halts (`publish_cron_user_error`), -/// which `socketService` routes into the durable UserErrorCenter entry. -/// -/// Metadata-only, exactly like the cron producer: a stable `kind` token in -/// `error_type` plus `error_source`, and never the raw provider text or the -/// configured endpoint (which can carry a private host). +/// by the developer Event Log panel — so that event alone reaches no user. The +/// payload and publisher live in `memory::tree::health::user_error` so this +/// producer and the embed-failure classifier emit one identical, tested shape. fn surface_local_model_unavailable_to_clients() { - log::debug!( - "[memory::factory] action=surface_user_error kind={LOCAL_MODEL_UNAVAILABLE_KIND} source=memory" + crate::openhuman::memory::tree::health::publish_local_model_unavailable_user_error( + "health_gate", ); - crate::openhuman::web_chat::publish_web_channel_event(local_model_unavailable_user_error()); -} - -/// The metadata-only `user_error` payload for the local-embedding-runtime -/// fallback. Split out from the publish so the no-leak contract is unit- -/// testable without a live socket. -fn local_model_unavailable_user_error() -> crate::core::socketio::WebChannelEvent { - crate::core::socketio::WebChannelEvent { - event: "user_error".to_string(), - // Every socket auto-joins the "system" room, so this reaches all - // connected clients rather than one chat session. - client_id: "system".to_string(), - error_type: Some(LOCAL_MODEL_UNAVAILABLE_KIND.to_string()), - error_source: Some("memory".to_string()), - ..Default::default() - } } /// Resets the once-per-process Sentry latch. Test-only — any test that @@ -612,6 +584,8 @@ pub fn create_memory_for_migration( #[cfg(test)] mod tests { use super::*; + use crate::openhuman::memory::tree::health::LOCAL_MODEL_UNAVAILABLE_KIND; + use axum::{routing::get, Json, Router}; use std::ffi::OsString; use std::net::SocketAddr; @@ -898,33 +872,6 @@ mod tests { assert_eq!(redact_ollama_host(""), "unknown"); } - /// #5354 — the `user_error` broadcast that actually reaches the UI. - /// - /// `DomainEvent::EmbeddingModelUnhealthy` is published beside it, but the - /// domain bus has no product-UI consumer, so this web-channel event is the - /// one that lands in the UserErrorCenter. Two things must hold: the wire - /// shape the frontend `socketService` handler reads, and the metadata-only - /// no-leak contract (no raw provider text, no configured endpoint). - #[test] - fn local_model_unavailable_user_error_is_metadata_only() { - let event = local_model_unavailable_user_error(); - - assert_eq!(event.event, "user_error"); - // The "system" room is the one every socket auto-joins. - assert_eq!(event.client_id, "system"); - assert_eq!( - event.error_type.as_deref(), - Some(LOCAL_MODEL_UNAVAILABLE_KIND) - ); - assert_eq!(event.error_source.as_deref(), Some("memory")); - - // No-leak contract: nothing that could carry the base URL, a model id, - // or raw provider prose may ride along. - assert!(event.message.is_none(), "must not carry raw error prose"); - assert!(event.full_response.is_none()); - assert!(event.thread_id.is_empty()); - } - /// #5354 — the client broadcast must NOT ride the once-per-process Sentry /// latch. /// @@ -964,15 +911,6 @@ mod tests { } } - /// The kind token is a cross-language contract: `app/src/types/userError.ts` - /// declares this exact `UserErrorKind` discriminator and `classify.ts` keys - /// on it. A rename on either side drops the signal with no compile error on - /// either side, so pin the wire string. - #[test] - fn local_model_unavailable_kind_matches_frontend_discriminator() { - assert_eq!(LOCAL_MODEL_UNAVAILABLE_KIND, "local_model_unavailable"); - } - /// First call to `report_ollama_health_gate_once` fires the report; /// subsequent calls in the same process must be suppressed. We can't /// observe the Sentry side effect directly here, but the boolean return diff --git a/src/openhuman/memory/tinycortex/queue_driver.rs b/src/openhuman/memory/tinycortex/queue_driver.rs index e6febc8f24..182cf4c2d9 100644 --- a/src/openhuman/memory/tinycortex/queue_driver.rs +++ b/src/openhuman/memory/tinycortex/queue_driver.rs @@ -182,6 +182,14 @@ async fn reembed_collect( } Err(e) => { let failure = health::classify_embed_error(&e); + // Correlation is the re-embed operation identity + typed + // outcome only — never the raw provider error or row content. + log::debug!( + "[tinycortex::queue_driver] action=classify_embed_failure op=reembed \ + label={label} id={id} sig={active_sig} code={} class={}", + failure.code.as_str(), + failure.class.as_str() + ); // #5354: name the local-runtime fix on the status panel now // rather than after the retry budget drains. health::mark_local_model_unavailable_if_applicable(&failure); diff --git a/src/openhuman/memory/tinycortex/seal.rs b/src/openhuman/memory/tinycortex/seal.rs index 704f2322c3..bd3a2178b1 100644 --- a/src/openhuman/memory/tinycortex/seal.rs +++ b/src/openhuman/memory/tinycortex/seal.rs @@ -26,6 +26,14 @@ impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { async fn embed(&self, text: &str) -> Result> { let vector = self.0.embed(text).await.map_err(|error| { let failure = crate::openhuman::memory::tree::health::classify_embed_error(&error); + // Correlation is the embedder identity + typed outcome only — never + // the raw provider error, endpoint, or the text being embedded. + log::debug!( + "[memory_tree::seal] action=classify_embed_failure embedder={} code={} class={}", + self.0.name(), + failure.code.as_str(), + failure.class.as_str() + ); // #5354: name the local-runtime fix on the status panel now rather // than after the retry budget drains. crate::openhuman::memory::tree::health::mark_local_model_unavailable_if_applicable( diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index feaeb7e185..267b4b3c66 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -27,6 +27,11 @@ use std::fmt; pub mod doctor; pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, StageHealth}; +mod user_error; +pub(crate) use user_error::{ + publish_local_model_unavailable_user_error, LOCAL_MODEL_UNAVAILABLE_KIND, +}; + /// Whether a failure should be retried (`Transient`) or fail fast /// (`Unrecoverable`). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -479,9 +484,14 @@ fn u8_to_code(v: u8) -> Option { /// Record that semantic recall is degraded (embeddings were skipped because no /// usable provider is available). `cause` names why so the status surface can /// lead the user to the fix. Idempotent / cheap; safe to call per embed-stage. +/// +/// The cause is published **before** the flag, and the flag with `Release`, so +/// a concurrent [`current_degraded_state`] that observes the flag set cannot +/// still read the previous degradation's cause and render the wrong +/// remediation (CodeRabbit, #5398). Same ordering in every `mark_*` below. pub fn mark_semantic_recall_degraded(cause: FailureCode) { - SEMANTIC_RECALL_DEGRADED.store(true, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + SEMANTIC_RECALL_DEGRADED.store(true, Ordering::Release); } /// Surface a local-runtime embed failure on the status panel immediately @@ -495,16 +505,40 @@ pub fn mark_semantic_recall_degraded(cause: FailureCode) { /// Setting the degraded flag at classification time puts the remediation on /// the panel from the first failure; the flag self-clears on the next /// successful embed, so a user who starts Ollama sees it disappear. +/// +/// This is also the **only** producer of the durable UserErrorCenter entry for +/// the "model was never pulled" half of the cause. The embedder health gate in +/// `memory::store::factories` probes `GET /api/tags`, which succeeds whenever +/// the daemon is up — so a running daemon with a missing model never trips that +/// gate and never publishes its `user_error` (codex, #5398). Publishing here +/// covers both halves from the one place that has actually classified the +/// failure. +/// +/// The broadcast fires only on the **transition** into the state, not on every +/// failed embed: the re-embed path calls this per row, and while the panel +/// store dedupes on the descriptor identity, emitting one socket event per +/// chunk would be pointless traffic. The flag doubles as the edge detector. pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { if failure.code != FailureCode::LocalModelUnavailable { return; } + // Edge detection before the mark: already-degraded-for-this-reason means a + // previous failure in this outage already told the clients. + let already_surfaced = SEMANTIC_RECALL_DEGRADED.load(Ordering::Acquire) + && u8_to_code(SEMANTIC_RECALL_CAUSE.load(Ordering::Relaxed)) + == Some(FailureCode::LocalModelUnavailable); + log::warn!( - "[memory_tree::health] embed failed against the local runtime — marking semantic \ - recall degraded (cause=local_model_unavailable, class={})", - failure.class.as_str() + "[memory_tree::health] action=mark_degraded surface=semantic_recall \ + cause=local_model_unavailable class={} transition={}", + failure.class.as_str(), + !already_surfaced ); mark_semantic_recall_degraded(FailureCode::LocalModelUnavailable); + + if !already_surfaced { + publish_local_model_unavailable_user_error("embed_classify"); + } } /// Clear the semantic-recall degraded flag — call when an embed succeeds, so @@ -518,8 +552,8 @@ pub fn clear_semantic_recall_degraded() { /// Record that wiki structure is degraded (extraction yielded nothing across /// the board). `cause` is typically [`FailureCode::ExtractionTimeout`]. pub fn mark_structure_degraded(cause: FailureCode) { - STRUCTURE_DEGRADED.store(true, Ordering::Relaxed); STRUCTURE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STRUCTURE_DEGRADED.store(true, Ordering::Release); } /// Clear the structure degraded flag — call when extraction yields entities. @@ -535,8 +569,8 @@ pub fn clear_structure_degraded() { /// worker's host-I/O arm so the status surface tells the user to check their /// disk; idempotent / cheap. pub fn mark_storage_degraded(cause: FailureCode) { - STORAGE_DEGRADED.store(true, Ordering::Relaxed); STORAGE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STORAGE_DEGRADED.store(true, Ordering::Release); } /// Clear the storage degraded flag — call when a claim succeeds (the DB opened, @@ -574,9 +608,13 @@ pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { /// doctor surface. The `cause` is populated from the last recorded /// [`FailureCode`] when either flag is set. pub fn current_degraded_state() -> DegradedState { - let semantic_recall = SEMANTIC_RECALL_DEGRADED.load(Ordering::Relaxed); - let structure = STRUCTURE_DEGRADED.load(Ordering::Relaxed); - let storage = STORAGE_DEGRADED.load(Ordering::Relaxed); + // Acquire pairs with the Release store on each flag in `mark_*_degraded`, + // which publishes the cause FIRST. A reader that observes a set flag is + // therefore guaranteed to observe the cause that was stored with it, never + // a stale one from a previous degradation (CodeRabbit, #5398). + let semantic_recall = SEMANTIC_RECALL_DEGRADED.load(Ordering::Acquire); + let structure = STRUCTURE_DEGRADED.load(Ordering::Acquire); + let storage = STORAGE_DEGRADED.load(Ordering::Acquire); // Each flag carries its own cause; pick the most actionable one to surface. // Storage degradation is reported first — the host FS can't open the DB, so // it's the foundational failure beneath both recall and structure (no point @@ -969,6 +1007,65 @@ mod tests { ); } + /// #5398 (codex) — the classifier is the ONLY producer of the durable + /// UserErrorCenter entry when Ollama is running but the model was never + /// pulled: the factory health gate probes `GET /api/tags`, which succeeds + /// in that case, so it never fires. It must broadcast on the transition + /// into the state, and must not re-broadcast per failed row afterwards. + #[test] + fn local_model_unavailable_broadcasts_once_per_transition() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // First failure of the outage → clients are told. + mark_local_model_unavailable_if_applicable(&failure); + let event = rx.try_recv().expect("transition must broadcast"); + assert_eq!(event.event, "user_error"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + + // Subsequent failures in the same outage must stay quiet — the re-embed + // path calls this per row. + mark_local_model_unavailable_if_applicable(&failure); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + rx.try_recv().is_err(), + "must not re-broadcast while already degraded for this cause" + ); + + // A successful embed clears the flag; the next outage is a new + // transition and must tell the clients again. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + rx.try_recv().is_ok(), + "a fresh outage after recovery must broadcast again" + ); + } + + /// A different active cause must not be mistaken for "already surfaced" — + /// recall degraded for an unrelated reason still needs the local-runtime + /// entry when Ollama then goes away. + #[test] + fn local_model_unavailable_broadcasts_over_a_different_active_cause() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + assert!( + rx.try_recv().is_ok(), + "a cause change into local_model_unavailable is a transition" + ); + } + /// The helper must stay a no-op for every other cause — a cloud budget or /// transport failure has nothing to do with the local runtime, and marking /// recall degraded there would show the wrong remediation. diff --git a/src/openhuman/memory/tree/health/user_error.rs b/src/openhuman/memory/tree/health/user_error.rs new file mode 100644 index 0000000000..e22bcd14e2 --- /dev/null +++ b/src/openhuman/memory/tree/health/user_error.rs @@ -0,0 +1,100 @@ +//! Client-facing `user_error` surfacing for memory-pipeline health causes. +//! +//! The memory pipeline already records typed causes for the status panel, but +//! the panel only exists while the user is looking at it. A cause the user must +//! act on outside the app — the local Ollama runtime being unusable — also +//! belongs in the durable UserErrorCenter, which is fed by the metadata-only +//! `user_error` web-channel event the cron scheduler introduced. +//! +//! This module owns that payload and its publisher so the two producers (the +//! embedder health gate in `memory::store::factories` and the failure +//! classifier in the parent module) emit one identical, tested shape. + +use crate::core::socketio::WebChannelEvent; + +/// Stable `error_type` token for the local-embedding-runtime user error. +/// +/// Mirrors the frontend `UserErrorKind` discriminator of the same name; the +/// classifier keys on this exact string, so a drift on either side drops the +/// signal silently. Kept as a constant so the FE-parity test names one symbol. +pub(crate) const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; + +/// `error_source` for everything published here. Drives the panel's scope +/// grouping (`socketService` maps it to the `memory` `UserErrorScope`). +const MEMORY_SOURCE: &str = "memory"; + +/// The metadata-only `user_error` payload for an unusable local embedding +/// runtime. Built separately from the publish so the no-leak contract is +/// unit-testable without a live socket. +/// +/// Metadata only, exactly like the cron producer: a stable `kind` token in +/// `error_type` plus `error_source`, and never the raw provider text, the model +/// id, or the configured endpoint (which can carry a private host). +pub(crate) fn local_model_unavailable_user_error() -> WebChannelEvent { + WebChannelEvent { + event: "user_error".to_string(), + // Every socket auto-joins the "system" room, so this reaches all + // connected clients rather than one chat session. + client_id: "system".to_string(), + error_type: Some(LOCAL_MODEL_UNAVAILABLE_KIND.to_string()), + error_source: Some(MEMORY_SOURCE.to_string()), + ..Default::default() + } +} + +/// Broadcast the local-runtime user error to every connected client. +/// +/// `origin` is a short, non-sensitive tag naming which producer fired +/// (`health_gate` / `embed_classify`) so the two paths stay distinguishable in +/// the log without threading a correlation id through the health API. +pub(crate) fn publish_local_model_unavailable_user_error(origin: &str) { + log::debug!( + "[memory_tree::health] action=surface_user_error kind={LOCAL_MODEL_UNAVAILABLE_KIND} \ + source={MEMORY_SOURCE} origin={origin}" + ); + crate::openhuman::web_chat::publish_web_channel_event(local_model_unavailable_user_error()); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Pins the wire shape the frontend `socketService` handler reads, plus the + /// metadata-only no-leak contract. + #[test] + fn payload_is_metadata_only() { + let event = local_model_unavailable_user_error(); + + assert_eq!(event.event, "user_error"); + // The "system" room is the one every socket auto-joins. + assert_eq!(event.client_id, "system"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + assert_eq!(event.error_source.as_deref(), Some(MEMORY_SOURCE)); + + // Nothing that could carry the base URL, a model id, or raw provider + // prose may ride along. + assert!(event.message.is_none(), "must not carry raw error prose"); + assert!(event.full_response.is_none()); + assert!(event.thread_id.is_empty()); + } + + /// The kind token is a cross-language contract: `app/src/types/userError.ts` + /// declares this exact `UserErrorKind` discriminator and `classify.ts` keys + /// on it. A rename on either side drops the signal with no compile error on + /// either side, so pin the wire string. + #[test] + fn kind_matches_frontend_discriminator() { + assert_eq!(LOCAL_MODEL_UNAVAILABLE_KIND, "local_model_unavailable"); + } + + /// `socketService` only maps `error_source == "memory"` onto the `memory` + /// scope; anything else falls back to the historical `cron` default, which + /// would file this entry under the wrong heading. + #[test] + fn source_matches_frontend_scope_mapping() { + assert_eq!(MEMORY_SOURCE, "memory"); + } +} From 566787026c4109648a81b568373032ac10c30210 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 4 Aug 2026 20:54:46 +0530 Subject: [PATCH 3/4] fix(memory): scope the local-model kind constant to its module The constant is only referenced across module boundaries from a test, so re-exporting it from the health module left an unused import in the non-test build, which `clippy -D warnings` rejects. Expose the module itself instead and let the one cross-module consumer path to it. --- src/openhuman/memory/store/factories.rs | 2 +- src/openhuman/memory/tree/health/mod.rs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory/store/factories.rs index b2e339d59c..ca8fee44de 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory/store/factories.rs @@ -584,7 +584,7 @@ pub fn create_memory_for_migration( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::tree::health::LOCAL_MODEL_UNAVAILABLE_KIND; + use crate::openhuman::memory::tree::health::user_error::LOCAL_MODEL_UNAVAILABLE_KIND; use axum::{routing::get, Json, Router}; use std::ffi::OsString; diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index 267b4b3c66..0290cabee0 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -27,10 +27,8 @@ use std::fmt; pub mod doctor; pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, StageHealth}; -mod user_error; -pub(crate) use user_error::{ - publish_local_model_unavailable_user_error, LOCAL_MODEL_UNAVAILABLE_KIND, -}; +pub(crate) mod user_error; +pub(crate) use user_error::publish_local_model_unavailable_user_error; /// Whether a failure should be retried (`Transient`) or fail fast /// (`Unrecoverable`). @@ -641,6 +639,7 @@ pub fn current_degraded_state() -> DegradedState { #[cfg(test)] mod tests { use super::*; + use user_error::LOCAL_MODEL_UNAVAILABLE_KIND; const ALL_CODES: [FailureCode; 11] = [ FailureCode::BudgetExhausted, From f61d55a2c2db48fdb8bced7dd27b7b288b210093 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 4 Aug 2026 21:53:16 +0530 Subject: [PATCH 4/4] fix(memory): claim the local-model announcement atomically Review round two on #5354. The transition check read the degraded flag and then set it, so two concurrent embed tasks could both conclude they were first and both publish. Claim the announcement with a compare_exchange on a dedicated latch instead, making check-and-claim indivisible. The latch is released by `clear_semantic_recall_degraded`, which every write-embedder build calls once per seal or re-embed operation. That turns the single announcement into bounded re-emission until recovery, which is what covers a client that was not connected when the outage began: `publish_web_channel_event` is an unbuffered broadcast with no replay, so an announcement made before anyone subscribed is simply gone. Make the remediation copy endpoint-aware and domain-neutral in all fourteen locales. With more than one Ollama instance reachable, "start Ollama and pull the model" can send the user to repair the wrong machine, and the string is shared by the chat, cron, and memory scopes so it must not describe the work as a task. --- app/src/lib/i18n/ar.ts | 2 +- app/src/lib/i18n/bn.ts | 2 +- app/src/lib/i18n/de.ts | 2 +- app/src/lib/i18n/en.ts | 2 +- app/src/lib/i18n/es.ts | 2 +- app/src/lib/i18n/fr.ts | 2 +- app/src/lib/i18n/hi.ts | 2 +- app/src/lib/i18n/id.ts | 2 +- app/src/lib/i18n/it.ts | 2 +- app/src/lib/i18n/ko.ts | 2 +- app/src/lib/i18n/pl.ts | 2 +- app/src/lib/i18n/pt.ts | 2 +- app/src/lib/i18n/ru.ts | 2 +- app/src/lib/i18n/zh-CN.ts | 2 +- src/openhuman/memory/tree/health/mod.rs | 110 ++++++++++++++++++++++-- 15 files changed, 115 insertions(+), 23 deletions(-) diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 2f77913278..41d4a686dc 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -7004,7 +7004,7 @@ const messages: TranslationMap = { 'لا يوجد مفتاح API لمزوّد الذكاء الاصطناعي. أضِفه في إعدادات المزوّد للمتابعة.', 'userErrors.localModelUnavailable.title': 'النموذج المحلي غير متاح', 'userErrors.localModelUnavailable.body': - 'إما أن Ollama لا يعمل أو أن النموذج المطلوب غير مثبّت. شغّل Ollama ونزّل النموذج، أو حوّل هذه المهمة إلى مزوّد سحابي.', + 'لا يمكن الوصول إلى Ollama على النقطة الطرفية المُهيأة، أو أن النموذج المطلوب غير مثبّت عليها. شغّل Ollama ونزّل النموذج على تلك النقطة الطرفية، أو حوّل هذا العمل إلى مزوّد سحابي.', 'userErrors.scope.chat': 'الدردشة', 'userErrors.scope.cron': 'مهمة مجدوَلة', 'userErrors.scope.memory': 'الذاكرة', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index f96d2a6ad6..3ff6720bb9 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -7163,7 +7163,7 @@ const messages: TranslationMap = { 'আপনার AI প্রদানকারীর কোনো API কী সেট নেই। চালিয়ে যেতে প্রদানকারী সেটিংসে একটি যোগ করুন।', 'userErrors.localModelUnavailable.title': 'লোকাল মডেল অনুপলব্ধ', 'userErrors.localModelUnavailable.body': - 'Ollama চলছে না, অথবা প্রয়োজনীয় মডেলটি ইনস্টল করা নেই। Ollama চালু করে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', + 'কনফিগার করা এন্ডপয়েন্টে Ollama-তে পৌঁছানো যাচ্ছে না, অথবা সেখানে প্রয়োজনীয় মডেলটি ইনস্টল করা নেই। Ollama চালু করে সেই এন্ডপয়েন্টে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', 'userErrors.scope.chat': 'চ্যাট', 'userErrors.scope.cron': 'নির্ধারিত কাজ', 'userErrors.scope.memory': 'মেমরি', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 78e1b12583..0df7caf585 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -7364,7 +7364,7 @@ const messages: TranslationMap = { 'Für deinen KI-Anbieter ist kein API-Schlüssel hinterlegt. Füge in den Anbietereinstellungen einen hinzu, um fortzufahren.', 'userErrors.localModelUnavailable.title': 'Lokales Modell nicht verfügbar', 'userErrors.localModelUnavailable.body': - 'Ollama läuft nicht, oder das benötigte Modell ist nicht installiert. Starte Ollama und lade das Modell, oder stelle diese Aufgabe auf einen Cloud-Anbieter um.', + 'Ollama ist unter dem konfigurierten Endpunkt nicht erreichbar, oder das benötigte Modell ist dort nicht installiert. Starte Ollama und lade das Modell auf diesem Endpunkt, oder verlagere diese Arbeit auf einen Cloud-Anbieter.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Geplante Aufgabe', 'userErrors.scope.memory': 'Speicher', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 3a98c33e85..62f01d45a4 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -7560,7 +7560,7 @@ const en: TranslationMap = { 'Your AI provider has no API key set. Add one in provider settings to continue.', 'userErrors.localModelUnavailable.title': 'Local model unavailable', 'userErrors.localModelUnavailable.body': - 'Ollama is not running, or the model it needs is not installed. Start Ollama and pull the model, or switch this workload to a cloud provider.', + 'Ollama is not reachable at the configured endpoint, or the required model is not installed there. Start Ollama and pull the model at that endpoint, or switch this workload to a cloud provider.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Scheduled job', 'userErrors.scope.memory': 'Memory', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index abaaa4a2ac..7d0485bbbc 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -7311,7 +7311,7 @@ const messages: TranslationMap = { 'Tu proveedor de IA no tiene una clave de API configurada. Añade una en los ajustes del proveedor para continuar.', 'userErrors.localModelUnavailable.title': 'Modelo local no disponible', 'userErrors.localModelUnavailable.body': - 'Ollama no se está ejecutando, o el modelo que necesita no está instalado. Inicia Ollama y descarga el modelo, o cambia esta tarea a un proveedor en la nube.', + 'No se puede acceder a Ollama en el punto de conexión configurado, o el modelo necesario no está instalado allí. Inicia Ollama y descarga el modelo en ese punto de conexión, o cambia este trabajo a un proveedor en la nube.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarea programada', 'userErrors.scope.memory': 'Memoria', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 5f111a35eb..e8d6d81e5c 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -7343,7 +7343,7 @@ const messages: TranslationMap = { "Aucune clé API n'est définie pour votre fournisseur d'IA. Ajoutez-en une dans les paramètres du fournisseur pour continuer.", 'userErrors.localModelUnavailable.title': 'Modèle local indisponible', 'userErrors.localModelUnavailable.body': - "Ollama n'est pas en cours d'exécution, ou le modèle requis n'est pas installé. Lancez Ollama et téléchargez le modèle, ou basculez cette tâche vers un fournisseur cloud.", + "Ollama n'est pas joignable sur le point de terminaison configuré, ou le modèle requis n'y est pas installé. Lancez Ollama et téléchargez le modèle sur ce point de terminaison, ou basculez cette charge de travail vers un fournisseur cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tâche planifiée', 'userErrors.scope.memory': 'Mémoire', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index b342f07907..e6b9949fcb 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -7161,7 +7161,7 @@ const messages: TranslationMap = { 'आपके AI प्रदाता के लिए कोई API कुंजी सेट नहीं है। जारी रखने के लिए प्रदाता सेटिंग्स में एक जोड़ें।', 'userErrors.localModelUnavailable.title': 'लोकल मॉडल उपलब्ध नहीं है', 'userErrors.localModelUnavailable.body': - 'या तो Ollama चल नहीं रहा है, या ज़रूरी मॉडल इंस्टॉल नहीं है। Ollama शुरू करके मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', + 'कॉन्फ़िगर किए गए एंडपॉइंट पर Ollama तक पहुँच नहीं है, या ज़रूरी मॉडल वहाँ इंस्टॉल नहीं है। Ollama शुरू करके उसी एंडपॉइंट पर मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', 'userErrors.scope.chat': 'चैट', 'userErrors.scope.cron': 'निर्धारित कार्य', 'userErrors.scope.memory': 'मेमोरी', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 53775a8730..fe672a6ac2 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -7199,7 +7199,7 @@ const messages: TranslationMap = { 'Penyedia AI Anda belum memiliki kunci API. Tambahkan satu di pengaturan penyedia untuk melanjutkan.', 'userErrors.localModelUnavailable.title': 'Model lokal tidak tersedia', 'userErrors.localModelUnavailable.body': - 'Ollama tidak berjalan, atau model yang dibutuhkan belum terpasang. Jalankan Ollama dan unduh modelnya, atau alihkan tugas ini ke penyedia cloud.', + 'Ollama tidak dapat dijangkau di endpoint yang dikonfigurasi, atau model yang dibutuhkan belum terpasang di sana. Jalankan Ollama dan unduh modelnya di endpoint tersebut, atau alihkan pekerjaan ini ke penyedia cloud.', 'userErrors.scope.chat': 'Obrolan', 'userErrors.scope.cron': 'Tugas terjadwal', 'userErrors.scope.memory': 'Memori', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 1f36ea773c..067596b64f 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -7296,7 +7296,7 @@ const messages: TranslationMap = { 'Il tuo provider IA non ha una chiave API impostata. Aggiungine una nelle impostazioni del provider per continuare.', 'userErrors.localModelUnavailable.title': 'Modello locale non disponibile', 'userErrors.localModelUnavailable.body': - 'Ollama non è in esecuzione, oppure il modello necessario non è installato. Avvia Ollama e scarica il modello, oppure sposta questa attività su un provider cloud.', + "Ollama non è raggiungibile sull'endpoint configurato, oppure il modello necessario non è installato lì. Avvia Ollama e scarica il modello su quell'endpoint, oppure sposta questo lavoro su un provider cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Attività pianificata', 'userErrors.scope.memory': 'Memoria', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 89df54936c..baff9ef2db 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -7082,7 +7082,7 @@ const messages: TranslationMap = { 'AI 제공업체에 API 키가 설정되지 않았습니다. 제공업체 설정에서 추가하세요.', 'userErrors.localModelUnavailable.title': '로컬 모델을 사용할 수 없음', 'userErrors.localModelUnavailable.body': - 'Ollama가 실행 중이 아니거나 필요한 모델이 설치되어 있지 않습니다. Ollama를 실행하고 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', + '구성된 엔드포인트에서 Ollama에 연결할 수 없거나 필요한 모델이 그곳에 설치되어 있지 않습니다. Ollama를 실행하고 해당 엔드포인트에 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', 'userErrors.scope.chat': '채팅', 'userErrors.scope.cron': '예약된 작업', 'userErrors.scope.memory': '메모리', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 964197ece7..e564acfc6b 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -7268,7 +7268,7 @@ const messages: TranslationMap = { 'Twój dostawca AI nie ma ustawionego klucza API. Dodaj go w ustawieniach dostawcy, aby kontynuować.', 'userErrors.localModelUnavailable.title': 'Model lokalny niedostępny', 'userErrors.localModelUnavailable.body': - 'Ollama nie działa albo wymagany model nie jest zainstalowany. Uruchom Ollamę i pobierz model lub przenieś to zadanie do dostawcy w chmurze.', + 'Ollama jest nieosiągalna pod skonfigurowanym punktem końcowym albo wymagany model nie jest tam zainstalowany. Uruchom Ollamę i pobierz model w tym punkcie końcowym lub przenieś tę pracę do dostawcy w chmurze.', 'userErrors.scope.chat': 'Czat', 'userErrors.scope.cron': 'Zaplanowane zadanie', 'userErrors.scope.memory': 'Pamięć', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 10c1408639..c6aecb0882 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -7279,7 +7279,7 @@ const messages: TranslationMap = { 'Seu provedor de IA não tem uma chave de API definida. Adicione uma nas configurações do provedor para continuar.', 'userErrors.localModelUnavailable.title': 'Modelo local indisponível', 'userErrors.localModelUnavailable.body': - 'O Ollama não está em execução, ou o modelo necessário não está instalado. Inicie o Ollama e baixe o modelo, ou mude esta tarefa para um provedor na nuvem.', + 'O Ollama não está acessível no endpoint configurado, ou o modelo necessário não está instalado nele. Inicie o Ollama e baixe o modelo nesse endpoint, ou mude este trabalho para um provedor na nuvem.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarefa agendada', 'userErrors.scope.memory': 'Memória', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index c22700a068..17323f2747 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -7241,7 +7241,7 @@ const messages: TranslationMap = { 'У провайдера ИИ не задан ключ API. Добавьте его в настройках провайдера.', 'userErrors.localModelUnavailable.title': 'Локальная модель недоступна', 'userErrors.localModelUnavailable.body': - 'Ollama не запущен либо нужная модель не установлена. Запустите Ollama и загрузите модель или переведите эту задачу на облачного провайдера.', + 'Ollama недоступен по настроенному адресу, либо нужная модель там не установлена. Запустите Ollama и загрузите модель по этому адресу или переведите эту работу на облачного провайдера.', 'userErrors.scope.chat': 'Чат', 'userErrors.scope.cron': 'Запланированная задача', 'userErrors.scope.memory': 'Память', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 10ebbb1e15..edcd1357c6 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -6777,7 +6777,7 @@ const messages: TranslationMap = { 'userErrors.apiKeyMissing.body': '您的 AI 提供商未设置 API 密钥，请在提供商设置中添加以继续。', 'userErrors.localModelUnavailable.title': '本地模型不可用', 'userErrors.localModelUnavailable.body': - 'Ollama 未运行，或所需模型未安装。请启动 Ollama 并拉取模型，或将此任务切换到云端提供商。', + '无法在配置的端点连接 Ollama，或所需模型未安装在该端点。请启动 Ollama 并在该端点拉取模型，或将此工作切换到云端提供商。', 'userErrors.scope.chat': '聊天', 'userErrors.scope.cron': '定时任务', 'userErrors.scope.memory': '记忆', diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index 0290cabee0..dd99ac8429 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -433,6 +433,15 @@ impl DegradedState { use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; static SEMANTIC_RECALL_DEGRADED: AtomicBool = AtomicBool::new(false); +/// Whether the clients have already been told about the *current* local-runtime +/// outage. Separate from [`SEMANTIC_RECALL_DEGRADED`] because "is recall +/// degraded" and "have we announced it" are different questions, and the +/// announcement must be claimed by exactly one caller: the embed path runs +/// concurrently across worker tasks, and a plain read-then-write of the +/// degraded flag lets two of them both decide they are the first +/// (CodeRabbit, #5398). Claimed with `compare_exchange`, released by +/// [`clear_semantic_recall_degraded`] so a later outage announces again. +static LOCAL_MODEL_USER_ERROR_SURFACED: AtomicBool = AtomicBool::new(false); static STRUCTURE_DEGRADED: AtomicBool = AtomicBool::new(false); /// The host filesystem can't service the memory_tree path (EIO/ENOSPC/EROFS). /// Set by the queue worker's host-I/O arm; cleared on the next successful @@ -515,26 +524,39 @@ pub fn mark_semantic_recall_degraded(cause: FailureCode) { /// The broadcast fires only on the **transition** into the state, not on every /// failed embed: the re-embed path calls this per row, and while the panel /// store dedupes on the descriptor identity, emitting one socket event per -/// chunk would be pointless traffic. The flag doubles as the edge detector. +/// chunk would be pointless traffic. +/// +/// The announcement is claimed with a `compare_exchange` on a dedicated latch +/// rather than by reading the degraded flag, so exactly one of several +/// concurrent embed tasks publishes (CodeRabbit, #5398). +/// +/// The latch is released by [`clear_semantic_recall_degraded`], which every +/// write-embedder build calls once per seal / re-embed operation. That is +/// deliberate: it makes the announcement **re-emit once per failing operation +/// until recovery**, so a client that was not yet connected when the outage +/// began still receives it on the next operation. `publish_web_channel_event` +/// is an unbuffered broadcast with no replay, so bounded re-emission is what +/// stands in for one. pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { if failure.code != FailureCode::LocalModelUnavailable { return; } - // Edge detection before the mark: already-degraded-for-this-reason means a - // previous failure in this outage already told the clients. - let already_surfaced = SEMANTIC_RECALL_DEGRADED.load(Ordering::Acquire) - && u8_to_code(SEMANTIC_RECALL_CAUSE.load(Ordering::Relaxed)) - == Some(FailureCode::LocalModelUnavailable); + // Claim the announcement before mutating anything else. `compare_exchange` + // makes the check-and-claim one indivisible step, so concurrent callers + // cannot all conclude they are first. + let claimed_announcement = LOCAL_MODEL_USER_ERROR_SURFACED + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(); log::warn!( "[memory_tree::health] action=mark_degraded surface=semantic_recall \ - cause=local_model_unavailable class={} transition={}", + cause=local_model_unavailable class={} announced={}", failure.class.as_str(), - !already_surfaced + claimed_announcement ); mark_semantic_recall_degraded(FailureCode::LocalModelUnavailable); - if !already_surfaced { + if claimed_announcement { publish_local_model_unavailable_user_error("embed_classify"); } } @@ -545,6 +567,11 @@ pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { pub fn clear_semantic_recall_degraded() { SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); + // Release the announcement claim so a later local-runtime failure tells the + // clients again. Called once per write-embedder build, which is what turns + // the single announcement into bounded re-emission until recovery — the + // reason a client connecting mid-outage still gets told. + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Release); } /// Record that wiki structure is degraded (extraction yielded nothing across @@ -594,6 +621,7 @@ pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { .lock() .unwrap_or_else(|p| p.into_inner()); SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Relaxed); STRUCTURE_DEGRADED.store(false, Ordering::Relaxed); STORAGE_DEGRADED.store(false, Ordering::Relaxed); SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); @@ -1046,6 +1074,70 @@ mod tests { ); } + /// #5398 (CodeRabbit) — concurrent embed tasks must not all decide they are + /// the first to announce. The claim is a `compare_exchange`, so exactly one + /// of N racing callers publishes. Deterministic: the assertion is on the + /// count of claims, which the atomic makes exact regardless of scheduling. + #[test] + fn concurrent_failures_announce_exactly_once() { + let _g = test_guard(); + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + + const THREADS: usize = 8; + std::thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|| { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + }); + } + }); + + let mut published = 0; + while rx.try_recv().is_ok() { + published += 1; + } + assert_eq!( + published, 1, + "{THREADS} concurrent failures must yield exactly one announcement" + ); + } + + /// #5398 (CodeRabbit) — `publish_web_channel_event` is an unbuffered + /// broadcast: an announcement made before any client subscribed is dropped + /// with no replay. Bounded re-emission is what covers that, so a client + /// connecting mid-outage must still be told on the next failing operation. + #[test] + fn announcement_reaches_a_client_that_connects_mid_outage() { + let _g = test_guard(); + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // Outage starts with nobody listening — this send goes nowhere. + mark_local_model_unavailable_if_applicable(&failure); + + // The client connects now, after the first failure. + let mut rx = crate::openhuman::web_chat::subscribe_web_channel_events(); + assert!( + rx.try_recv().is_err(), + "the pre-subscription announcement is genuinely gone, not buffered" + ); + + // Next seal / re-embed operation builds its write embedder, which + // clears the degraded state, then fails again against the same dead + // runtime. The late subscriber must receive that one. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + + let event = rx + .try_recv() + .expect("a client connecting mid-outage must still be told"); + assert_eq!( + event.error_type.as_deref(), + Some(LOCAL_MODEL_UNAVAILABLE_KIND) + ); + } + /// A different active cause must not be mistaken for "already surfaced" — /// recall degraded for an unrelated reason still needs the local-runtime /// entry when Ollama then goes away.