diff --git a/app/src/App.tsx b/app/src/App.tsx index 09969b5df9..3abeb88b3e 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -34,6 +34,7 @@ import SecurityBanner from './components/SecurityBanner'; import SettingsModal from './components/settings/modal/SettingsModal'; import { resolveSettingsOverlay } from './components/settings/modal/settingsOverlay'; import GlobalUpsellBanner from './components/upsell/GlobalUpsellBanner'; +import MemoryEmbeddingBudgetBanner from './components/upsell/MemoryEmbeddingBudgetBanner'; import UserErrorCenter from './components/userErrors/UserErrorCenter'; import AppWalkthrough from './components/walkthrough/AppWalkthrough'; import { MascotFrameProducer } from './features/meet/MascotFrameProducer'; @@ -292,6 +293,12 @@ export function AppShellDesktop() { const content = (
+ {/* #5324: memory-specific budget warning. Distinct from the banner + above — that one sells a plan upgrade, this one steers to the + embedding fixes (local Ollama / BYO key) that keep memory growing. + Only renders for users whose embeddings actually bill against the + managed budget. */} + {activeProviderAccount && !accountsOverlayOpen && (
diff --git a/app/src/__tests__/App.webviewOverlay.test.tsx b/app/src/__tests__/App.webviewOverlay.test.tsx index 17fd5908fc..5b31bb4585 100644 --- a/app/src/__tests__/App.webviewOverlay.test.tsx +++ b/app/src/__tests__/App.webviewOverlay.test.tsx @@ -109,6 +109,10 @@ vi.mock('../components/layout/shell/SidebarSlot', () => ({ })); vi.mock('../components/OpenhumanLinkModal', () => ({ default: () => null })); vi.mock('../components/upsell/GlobalUpsellBanner', () => ({ default: () => null })); +// Same reason as the banner above: it reads billing usage via `useUsageState`, +// which needs the real `CoreStateProvider` snapshot this suite deliberately +// stubs out. This suite is about webview overlay visibility, not banners. +vi.mock('../components/upsell/MemoryEmbeddingBudgetBanner', () => ({ default: () => null })); vi.mock('../features/meet/MascotFrameProducer', () => ({ MascotFrameProducer: () => null })); vi.mock('../components/walkthrough/AppWalkthrough', () => ({ default: () => null })); diff --git a/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx b/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx index bc1d275e34..457b08f40c 100644 --- a/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx +++ b/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx @@ -22,6 +22,14 @@ import { const mockPipelineStatus = vi.fn(); const mockSetEnabled = vi.fn(); const mockSyncStatusList = vi.fn(); +// #5324: the panel now navigates (budget CTA) and dispatches (escalating the +// blocking cause to the shell-mounted UserErrorCenter). Stub both so the +// suite keeps rendering the panel bare, without a Router or a Redux store. +const mockNavigate = vi.fn(); +const mockDispatch = vi.fn(); + +vi.mock('react-router-dom', () => ({ useNavigate: () => mockNavigate })); +vi.mock('../../store/hooks', () => ({ useAppDispatch: () => mockDispatch })); vi.mock('../../utils/tauriCommands', async importOriginal => { // Inherit everything else (types, sibling wrappers) verbatim so the panel @@ -408,6 +416,140 @@ describe('', () => { }); expect(screen.queryByTestId('memory-tree-blocking-cause')).not.toBeInTheDocument(); }); + + // ── #5324: budget-exhausted state ─────────────────────────────────────── + + /** A pipeline parked on a spent managed embedding budget. */ + function budgetExhaustedPayload() { + return payload({ + status: 'error', + reason: '936 unrecoverable failure(s) need action', + pipeline_jobs: { ready: 12, running: 0, failed: 936 }, + first_blocking_cause: { + code: 'budget_exhausted', + class: 'unrecoverable', + remediation_key: 'memory.health.remediation.budget_exhausted', + }, + }); + } + + it('names the budget-exhausted state instead of a generic error', async () => { + mockPipelineStatus.mockResolvedValueOnce(budgetExhaustedPayload()); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-status-label')).toHaveTextContent( + /embedding budget reached/i + ); + }); + // "Error" alone told the user nothing they could act on. + expect(screen.getByTestId('memory-tree-status-label')).not.toHaveTextContent(/^Error$/); + }); + + it('offers a one-click CTA to the embeddings configuration screen', async () => { + mockPipelineStatus.mockResolvedValueOnce(budgetExhaustedPayload()); + render(); + + const cta = await screen.findByTestId('memory-tree-budget-cta'); + fireEvent.click(cta); + expect(mockNavigate).toHaveBeenCalledWith('/connections?tab=embeddings'); + }); + + it('escalates the budget cause out of this panel into the global error center', async () => { + // The whole point of the issue: a warning only visible inside this panel + // is a warning nobody sees. + mockPipelineStatus.mockResolvedValueOnce(budgetExhaustedPayload()); + render(); + + await waitFor(() => { + expect(mockDispatch).toHaveBeenCalled(); + }); + }); + + it('keeps the paused label when the user paused a tree carrying an old budget failure', async () => { + // `first_blocking_cause` reports the most recent failed job regardless of + // why the pipeline is currently stopped. Relabelling a manually-paused + // tree would hide the real reason it is not running. + mockPipelineStatus.mockResolvedValueOnce( + payload({ + status: 'paused', + is_paused: true, + reason: 'scheduler gate mode = off', + first_blocking_cause: { + code: 'budget_exhausted', + class: 'unrecoverable', + remediation_key: 'memory.health.remediation.budget_exhausted', + }, + }) + ); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-status-label')).toHaveTextContent(/paused/i); + }); + expect(screen.getByTestId('memory-tree-status-label')).not.toHaveTextContent( + /embedding budget reached/i + ); + }); + + it('does not show the budget CTA for other blocking causes', async () => { + mockPipelineStatus.mockResolvedValueOnce( + payload({ + status: 'error', + first_blocking_cause: { + code: 'embedding_dim_mismatch', + class: 'unrecoverable', + remediation_key: 'memory.health.remediation.embedding_dim_mismatch', + }, + }) + ); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-blocking-cause')).toBeInTheDocument(); + }); + expect(screen.queryByTestId('memory-tree-budget-cta')).not.toBeInTheDocument(); + expect(screen.getByTestId('memory-tree-status-label')).toHaveTextContent(/error/i); + }); + + it('handles the legacy degraded.cause payload shape (no first_blocking_cause)', async () => { + // Older/degraded-only payloads carry the cause on `degraded.cause` and omit + // `first_blocking_cause`. The label, CTA, and escalation must all key off + // the same resolved cause, so this shape must behave exactly like the + // `first_blocking_cause` one — not render the banner while silently + // dropping the budget label, CTA, and the global escalation. + mockPipelineStatus.mockResolvedValueOnce( + payload({ + status: 'degraded', + reason: 'queue has not completed any job in 8h — memory is not growing', + degraded: { + semantic_recall: false, + structure: false, + cause: { + code: 'budget_exhausted', + class: 'unrecoverable', + remediation_key: 'memory.health.remediation.budget_exhausted', + }, + }, + }) + ); + render(); + + // Named budget state, not a bare "degraded". + await waitFor(() => { + expect(screen.getByTestId('memory-tree-status-label')).toHaveTextContent( + /embedding budget reached/i + ); + }); + // CTA present… + expect(screen.getByTestId('memory-tree-budget-cta')).toBeInTheDocument(); + // …and the cause still escalates out of this panel. Escalation runs from an + // effect after the status resolves, so wait for it rather than asserting + // synchronously (matches the `first_blocking_cause` escalation test above). + await waitFor(() => { + expect(mockDispatch).toHaveBeenCalled(); + }); + }); }); describe('integration health helpers', () => { diff --git a/app/src/components/intelligence/MemoryTreeStatusPanel.tsx b/app/src/components/intelligence/MemoryTreeStatusPanel.tsx index cde23a4b72..1cef30c3ca 100644 --- a/app/src/components/intelligence/MemoryTreeStatusPanel.tsx +++ b/app/src/components/intelligence/MemoryTreeStatusPanel.tsx @@ -20,8 +20,11 @@ * `settings/panels/AIPanel.tsx` (switch markup). */ import { useCallback, useEffect, useRef, useState } from 'react'; +import { useNavigate } from 'react-router-dom'; import { useT } from '../../lib/i18n/I18nContext'; +import { reportMemoryPipelineFailure } from '../../lib/userErrors/report'; +import { useAppDispatch } from '../../store/hooks'; import type { ToastNotification } from '../../types/intelligence'; import { memorySyncStatusList, @@ -331,9 +334,31 @@ function IntegrationHealthStrip({ */ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { const { t } = useT(); + const navigate = useNavigate(); + const dispatch = useAppDispatch(); const { status, integrations, loading, error, refresh } = useMemoryTreeStatus(); const [toggleBusy, setToggleBusy] = useState(false); + // #002 (FR-004): the single first blocking cause. Prefer the explicit + // `first_blocking_cause`; fall back to the active degradation cause so older + // payload shapes still surface something actionable. Derived ONCE here so the + // escalation, the status label, the CTA, and the banner all key off the same + // cause — a payload that carries only `degraded.cause` (and no + // `first_blocking_cause`) must not render the banner one way while the + // escalation and budget label read a different, empty cause. + const blockingCause = status?.first_blocking_cause ?? status?.degraded?.cause ?? null; + + // #5324: this panel was the ONLY place a budget-exhausted memory pipeline + // was ever surfaced, so users who never opened it experienced weeks of + // silently broken memory. Escalate the typed cause into the shell-mounted + // UserErrorCenter, which stays visible across routes and after the panel + // unmounts. The store dedupes on descriptor id, so polling re-reports bump + // the recurrence count rather than stacking entries. + const blockingCauseCode = blockingCause?.code ?? null; + useEffect(() => { + reportMemoryPipelineFailure(dispatch, blockingCauseCode); + }, [dispatch, blockingCauseCode]); + const handleToggle = useCallback(async () => { if (!status || toggleBusy) return; const nextEnabled = status.is_paused; // currently paused ⇒ enable @@ -352,7 +377,21 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { }, [status, toggleBusy, refresh, onToast, t]); const statusKind = status?.status ?? 'idle'; + // #5324: "Error — 936 unrecoverable failures need action" told the user + // nothing they could act on. When the blocking cause is a spent embedding + // budget, name that state exactly. Derived here rather than as a new wire + // status so older clients keep deserialising the payload unchanged and the + // existing `status` precedence rules stay untouched. + // + // Scoped to the states the budget actually explains. `first_blocking_cause` + // reports the most recent failed job even when the user has since paused the + // tree themselves, so without this guard a manually-paused tree carrying an + // old budget failure would be relabelled and hide the real reason it stopped. + const isBudgetExhausted = + blockingCause?.code === 'budget_exhausted' && + (statusKind === 'error' || statusKind === 'degraded'); const statusLabel: string = (() => { + if (isBudgetExhausted) return t('memoryTree.status.statusBudgetExhausted'); switch (statusKind) { case 'running': return t('memoryTree.status.statusRunning'); @@ -370,11 +409,8 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { } })(); - // #002 (FR-004): the single first blocking cause, rendered verbatim with a - // localized remediation. Prefer the explicit `first_blocking_cause`; fall - // back to the active degradation cause so older payload shapes still surface - // something actionable. - const blockingCause = status?.first_blocking_cause ?? status?.degraded?.cause ?? null; + // `blockingCause` (derived above) is rendered verbatim in the banner below + // with a localized remediation. const degraded = status?.degraded; const checked = !(status?.is_paused ?? false); @@ -419,6 +455,23 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) {
{t(blockingCause.remediation_key, t('memory.health.remediation.unknown'))}
+ {/* #5324: the remediation text names the fix ("set up local Ollama + embeddings or add your own key") but left the user to find that + screen themselves. One click, no embeddings knowledge required. */} + {isBudgetExhausted ? ( +
+ +
+ ) : null} {degraded?.semantic_recall || degraded?.structure ? (
{degraded?.semantic_recall ? ( diff --git a/app/src/components/upsell/MemoryEmbeddingBudgetBanner.tsx b/app/src/components/upsell/MemoryEmbeddingBudgetBanner.tsx new file mode 100644 index 0000000000..e559d00bb8 --- /dev/null +++ b/app/src/components/upsell/MemoryEmbeddingBudgetBanner.tsx @@ -0,0 +1,106 @@ +/** + * Memory-embedding budget banner (#5324). + * + * Shell-mounted beside {@link GlobalUpsellBanner}, so the warning reaches the + * user on whatever screen they are on rather than waiting for them to open + * Memory Tree settings — the failure mode this issue is about. + * + * Escalation, matching the issue's acceptance criteria: + * + * | Consumption | Behaviour | + * | ----------- | ------------------------------------------------------ | + * | ≥ 75% | dismissible warning — "set up local embeddings or …" | + * | ≥ 90% | non-dismissible warning with the same CTA | + * | exhausted | non-dismissible, and memory has already stopped growing | + * + * Dismissal is per-session and per-level on purpose: dismissing the 75% + * warning must not also silence the 90% escalation, or the user is back to a + * silent failure. It is deliberately not persisted — a warning that survives + * a restart it no longer applies to is worse than one shown twice. + * + * The CTA deep-links to the embeddings configuration screen. It never asks + * the user to know what an embedding is: the copy names the two fixes (local + * Ollama, own API key) and the button takes them to the one screen where both + * are done. + */ +import { useEffect, useState } from 'react'; +import { useNavigate } from 'react-router-dom'; + +import { + type EmbeddingBudgetLevel, + useEmbeddingBudgetState, +} from '../../hooks/useEmbeddingBudgetState'; +import { useT } from '../../lib/i18n/I18nContext'; +import { showNativeNotification } from '../../lib/nativeNotifications/tauriBridge'; +import UpsellBanner from './UpsellBanner'; + +/** Where both remediations (local Ollama, BYO key) are configured. */ +export const EMBEDDINGS_SETTINGS_ROUTE = '/connections?tab=embeddings'; + +/** Only the early warning can be silenced; escalations cannot. */ +function isDismissible(level: EmbeddingBudgetLevel): boolean { + return level === 'warn'; +} + +/** + * Module-scoped so the OS notification fires at most once per app session. + * The banner re-renders on every usage poll; without this the user would get + * a notification every 60s, which trains them to mute the app. + */ +let nativeNotificationSent = false; + +/** Test seam — resets the once-per-session latch. */ +export function __resetNativeNotificationLatchForTests() { + nativeNotificationSent = false; +} + +export default function MemoryEmbeddingBudgetBanner() { + const { t } = useT(); + const navigate = useNavigate(); + const { level, pct } = useEmbeddingBudgetState(); + const [dismissedLevel, setDismissedLevel] = useState(null); + + // Push an OS-level notification the first time the budget is actually spent. + // The in-app banner and UserErrorCenter only reach a user who is looking at + // the app; the whole point of this issue is that memory broke while nobody + // was looking. Email is the backend's job (tracked separately) — this is the + // client-side half. + // + // Fires only on `exhausted`, never on the 75%/90% warnings: those are not + // yet a broken state, and an OS notification for them would be noise. + useEffect(() => { + if (level !== 'exhausted' || nativeNotificationSent) return; + nativeNotificationSent = true; + void showNativeNotification({ + title: t('memoryBudget.exhaustedTitle'), + body: t('memoryBudget.exhaustedMessage'), + tag: 'memory-embedding-budget-exhausted', + }); + }, [level, t]); + + if (level === 'none') return null; + if (dismissedLevel === level) return null; + + const isExhausted = level === 'exhausted'; + const title = isExhausted ? t('memoryBudget.exhaustedTitle') : t('memoryBudget.approachingTitle'); + const message = isExhausted + ? t('memoryBudget.exhaustedMessage') + : t('memoryBudget.approachingMessage').replace('{pct}', String(pct)); + + return ( +
+ setDismissedLevel(level)} + onCtaClick={() => { + navigate(EMBEDDINGS_SETTINGS_ROUTE); + }} + /> +
+ ); +} diff --git a/app/src/components/upsell/__tests__/MemoryEmbeddingBudgetBanner.test.tsx b/app/src/components/upsell/__tests__/MemoryEmbeddingBudgetBanner.test.tsx new file mode 100644 index 0000000000..f20dc663e6 --- /dev/null +++ b/app/src/components/upsell/__tests__/MemoryEmbeddingBudgetBanner.test.tsx @@ -0,0 +1,165 @@ +/** + * MemoryEmbeddingBudgetBanner tests (#5324). + * + * The behaviours worth pinning are the ones that decide whether a user finds + * out their memory stopped growing: which levels render, which can be + * silenced, where the CTA goes, and that the OS notification fires exactly + * once for an exhausted budget. + */ +import { fireEvent, render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import MemoryEmbeddingBudgetBanner, { + __resetNativeNotificationLatchForTests, + EMBEDDINGS_SETTINGS_ROUTE, +} from '../MemoryEmbeddingBudgetBanner'; + +const mockUseEmbeddingBudgetState = vi.hoisted(() => + vi.fn(() => ({ + level: 'none' as 'none' | 'warn' | 'urgent' | 'exhausted', + pct: 0, + isLoading: false, + isManagedEmbeddings: true, + })) +); +const mockNavigate = vi.hoisted(() => vi.fn()); +const mockShowNativeNotification = vi.hoisted(() => + vi.fn(() => Promise.resolve({ delivered: true })) +); + +vi.mock('../../../hooks/useEmbeddingBudgetState', () => ({ + useEmbeddingBudgetState: mockUseEmbeddingBudgetState, +})); +vi.mock('react-router-dom', () => ({ useNavigate: () => mockNavigate })); +vi.mock('../../../lib/nativeNotifications/tauriBridge', () => ({ + showNativeNotification: mockShowNativeNotification, +})); +vi.mock('../../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (key: string) => key }) })); + +vi.mock('../UpsellBanner', () => ({ + default: ({ + title, + message, + ctaLabel, + onCtaClick, + dismissible, + onDismiss, + }: { + title: string; + message: string; + ctaLabel?: string; + onCtaClick?: () => void; + dismissible?: boolean; + onDismiss?: () => void; + }) => ( +
+ {title} + {message} + + {dismissible && ( + + )} +
+ ), +})); + +function setLevel(level: 'none' | 'warn' | 'urgent' | 'exhausted', pct = 0) { + mockUseEmbeddingBudgetState.mockReturnValue({ + level, + pct, + isLoading: false, + isManagedEmbeddings: true, + }); +} + +describe('MemoryEmbeddingBudgetBanner', () => { + beforeEach(() => { + mockNavigate.mockReset(); + mockShowNativeNotification.mockClear(); + __resetNativeNotificationLatchForTests(); + setLevel('none'); + }); + + it('renders nothing below the warning threshold', () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it('shows a dismissible warning at the 75% level', () => { + setLevel('warn', 76); + render(); + expect(screen.getByTestId('upsell-banner')).toHaveAttribute('data-dismissible', 'true'); + expect(screen.getByText('memoryBudget.approachingTitle')).toBeInTheDocument(); + }); + + it('interpolates the consumed percentage into the warning copy', () => { + setLevel('warn', 76); + render(); + // The mocked translator returns the key, so the replace() target is + // absent — what matters is that the component does not crash and renders + // the approaching message slot. + expect(screen.getByTestId('banner-message')).toBeInTheDocument(); + }); + + it('makes the 90% escalation non-dismissible', () => { + setLevel('urgent', 92); + render(); + expect(screen.getByTestId('upsell-banner')).toHaveAttribute('data-dismissible', 'false'); + expect(screen.queryByTestId('banner-dismiss')).toBeNull(); + }); + + it('makes the exhausted state non-dismissible and names it distinctly', () => { + setLevel('exhausted', 100); + render(); + expect(screen.getByTestId('upsell-banner')).toHaveAttribute('data-dismissible', 'false'); + expect(screen.getByText('memoryBudget.exhaustedTitle')).toBeInTheDocument(); + }); + + it('hides the warning once dismissed', () => { + setLevel('warn', 76); + render(); + fireEvent.click(screen.getByTestId('banner-dismiss')); + expect(screen.queryByTestId('upsell-banner')).toBeNull(); + }); + + it('re-shows at the next level after the warning was dismissed', () => { + setLevel('warn', 76); + const { rerender } = render(); + fireEvent.click(screen.getByTestId('banner-dismiss')); + expect(screen.queryByTestId('upsell-banner')).toBeNull(); + + // Dismissing 75% must not silence the 90% escalation — otherwise the + // user is back to a silent failure. + setLevel('urgent', 92); + rerender(); + expect(screen.getByTestId('upsell-banner')).toBeInTheDocument(); + }); + + it('deep-links the CTA to the embeddings configuration screen', () => { + setLevel('exhausted', 100); + render(); + fireEvent.click(screen.getByText('memoryBudget.cta')); + expect(mockNavigate).toHaveBeenCalledWith(EMBEDDINGS_SETTINGS_ROUTE); + }); + + it('fires an OS notification once when the budget is exhausted', () => { + setLevel('exhausted', 100); + const { rerender } = render(); + expect(mockShowNativeNotification).toHaveBeenCalledTimes(1); + expect(mockShowNativeNotification).toHaveBeenCalledWith( + expect.objectContaining({ tag: 'memory-embedding-budget-exhausted' }) + ); + + // The usage hook re-renders on every poll; the notification must not. + rerender(); + expect(mockShowNativeNotification).toHaveBeenCalledTimes(1); + }); + + it('does not fire an OS notification for the pre-exhaustion warnings', () => { + setLevel('urgent', 92); + render(); + expect(mockShowNativeNotification).not.toHaveBeenCalled(); + }); +}); diff --git a/app/src/components/userErrors/UserErrorCenter.tsx b/app/src/components/userErrors/UserErrorCenter.tsx index caea70c322..8594c23f06 100644 --- a/app/src/components/userErrors/UserErrorCenter.tsx +++ b/app/src/components/userErrors/UserErrorCenter.tsx @@ -28,12 +28,17 @@ import type { UserActionableError, UserErrorAction } from '../../types/userError const ACTION_ROUTE: Record, string> = { open_billing: '/settings/billing', open_provider_settings: '/settings/llm', + // #5324: both memory-embedding remediations (local Ollama, BYO key) live on + // this one screen, so a single CTA covers them without the user needing to + // know which one applies. + open_embeddings_settings: '/connections?tab=embeddings', }; /** i18n key for each primary action's button label. */ const ACTION_LABEL_KEY: Record, string> = { open_billing: 'userErrors.action.openBilling', open_provider_settings: 'userErrors.action.openProviderSettings', + open_embeddings_settings: 'userErrors.action.openEmbeddingsSettings', }; // Wall-clock read for the resolve/dismiss timestamps. Defined at module scope diff --git a/app/src/hooks/__tests__/useEmbeddingBudgetState.test.ts b/app/src/hooks/__tests__/useEmbeddingBudgetState.test.ts new file mode 100644 index 0000000000..dc45c31a2c --- /dev/null +++ b/app/src/hooks/__tests__/useEmbeddingBudgetState.test.ts @@ -0,0 +1,344 @@ +/** + * useEmbeddingBudgetState tests (#5324). + * + * The two things that must never break: the thresholds the issue specifies, + * and the guard that keeps users who fund their own embeddings from ever + * seeing a managed-budget warning. A false alarm here trains users to ignore + * the real one. + */ +import { act, renderHook } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { requestUsageRefresh } from '../usageRefresh'; +import { + EMBEDDING_BUDGET_URGENT_PCT, + EMBEDDING_BUDGET_WARN_PCT, + embeddingBudgetLevel, + isManagedEmbeddingProvider, + useEmbeddingBudgetState, +} from '../useEmbeddingBudgetState'; + +const mockLoadEmbeddingsSettings = vi.hoisted(() => vi.fn()); +const mockUseUsageState = vi.hoisted(() => vi.fn()); +const mockUseCoreState = vi.hoisted(() => vi.fn()); +const mockGetTeamUsage = vi.hoisted(() => vi.fn()); + +vi.mock('../../services/api/embeddingsApi', () => ({ + loadEmbeddingsSettings: mockLoadEmbeddingsSettings, +})); + +vi.mock('../useUsageState', () => ({ useUsageState: mockUseUsageState })); + +vi.mock('../../providers/CoreStateProvider', () => ({ useCoreState: mockUseCoreState })); + +vi.mock('../../services/api/creditsApi', () => ({ + creditsApi: { getTeamUsage: mockGetTeamUsage }, +})); + +/** Authenticated session with a managed cycle budget half-consumed. */ +function defaultMocks() { + mockUseCoreState.mockReturnValue({ snapshot: { auth: { isAuthenticated: true } } }); + mockUseUsageState.mockReturnValue({ + usagePct: 0.5, + isBudgetExhausted: false, + isLoading: false, + teamUsage: { cycleBudgetUsd: 10, remainingUsd: 5 }, + }); +} + +describe('embeddingBudgetLevel', () => { + it('stays silent below the warning threshold', () => { + expect(embeddingBudgetLevel(0, false)).toBe('none'); + expect(embeddingBudgetLevel(EMBEDDING_BUDGET_WARN_PCT - 0.01, false)).toBe('none'); + }); + + it('warns at exactly 75%', () => { + expect(embeddingBudgetLevel(EMBEDDING_BUDGET_WARN_PCT, false)).toBe('warn'); + }); + + it('escalates at exactly 90%', () => { + expect(embeddingBudgetLevel(EMBEDDING_BUDGET_URGENT_PCT, false)).toBe('urgent'); + expect(embeddingBudgetLevel(EMBEDDING_BUDGET_URGENT_PCT - 0.001, false)).toBe('warn'); + }); + + it('reports exhausted regardless of the derived percentage', () => { + // The hard `remainingUsd <= 0` verdict is authoritative: a percentage that + // rounds below 100 must not downgrade an actually-spent budget. + expect(embeddingBudgetLevel(0.97, true)).toBe('exhausted'); + expect(embeddingBudgetLevel(0, true)).toBe('exhausted'); + }); +}); + +describe('isManagedEmbeddingProvider', () => { + it('recognises the managed provider slugs', () => { + expect(isManagedEmbeddingProvider('openhuman')).toBe(true); + expect(isManagedEmbeddingProvider('managed')).toBe(true); + expect(isManagedEmbeddingProvider('cloud')).toBe(true); + }); + + it('matches on the slug when a model suffix is present', () => { + expect(isManagedEmbeddingProvider('openhuman:voyage-3')).toBe(true); + expect(isManagedEmbeddingProvider('ollama:nomic-embed-text')).toBe(false); + }); + + it('treats user-funded providers as unaffected by the managed budget', () => { + for (const p of ['ollama', 'openai', 'voyage', 'custom:http://localhost:1234']) { + expect(isManagedEmbeddingProvider(p)).toBe(false); + } + }); + + it('is conservative about an unknown provider', () => { + // A failed provider read must never manufacture a budget warning. + expect(isManagedEmbeddingProvider(null)).toBe(false); + expect(isManagedEmbeddingProvider(undefined)).toBe(false); + expect(isManagedEmbeddingProvider('')).toBe(false); + }); + + it('ignores case and surrounding whitespace', () => { + expect(isManagedEmbeddingProvider(' OpenHuman ')).toBe(true); + }); +}); + +// ── #5324: the provider must be re-read, or the warning outlives its fix ──── + +describe('useEmbeddingBudgetState provider refresh', () => { + beforeEach(() => { + mockLoadEmbeddingsSettings.mockReset(); + mockGetTeamUsage.mockReset(); + defaultMocks(); + }); + + it('re-reads the provider on an interval while embeddings are managed', async () => { + vi.useFakeTimers(); + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + const { result, unmount } = renderHook(() => useEmbeddingBudgetState()); + + // Flush the initial read inside `act` so `setProvider` commits before the + // assertions — otherwise the managed-gated interval below never arms. + await act(async () => { + await vi.advanceTimersByTimeAsync(0); + }); + expect(mockLoadEmbeddingsSettings).toHaveBeenCalledTimes(1); + expect(result.current.isManagedEmbeddings).toBe(true); + + // A user who follows the CTA and switches to local Ollama must stop being + // told their memory is broken, without restarting the app. + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'ollama:nomic-embed-text' }); + await act(async () => { + await vi.advanceTimersByTimeAsync(60_000); + }); + expect(mockLoadEmbeddingsSettings).toHaveBeenCalledTimes(2); + expect(result.current.isManagedEmbeddings).toBe(false); + + // Once off the managed budget the polling stops — no cost for the majority + // of users who fund their own embeddings. + await act(async () => { + await vi.advanceTimersByTimeAsync(180_000); + }); + expect(mockLoadEmbeddingsSettings).toHaveBeenCalledTimes(2); + + unmount(); + vi.useRealTimers(); + }); + + it('re-reads the provider when usage refreshes', async () => { + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'ollama' }); + renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalledTimes(1)); + + // The other direction: a user switching ONTO managed embeddings starts + // being warned without waiting for a remount. + requestUsageRefresh(); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalledTimes(2)); + }); +}); + +// ── #5324: session boundaries + the routed-away managed-embeddings gap ────── + +describe('useEmbeddingBudgetState session + managed-embeddings gaps', () => { + beforeEach(() => { + mockLoadEmbeddingsSettings.mockReset(); + mockGetTeamUsage.mockReset(); + defaultMocks(); + }); + + // CodeRabbit: a managed provider carried over from a previous user must not + // combine with a new session's usage into a false warning. + it('clears a stale managed provider when the session ends', async () => { + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + const { result, rerender } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(result.current.isManagedEmbeddings).toBe(true)); + + // Sign out: no live session, no usage payload. + mockUseCoreState.mockReturnValue({ snapshot: { auth: { isAuthenticated: false } } }); + mockUseUsageState.mockReturnValue({ + usagePct: 0, + isBudgetExhausted: false, + isLoading: false, + teamUsage: null, + }); + rerender(); + + await vi.waitFor(() => { + expect(result.current.isManagedEmbeddings).toBe(false); + expect(result.current.level).toBe('none'); + }); + }); + + // Codex: chat + background workloads routed off OpenHuman (so useUsageState + // reports no payload) while embeddings stay on the managed budget. The + // warning must still reach this user via a direct budget read. + it('warns a routed-away user whose embeddings still bill against the managed budget', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0, + isBudgetExhausted: false, + isLoading: false, + teamUsage: null, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + mockGetTeamUsage.mockResolvedValue({ cycleBudgetUsd: 10, remainingUsd: 1 }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + + await vi.waitFor(() => { + expect(mockGetTeamUsage).toHaveBeenCalledTimes(1); + expect(result.current.level).toBe('urgent'); + expect(result.current.pct).toBe(90); + }); + }); + + // A failed direct read must never manufacture a warning. + it('stays silent when the direct budget read fails', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0, + isBudgetExhausted: false, + isLoading: false, + teamUsage: null, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + mockGetTeamUsage.mockRejectedValue(new Error('usage unavailable')); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + + await vi.waitFor(() => expect(mockGetTeamUsage).toHaveBeenCalled()); + expect(result.current.level).toBe('none'); + expect(result.current.isManagedEmbeddings).toBe(true); + }); + + // The common managed user (useUsageState already has the figure) must not + // pay for a second billing round-trip. + it('does not issue a direct budget read when useUsageState already has usage', async () => { + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(result.current.isManagedEmbeddings).toBe(true)); + expect(mockGetTeamUsage).not.toHaveBeenCalled(); + expect(result.current.pct).toBe(50); + }); + + // A routed-away user on local/BYO embeddings still sees nothing — and we do + // not even issue the direct read for them. + it('never reads the managed budget for a routed-away BYO-embeddings user', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0, + isBudgetExhausted: false, + isLoading: false, + teamUsage: null, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'ollama:nomic-embed-text' }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalled()); + expect(mockGetTeamUsage).not.toHaveBeenCalled(); + expect(result.current.level).toBe('none'); + expect(result.current.isManagedEmbeddings).toBe(false); + }); + + // CodeRabbit: `teamUsage` is also null while useUsageState's own request is + // still in flight — the fallback must NOT fire then, or a normal managed user + // duplicates the getTeamUsage() call useUsageState is about to make. + it('does not read the managed budget while the primary usage request is loading', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0, + isBudgetExhausted: false, + isLoading: true, // primary usage request pending, not routed-away + teamUsage: null, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalled()); + expect(mockGetTeamUsage).not.toHaveBeenCalled(); + expect(result.current.level).toBe('none'); // still loading → silent + }); + + // Reviewer M3gA-Mind (#5402): `provider` is the picker's own setting and is + // NOT authoritative for how embeddings are funded. A user who enabled local + // embeddings through Local AI Settings runs fully local, bills nothing — and + // still reads `provider: "cloud"`, because nothing rewrites that field. The + // core now resolves the real ladder and sends `effective_provider`; gating on + // the stale field would put a non-dismissible "memory has stopped growing" + // banner on every screen the moment their CHAT budget crossed 90%. + it('gates on the effective embedder, not the stale provider setting', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0.95, + isBudgetExhausted: false, + isLoading: false, + teamUsage: { cycleBudgetUsd: 10, remainingUsd: 0.5 }, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ + provider: 'cloud', + effective_provider: 'ollama', + }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalled()); + expect(result.current.isManagedEmbeddings).toBe(false); + expect(result.current.level).toBe('none'); + }); + + // The other direction: `effective_provider` must be able to turn the warning + // ON as well, so it is a correction of the signal and not a mute switch. + it('warns when the effective embedder is the managed cloud one', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0.95, + isBudgetExhausted: false, + isLoading: false, + teamUsage: { cycleBudgetUsd: 10, remainingUsd: 0.5 }, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ + provider: 'ollama:nomic-embed-text', + effective_provider: 'cloud', + }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(result.current.isManagedEmbeddings).toBe(true)); + expect(result.current.level).toBe('urgent'); + }); + + // `unconfigured` (signed in, but the ladder found no usable provider) bills + // nothing, so it must not be treated as managed either. + it('treats an unconfigured effective embedder as unmanaged', async () => { + mockUseUsageState.mockReturnValue({ + usagePct: 0.95, + isBudgetExhausted: true, + isLoading: false, + teamUsage: { cycleBudgetUsd: 10, remainingUsd: 0 }, + }); + mockLoadEmbeddingsSettings.mockResolvedValue({ + provider: 'cloud', + effective_provider: 'unconfigured', + }); + + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(mockLoadEmbeddingsSettings).toHaveBeenCalled()); + expect(result.current.isManagedEmbeddings).toBe(false); + expect(result.current.level).toBe('none'); + }); + + // A core old enough not to send the field must keep working off `provider`. + it('falls back to the provider setting when the core sends no effective_provider', async () => { + mockLoadEmbeddingsSettings.mockResolvedValue({ provider: 'openhuman' }); + const { result } = renderHook(() => useEmbeddingBudgetState()); + await vi.waitFor(() => expect(result.current.isManagedEmbeddings).toBe(true)); + }); +}); diff --git a/app/src/hooks/useEmbeddingBudgetState.ts b/app/src/hooks/useEmbeddingBudgetState.ts new file mode 100644 index 0000000000..8b0937076a --- /dev/null +++ b/app/src/hooks/useEmbeddingBudgetState.ts @@ -0,0 +1,280 @@ +/** + * Memory-embedding budget state (#5324). + * + * The failure this exists to prevent: a heavy user's managed embedding budget + * runs out, every embed job fails as `unrecoverable`, and the Memory Tree + * silently stops growing. The only signal was a yellow banner inside a + * settings panel nobody opens, so users experienced it as "the app has been + * broken for a month" without knowing why. + * + * ## Which budget this reads, and why + * + * There is no separate embedding meter. Managed embeddings are billed against + * the *same* managed cycle budget as chat — the cloud embed route returns the + * identical `USER_INSUFFICIENT_CREDITS` / "Insufficient budget" error — so + * `useUsageState().usagePct` is the authoritative consumption figure for both. + * This hook adds the memory-specific *framing* on top: it only fires when the + * user's embeddings actually route through that managed budget, and it steers + * toward the embedding-specific fixes (local Ollama, BYO key) rather than the + * plan upgrade `GlobalUpsellBanner` already offers. + * + * A user on local Ollama or a BYO key is unaffected by the managed budget for + * embeddings, so they must never see this — a false alarm here would teach + * users to ignore the real one. + */ +import { useCallback, useEffect, useState } from 'react'; + +import { useCoreState } from '../providers/CoreStateProvider'; +import { creditsApi, type TeamUsage } from '../services/api/creditsApi'; +import { loadEmbeddingsSettings } from '../services/api/embeddingsApi'; +import { CoreRpcError } from '../services/coreRpcClient'; +import { subscribeUsageRefresh } from './usageRefresh'; +import { useUsageState } from './useUsageState'; + +/** Consumption at which the dismissible early warning appears. */ +export const EMBEDDING_BUDGET_WARN_PCT = 0.75; +/** Consumption at which the warning becomes non-dismissible. */ +export const EMBEDDING_BUDGET_URGENT_PCT = 0.9; + +/** + * Provider slugs that bill against the managed cycle budget. Everything else + * (`ollama:*`, `openai`, `voyage`, `custom:*`, `none`, `unconfigured`, + * `unknown`, …) is funded by the user — or by nobody — so the managed budget + * running out does not stop their memory from growing. + */ +const MANAGED_PROVIDER_SLUGS = ['openhuman', 'managed', 'cloud']; + +/** Grep prefix for this hook's lifecycle diagnostics. */ +const LOG = '[embedding-budget]'; + +/** + * Privacy-safe error label. Never log the raw error: it can carry endpoint + * URLs, request bodies, or backend messages quoting user content. A kind plus + * (for our own typed errors) the stable discriminator is enough to diagnose. + */ +function errorKind(err: unknown): string { + if (err instanceof CoreRpcError) return `core_rpc:${err.kind}`; + if (err instanceof Error) return `error:${err.name}`; + return 'unknown'; +} + +export type EmbeddingBudgetLevel = 'none' | 'warn' | 'urgent' | 'exhausted'; + +export interface EmbeddingBudgetState { + /** Which banner (if any) the user should see. `none` renders nothing. */ + level: EmbeddingBudgetLevel; + /** Whole-percent consumption, for the warning copy. */ + pct: number; + /** True while the provider or usage read is still in flight. */ + isLoading: boolean; + /** True when embeddings bill against the managed budget. */ + isManagedEmbeddings: boolean; +} + +/** True when `provider` bills against the managed cycle budget. */ +export function isManagedEmbeddingProvider(provider: string | null | undefined): boolean { + if (!provider) return false; + // Providers are stored either bare (`openhuman`) or as `slug:model` + // (`ollama:nomic-embed-text`), so compare on the slug only. + const slug = provider.trim().toLowerCase().split(':')[0]; + return MANAGED_PROVIDER_SLUGS.includes(slug); +} + +/** + * Pure threshold mapping, exported so the levels can be tested without + * mocking the RPC layer. + * + * `isExhausted` wins over the percentage because the two can disagree: a + * hard `remainingUsd <= 0` verdict is authoritative even if the derived + * percentage rounds to something under 100. + */ +export function embeddingBudgetLevel(usagePct: number, isExhausted: boolean): EmbeddingBudgetLevel { + if (isExhausted) return 'exhausted'; + if (usagePct >= EMBEDDING_BUDGET_URGENT_PCT) return 'urgent'; + if (usagePct >= EMBEDDING_BUDGET_WARN_PCT) return 'warn'; + return 'none'; +} + +/** Derived budget snapshot: percent consumed + hard-exhausted verdict. */ +interface DerivedBudget { + pct: number; + exhausted: boolean; +} + +/** + * Percent-consumed + exhausted verdict from a raw `TeamUsage`. Byte-identical + * to `useUsageState`'s own derivation so the fallback path (below) can never + * disagree with the primary path on the same underlying budget. + */ +function deriveBudget(usage: TeamUsage): DerivedBudget { + const pct = + usage.cycleBudgetUsd > 0.01 + ? Math.max(0, Math.min(1, (usage.cycleBudgetUsd - usage.remainingUsd) / usage.cycleBudgetUsd)) + : 0; + const exhausted = usage.cycleBudgetUsd > 0.01 && usage.remainingUsd <= 0.01; + return { pct, exhausted }; +} + +/** + * How often the embeddings provider is re-read while it still bills against + * the managed budget. Matches `useUsageState`'s cache TTL. + */ +const PROVIDER_RECHECK_MS = 60_000; + +export function useEmbeddingBudgetState(): EmbeddingBudgetState { + const { snapshot } = useCoreState(); + const isAuthenticated = snapshot.auth.isAuthenticated; + const { usagePct, isBudgetExhausted, isLoading: usageLoading, teamUsage } = useUsageState(); + const [provider, setProvider] = useState(null); + const [providerLoading, setProviderLoading] = useState(true); + // Fallback budget snapshot for the routed-away-but-managed-embeddings case. + // `useUsageState` deliberately returns no `teamUsage` when every chat + + // background workload is routed off OpenHuman (#2020 privacy optimisation) — + // but managed embeddings still bill against the managed cycle budget, so we + // read it directly there rather than letting that bypass silence the warning. + const [fallbackUsage, setFallbackUsage] = useState(null); + const [reloadCount, setReloadCount] = useState(0); + + const reload = useCallback(() => setReloadCount(n => n + 1), []); + + // Depend on the *presence* of a usage payload, not the object itself — the + // object identity is not part of this hook's contract, and keying an effect + // on it would re-fire the read on every render for any caller whose + // `useUsageState` returns a fresh object. + const hasUsage = teamUsage !== null; + + useEffect(() => { + // Gate the provider/budget reads on a live session, NOT on the presence of + // a usage payload. `teamUsage` is null both when signed out AND when an + // authenticated user has routed chat away while keeping managed embeddings + // — the two must be handled differently, so `hasUsage` cannot be the gate. + // + // Signed out / offline: clear any provider carried over from a previous + // user so the next session cannot combine this user's usage with the prior + // user's managed provider (which would show a false managed-budget warning + // before the fresh read resolves). Then skip the RPCs, which require a + // session anyway. + if (!isAuthenticated) { + console.debug(`${LOG} skip: not authenticated — cleared provider + fallback budget`); + setProvider(null); + setFallbackUsage(null); + setProviderLoading(false); + return; + } + let cancelled = false; + setProviderLoading(true); + console.debug(`${LOG} provider read start (hasUsage=${hasUsage} usageLoading=${usageLoading})`); + void (async () => { + try { + const settings = await loadEmbeddingsSettings(); + if (cancelled) return; + // Gate on the *effective* embedder, not the picker setting. The core + // resolves local Ollama from the Local AI "Memory embeddings" toggle or + // the `memory_tree.embedding_endpoint` override, and neither rewrites + // `provider` — so a fully-local user still reads `provider: "cloud"` + // there and would be told their memory stopped growing while it is + // growing fine (reviewer M3gA-Mind, #5402). Fall back to `provider` + // only for a core old enough not to send the field. + const nextProvider = settings.effective_provider ?? settings.provider; + const managed = isManagedEmbeddingProvider(nextProvider); + console.debug( + `${LOG} provider read ok: effective=${nextProvider} ` + + `configured=${settings.provider} managed=${managed}` + ); + setProvider(nextProvider); + // Only reach for the direct budget read when it is actually needed: + // embeddings bill against the managed budget AND `useUsageState` has + // SETTLED with no payload (chat routed away). `teamUsage` is also null + // while `useUsageState`'s own request is still in flight, so gate on + // `!usageLoading` too — otherwise a normal managed user whose provider + // read resolves first fires a redundant `getTeamUsage()` that + // `useUsageState` is about to make anyway. + if (managed && !hasUsage && !usageLoading) { + console.debug(`${LOG} fallback budget read start (managed + usage settled empty)`); + try { + const usage = await creditsApi.getTeamUsage(); + if (!cancelled) setFallbackUsage(deriveBudget(usage)); + console.debug(`${LOG} fallback budget read ok`); + } catch (err) { + // Auth-expired is handled globally by coreRpcClient; any failure + // here just means "no budget data", so stay silent rather than + // guess. Never manufacture a warning from a failed read. + if (err instanceof CoreRpcError && err.kind === 'auth_expired') { + console.debug(`${LOG} fallback budget read skipped: session expired`); + if (!cancelled) setFallbackUsage(null); + } else { + console.warn(`${LOG} fallback budget read failed kind=${errorKind(err)}`); + if (!cancelled) setFallbackUsage(null); + } + } + } else if (!cancelled) { + // Not needed (BYO/local, or `useUsageState` already has the figure). + console.debug( + `${LOG} fallback budget read not needed ` + + `(managed=${managed} hasUsage=${hasUsage} usageLoading=${usageLoading})` + ); + setFallbackUsage(null); + } + } catch (err) { + // Conservative on failure: an unknown provider is treated as + // NOT managed, so a transient RPC error can never manufacture a + // budget warning for a user who funds their own embeddings. + console.warn(`${LOG} provider read failed kind=${errorKind(err)} — treating as unmanaged`); + if (!cancelled) { + setProvider(null); + setFallbackUsage(null); + } + } finally { + if (!cancelled) setProviderLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [reloadCount, isAuthenticated, hasUsage, usageLoading]); + + const isManagedEmbeddings = isManagedEmbeddingProvider(provider); + + // Without this the warning outlives its own fix. The banner is mounted for + // the app's lifetime, so a single mount-time read means a user who follows + // the CTA and switches to local Ollama keeps being told their memory is + // broken until they restart the app — which would make the remediation look + // like it did not work. + // + // Only re-polls while embeddings still bill against the managed budget, so + // the majority of users (BYO key, local) cost nothing. The subscription + // below covers the other direction. + useEffect(() => { + if (!isManagedEmbeddings) { + console.debug(`${LOG} polling off (embeddings do not bill the managed budget)`); + return; + } + console.debug(`${LOG} polling on every ${PROVIDER_RECHECK_MS}ms`); + const id = window.setInterval(reload, PROVIDER_RECHECK_MS); + return () => { + console.debug(`${LOG} polling stopped`); + window.clearInterval(id); + }; + }, [isManagedEmbeddings, reload]); + + // Any usage refresh (sign-in, plan change, manual refresh) also re-reads the + // provider, so a user who *switches onto* managed embeddings starts being + // warned without waiting for a remount. + useEffect(() => subscribeUsageRefresh(reload), [reload]); + const isLoading = usageLoading || providerLoading; + + // Prefer `useUsageState`'s figure; fall back to the direct read for the + // routed-away managed-embeddings case. When neither is available the session + // never reached the billing API (signed out / offline) — claiming a budget + // state from that is guesswork, so stay silent. + const budget: DerivedBudget | null = teamUsage + ? { pct: usagePct, exhausted: isBudgetExhausted } + : fallbackUsage; + + const level = + isLoading || !isManagedEmbeddings || !budget + ? 'none' + : embeddingBudgetLevel(budget.pct, budget.exhausted); + + return { level, pct: Math.round((budget?.pct ?? 0) * 100), isLoading, isManagedEmbeddings }; +} diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index bcfd00ac3a..470709998b 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -6680,6 +6680,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'وضع تخزين الأسرار وحالة سلسلة المفاتيح', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'متدهور', + 'memoryTree.status.statusBudgetExhausted': 'متوقف مؤقتًا: انتهت ميزانية التضمينات', 'memoryTree.status.degradedRecall': 'الاسترجاع الدلالي معطّل', 'memoryTree.status.degradedStructure': 'بنية الويكي غير مكتملة', 'memoryTree.status.extractionCoverage': 'تغطية الاستخراج: {pct}% من الأجزاء لها بنية', @@ -6995,6 +6996,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'تجاهل', 'userErrors.action.openBilling': 'فتح الفوترة', 'userErrors.action.openProviderSettings': 'إعدادات المزود', + 'userErrors.action.openEmbeddingsSettings': 'إعداد التضمينات', 'userErrors.budgetExceeded.title': 'تم استنفاد الميزانية المُدارة', 'userErrors.budgetExceeded.body': 'نفدت الميزانية المُدارة. أضف ميزانية أو غيّر خطتك.', 'userErrors.insufficientCredits.title': 'مطلوب رصيد المزود', @@ -7007,6 +7009,17 @@ const messages: TranslationMap = { 'لا يمكن الوصول إلى Ollama على النقطة الطرفية المُهيأة، أو أن النموذج المطلوب غير مثبّت عليها. شغّل Ollama ونزّل النموذج على تلك النقطة الطرفية، أو حوّل هذا العمل إلى مزوّد سحابي.', 'userErrors.scope.chat': 'الدردشة', 'userErrors.scope.cron': 'مهمة مجدوَلة', + 'userErrors.scope.workspace': 'مساحة العمل', + 'userErrors.memoryBudgetExhausted.title': 'توقفت الذاكرة عن النمو', + 'userErrors.memoryBudgetExhausted.body': + 'انتهت ميزانية التضمينات لديك، لذلك لم يعد المحتوى الجديد يُضاف إلى الذاكرة. أعدّ تضمينات محلية أو أضف مفتاح API الخاص بك للمتابعة.', + 'memoryBudget.approachingTitle': 'الذاكرة تقترب من حد التضمينات', + 'memoryBudget.approachingMessage': + 'لقد استخدمت {pct}% من ميزانية التضمينات. أعدّ تضمينات محلية أو أضف مفتاح API الخاص بك كي تستمر الذاكرة في النمو دون انقطاع.', + 'memoryBudget.exhaustedTitle': 'توقفت الذاكرة عن النمو', + 'memoryBudget.exhaustedMessage': + 'انتهت ميزانية التضمينات لديك، لذلك لم يعد المحتوى الجديد يُضاف إلى الذاكرة. أعدّ تضمينات محلية أو أضف مفتاح API الخاص بك للمتابعة.', + 'memoryBudget.cta': 'إعداد التضمينات', 'userErrors.scope.memory': 'الذاكرة', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'المبلغ', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 109e86164d..be33c531e0 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -6832,6 +6832,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'গোপনীয়তা সঞ্চয়স্থান মোড এবং কিচেন অবস্থা', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'অবনমিত', + 'memoryTree.status.statusBudgetExhausted': 'বিরতি: এমবেডিং বাজেট শেষ', 'memoryTree.status.degradedRecall': 'সিম্যান্টিক রিকল নিষ্ক্রিয়', 'memoryTree.status.degradedStructure': 'উইকি কাঠামো অসম্পূর্ণ', 'memoryTree.status.extractionCoverage': 'এক্সট্র্যাকশন কভারেজ: {pct}% অংশের কাঠামো আছে', @@ -7153,6 +7154,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'বাতিল করুন', 'userErrors.action.openBilling': 'বিলিং খুলুন', 'userErrors.action.openProviderSettings': 'প্রদানকারী সেটিংস', + 'userErrors.action.openEmbeddingsSettings': 'এমবেডিং সেট আপ করুন', 'userErrors.budgetExceeded.title': 'পরিচালিত বাজেট শেষ', 'userErrors.budgetExceeded.body': 'পরিচালিত AI বাজেট শেষ। বাজেট যোগ করুন বা প্ল্যান বদলান।', 'userErrors.insufficientCredits.title': 'প্রদানকারীর ক্রেডিট প্রয়োজন', @@ -7166,6 +7168,17 @@ const messages: TranslationMap = { 'কনফিগার করা এন্ডপয়েন্টে Ollama-তে পৌঁছানো যাচ্ছে না, অথবা সেখানে প্রয়োজনীয় মডেলটি ইনস্টল করা নেই। Ollama চালু করে সেই এন্ডপয়েন্টে মডেলটি পুল করুন, অথবা এই কাজটি কোনো ক্লাউড প্রোভাইডারে সরিয়ে নিন।', 'userErrors.scope.chat': 'চ্যাট', 'userErrors.scope.cron': 'নির্ধারিত কাজ', + 'userErrors.scope.workspace': 'ওয়ার্কস্পেস', + 'userErrors.memoryBudgetExhausted.title': 'মেমরি আর বাড়ছে না', + 'userErrors.memoryBudgetExhausted.body': + 'আপনার এমবেডিং বাজেট শেষ, তাই নতুন কনটেন্ট আর মেমরিতে যুক্ত হচ্ছে না। আবার শুরু করতে লোকাল এমবেডিং সেট আপ করুন বা নিজের API কী যোগ করুন।', + 'memoryBudget.approachingTitle': 'মেমরি এমবেডিং সীমার কাছাকাছি', + 'memoryBudget.approachingMessage': + 'আপনি এমবেডিং বাজেটের {pct}% ব্যবহার করেছেন। মেমরি নিরবচ্ছিন্নভাবে বাড়তে থাকুক, তার জন্য লোকাল এমবেডিং সেট আপ করুন বা নিজের API কী যোগ করুন।', + 'memoryBudget.exhaustedTitle': 'মেমরি আর বাড়ছে না', + 'memoryBudget.exhaustedMessage': + 'আপনার এমবেডিং বাজেট শেষ, তাই নতুন কনটেন্ট আর মেমরিতে যুক্ত হচ্ছে না। আবার শুরু করতে লোকাল এমবেডিং সেট আপ করুন বা নিজের API কী যোগ করুন।', + 'memoryBudget.cta': 'এমবেডিং সেট আপ করুন', 'userErrors.scope.memory': 'মেমরি', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'পরিমাণ', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index a77d97e707..cfba548c60 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -7012,6 +7012,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Geheimnisspeicher-Modus und Schlüsselbund-Status', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Eingeschränkt', + 'memoryTree.status.statusBudgetExhausted': 'Pausiert: Embedding-Budget erreicht', 'memoryTree.status.degradedRecall': 'Semantische Suche deaktiviert', 'memoryTree.status.degradedStructure': 'Wiki-Struktur unvollständig', 'memoryTree.status.extractionCoverage': @@ -7353,6 +7354,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Verwerfen', 'userErrors.action.openBilling': 'Abrechnung öffnen', 'userErrors.action.openProviderSettings': 'Anbietereinstellungen', + 'userErrors.action.openEmbeddingsSettings': 'Embeddings einrichten', 'userErrors.budgetExceeded.title': 'Verwaltetes Budget erreicht', 'userErrors.budgetExceeded.body': 'Dein verwaltetes KI-Budget ist aufgebraucht. Füge Budget hinzu oder ändere deinen Tarif.', @@ -7367,6 +7369,17 @@ const messages: TranslationMap = { 'Ollama ist unter dem konfigurierten Endpunkt nicht erreichbar, oder das benötigte Modell ist dort nicht installiert. Starte Ollama und lade das Modell auf diesem Endpunkt, oder verlagere diese Arbeit auf einen Cloud-Anbieter.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Geplante Aufgabe', + 'userErrors.scope.workspace': 'Arbeitsbereich', + 'userErrors.memoryBudgetExhausted.title': 'Das Gedächtnis wächst nicht mehr', + 'userErrors.memoryBudgetExhausted.body': + 'Dein Embedding-Budget ist aufgebraucht, daher werden keine neuen Inhalte mehr ins Gedächtnis aufgenommen. Richte lokale Embeddings ein oder hinterlege deinen eigenen API-Schlüssel, um fortzufahren.', + 'memoryBudget.approachingTitle': 'Das Gedächtnis nähert sich seinem Embedding-Limit', + 'memoryBudget.approachingMessage': + 'Du hast {pct} % deines Embedding-Budgets verbraucht. Richte lokale Embeddings ein oder hinterlege deinen eigenen API-Schlüssel, damit das Gedächtnis ohne Unterbrechung weiterwächst.', + 'memoryBudget.exhaustedTitle': 'Das Gedächtnis wächst nicht mehr', + 'memoryBudget.exhaustedMessage': + 'Dein Embedding-Budget ist aufgebraucht, daher werden keine neuen Inhalte mehr ins Gedächtnis aufgenommen. Richte lokale Embeddings ein oder hinterlege deinen eigenen API-Schlüssel, um fortzufahren.', + 'memoryBudget.cta': 'Embeddings einrichten', 'userErrors.scope.memory': 'Speicher', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Betrag', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index a96bd4dece..ab67103e5f 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -1210,6 +1210,9 @@ const en: TranslationMap = { 'memoryTree.status.statusError': 'Error', 'memoryTree.status.statusIdle': 'Idle', 'memoryTree.status.statusDegraded': 'Degraded', + // #5324: a spent embedding budget is a distinct state from a generic error — + // memory is paused, not broken, and the fix is the user's to make. + 'memoryTree.status.statusBudgetExhausted': 'Paused: embedding budget reached', 'memoryTree.status.never': 'Never', // #002: degraded badges + typed remediation strings. The Rust core sends a // `remediation_key` (one of memory.health.remediation.*) which the status @@ -7549,6 +7552,7 @@ const en: TranslationMap = { 'userErrors.dismiss': 'Dismiss', 'userErrors.action.openBilling': 'Open billing', 'userErrors.action.openProviderSettings': 'Provider settings', + 'userErrors.action.openEmbeddingsSettings': 'Set up embeddings', 'userErrors.budgetExceeded.title': 'Managed budget reached', 'userErrors.budgetExceeded.body': 'Your managed AI budget is used up. Add budget or change your plan to continue.', @@ -7558,12 +7562,25 @@ const en: TranslationMap = { 'userErrors.apiKeyMissing.title': 'API key required', 'userErrors.apiKeyMissing.body': 'Your AI provider has no API key set. Add one in provider settings to continue.', + 'userErrors.memoryBudgetExhausted.title': 'Memory has stopped growing', + 'userErrors.memoryBudgetExhausted.body': + 'Your embedding budget is used up, so new content is no longer being added to memory. Set up local embeddings or add your own API key to resume.', 'userErrors.localModelUnavailable.title': 'Local model unavailable', 'userErrors.localModelUnavailable.body': 'Ollama is not reachable at the configured endpoint, or the required model is not installed there. Start Ollama and pull the model at that endpoint, or switch this workload to a cloud provider.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Scheduled job', + 'userErrors.scope.workspace': 'Workspace', 'userErrors.scope.memory': 'Memory', + + // Memory embedding budget banners (#5324) + 'memoryBudget.approachingTitle': 'Memory is approaching its embedding limit', + 'memoryBudget.approachingMessage': + "You've used {pct}% of your embedding budget. Set up local embeddings or add your own API key to keep building memory without interruption.", + 'memoryBudget.exhaustedTitle': 'Memory has stopped growing', + 'memoryBudget.exhaustedMessage': + 'Your embedding budget is used up, so new content is no longer being added to memory. Set up local embeddings or add your own API key to resume.', + 'memoryBudget.cta': 'Set up embeddings', 'memorySources.codingSessions.title': 'Coding-agent sessions', 'memorySources.codingSessions.description': 'Turn your Codex and Claude Code decisions and corrections into private persona memory.', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 0352227b23..b7e87ffdb4 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -6962,6 +6962,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Modo de almacenamiento de secretos y estado del llavero', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Degradado', + 'memoryTree.status.statusBudgetExhausted': 'En pausa: se alcanzó el límite de embeddings', 'memoryTree.status.degradedRecall': 'Recuperación semántica desactivada', 'memoryTree.status.degradedStructure': 'Estructura de la wiki incompleta', 'memoryTree.status.extractionCoverage': @@ -7300,6 +7301,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Descartar', 'userErrors.action.openBilling': 'Abrir facturación', 'userErrors.action.openProviderSettings': 'Configuración del proveedor', + 'userErrors.action.openEmbeddingsSettings': 'Configurar embeddings', 'userErrors.budgetExceeded.title': 'Presupuesto gestionado agotado', 'userErrors.budgetExceeded.body': 'Tu presupuesto de IA gestionado se ha agotado. Añade presupuesto o cambia de plan.', @@ -7314,6 +7316,17 @@ const messages: TranslationMap = { 'No se puede acceder a Ollama en el punto de conexión configurado, o el modelo necesario no está instalado allí. Inicia Ollama y descarga el modelo en ese punto de conexión, o cambia este trabajo a un proveedor en la nube.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarea programada', + 'userErrors.scope.workspace': 'Espacio de trabajo', + 'userErrors.memoryBudgetExhausted.title': 'La memoria dejó de crecer', + 'userErrors.memoryBudgetExhausted.body': + 'Tu presupuesto de embeddings se agotó, así que el contenido nuevo ya no se añade a la memoria. Configura embeddings locales o añade tu propia clave de API para reanudar.', + 'memoryBudget.approachingTitle': 'La memoria se acerca a su límite de embeddings', + 'memoryBudget.approachingMessage': + 'Has usado el {pct}% de tu presupuesto de embeddings. Configura embeddings locales o añade tu propia clave de API para que la memoria siga creciendo sin interrupciones.', + 'memoryBudget.exhaustedTitle': 'La memoria dejó de crecer', + 'memoryBudget.exhaustedMessage': + 'Tu presupuesto de embeddings se agotó, así que el contenido nuevo ya no se añade a la memoria. Configura embeddings locales o añade tu propia clave de API para reanudar.', + 'memoryBudget.cta': 'Configurar embeddings', 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importe', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 9cd3fc261b..555dcc3aa6 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -6995,6 +6995,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Mode de stockage des secrets et état du trousseau', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Dégradé', + 'memoryTree.status.statusBudgetExhausted': "En pause : budget d'embeddings atteint", 'memoryTree.status.degradedRecall': 'Rappel sémantique désactivé', 'memoryTree.status.degradedStructure': 'Structure du wiki incomplète', 'memoryTree.status.extractionCoverage': @@ -7332,6 +7333,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Ignorer', 'userErrors.action.openBilling': 'Ouvrir la facturation', 'userErrors.action.openProviderSettings': 'Paramètres du fournisseur', + 'userErrors.action.openEmbeddingsSettings': 'Configurer les embeddings', 'userErrors.budgetExceeded.title': 'Budget géré atteint', 'userErrors.budgetExceeded.body': 'Votre budget IA géré est épuisé. Ajoutez du budget ou changez de forfait.', @@ -7346,6 +7348,17 @@ const messages: TranslationMap = { "Ollama n'est pas joignable sur le point de terminaison configuré, ou le modèle requis n'y est pas installé. Lancez Ollama et téléchargez le modèle sur ce point de terminaison, ou basculez cette charge de travail vers un fournisseur cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tâche planifiée', + 'userErrors.scope.workspace': 'Espace de travail', + 'userErrors.memoryBudgetExhausted.title': 'La mémoire a cessé de grandir', + 'userErrors.memoryBudgetExhausted.body': + "Votre budget d'embeddings est épuisé, les nouveaux contenus ne sont donc plus ajoutés à la mémoire. Configurez des embeddings locaux ou ajoutez votre propre clé API pour reprendre.", + 'memoryBudget.approachingTitle': "La mémoire approche de la limite de son budget d'embeddings", + 'memoryBudget.approachingMessage': + "Vous avez utilisé {pct}% de votre budget d'embeddings. Configurez des embeddings locaux ou ajoutez votre propre clé API pour que la mémoire continue de croître sans interruption.", + 'memoryBudget.exhaustedTitle': 'La mémoire a cessé de grandir', + 'memoryBudget.exhaustedMessage': + "Votre budget d'embeddings est épuisé, les nouveaux contenus ne sont donc plus ajoutés à la mémoire. Configurez des embeddings locaux ou ajoutez votre propre clé API pour reprendre.", + 'memoryBudget.cta': 'Configurer les embeddings', 'userErrors.scope.memory': 'Mémoire', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Montant', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 54e2de9ef3..1872addd02 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -6829,6 +6829,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'रहस्य भंडारण मोड और कीचेन स्थिति', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'अवक्रमित', + 'memoryTree.status.statusBudgetExhausted': 'रुका हुआ: एम्बेडिंग बजट समाप्त', 'memoryTree.status.degradedRecall': 'सिमेंटिक रिकॉल अक्षम', 'memoryTree.status.degradedStructure': 'विकी संरचना अधूरी', 'memoryTree.status.extractionCoverage': 'एक्सट्रैक्शन कवरेज: {pct}% खंडों में संरचना है', @@ -7151,6 +7152,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'खारिज करें', 'userErrors.action.openBilling': 'बिलिंग खोलें', 'userErrors.action.openProviderSettings': 'प्रदाता सेटिंग्स', + 'userErrors.action.openEmbeddingsSettings': 'एम्बेडिंग सेट करें', 'userErrors.budgetExceeded.title': 'प्रबंधित बजट समाप्त', 'userErrors.budgetExceeded.body': 'प्रबंधित AI बजट समाप्त। बजट जोड़ें या प्लान बदलें।', 'userErrors.insufficientCredits.title': 'प्रदाता क्रेडिट आवश्यक', @@ -7164,6 +7166,17 @@ const messages: TranslationMap = { 'कॉन्फ़िगर किए गए एंडपॉइंट पर Ollama तक पहुँच नहीं है, या ज़रूरी मॉडल वहाँ इंस्टॉल नहीं है। Ollama शुरू करके उसी एंडपॉइंट पर मॉडल पुल करें, या इस काम को किसी क्लाउड प्रोवाइडर पर ले जाएँ।', 'userErrors.scope.chat': 'चैट', 'userErrors.scope.cron': 'निर्धारित कार्य', + 'userErrors.scope.workspace': 'वर्कस्पेस', + 'userErrors.memoryBudgetExhausted.title': 'मेमोरी बढ़ना बंद हो गई है', + 'userErrors.memoryBudgetExhausted.body': + 'आपका एम्बेडिंग बजट खत्म हो गया है, इसलिए नई सामग्री अब मेमोरी में नहीं जुड़ रही। दोबारा शुरू करने के लिए लोकल एम्बेडिंग सेट करें या अपनी API कुंजी जोड़ें।', + 'memoryBudget.approachingTitle': 'मेमोरी अपनी एम्बेडिंग सीमा के पास पहुंच रही है', + 'memoryBudget.approachingMessage': + 'आपने अपने एम्बेडिंग बजट का {pct}% इस्तेमाल कर लिया है। मेमोरी बिना रुकावट बढ़ती रहे, इसके लिए लोकल एम्बेडिंग सेट करें या अपनी API कुंजी जोड़ें।', + 'memoryBudget.exhaustedTitle': 'मेमोरी बढ़ना बंद हो गई है', + 'memoryBudget.exhaustedMessage': + 'आपका एम्बेडिंग बजट खत्म हो गया है, इसलिए नई सामग्री अब मेमोरी में नहीं जुड़ रही। दोबारा शुरू करने के लिए लोकल एम्बेडिंग सेट करें या अपनी API कुंजी जोड़ें।', + 'memoryBudget.cta': 'एम्बेडिंग सेट करें', 'userErrors.scope.memory': 'मेमोरी', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'राशि', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index f661e55f1f..2ac006c98e 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -6861,6 +6861,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Mode penyimpanan rahasia dan status keychain', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Terdegradasi', + 'memoryTree.status.statusBudgetExhausted': 'Dijeda: batas embedding tercapai', 'memoryTree.status.degradedRecall': 'Recall semantik dinonaktifkan', 'memoryTree.status.degradedStructure': 'Struktur wiki tidak lengkap', 'memoryTree.status.extractionCoverage': 'Cakupan ekstraksi: {pct}% bagian memiliki struktur', @@ -7188,6 +7189,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Tutup', 'userErrors.action.openBilling': 'Buka penagihan', 'userErrors.action.openProviderSettings': 'Pengaturan penyedia', + 'userErrors.action.openEmbeddingsSettings': 'Siapkan embedding', 'userErrors.budgetExceeded.title': 'Anggaran terkelola habis', 'userErrors.budgetExceeded.body': 'Anggaran AI terkelola Anda sudah habis. Tambahkan anggaran atau ubah paket.', @@ -7202,6 +7204,17 @@ const messages: TranslationMap = { 'Ollama tidak dapat dijangkau di endpoint yang dikonfigurasi, atau model yang dibutuhkan belum terpasang di sana. Jalankan Ollama dan unduh modelnya di endpoint tersebut, atau alihkan pekerjaan ini ke penyedia cloud.', 'userErrors.scope.chat': 'Obrolan', 'userErrors.scope.cron': 'Tugas terjadwal', + 'userErrors.scope.workspace': 'Ruang kerja', + 'userErrors.memoryBudgetExhausted.title': 'Memori berhenti bertambah', + 'userErrors.memoryBudgetExhausted.body': + 'Anggaran embedding Anda sudah habis, sehingga konten baru tidak lagi ditambahkan ke memori. Siapkan embedding lokal atau tambahkan kunci API Anda sendiri untuk melanjutkan.', + 'memoryBudget.approachingTitle': 'Memori hampir mencapai batas embedding', + 'memoryBudget.approachingMessage': + 'Anda telah memakai {pct}% anggaran embedding. Siapkan embedding lokal atau tambahkan kunci API Anda sendiri agar memori terus bertambah tanpa gangguan.', + 'memoryBudget.exhaustedTitle': 'Memori berhenti bertambah', + 'memoryBudget.exhaustedMessage': + 'Anggaran embedding Anda sudah habis, sehingga konten baru tidak lagi ditambahkan ke memori. Siapkan embedding lokal atau tambahkan kunci API Anda sendiri untuk melanjutkan.', + 'memoryBudget.cta': 'Siapkan embedding', 'userErrors.scope.memory': 'Memori', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Jumlah', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 271d01b37b..42d1b303b9 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -6947,6 +6947,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Modalità archiviazione segreti e stato del portachiavi', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Degradato', + 'memoryTree.status.statusBudgetExhausted': 'In pausa: budget di embedding raggiunto', 'memoryTree.status.degradedRecall': 'Richiamo semantico disattivato', 'memoryTree.status.degradedStructure': 'Struttura del wiki incompleta', 'memoryTree.status.extractionCoverage': @@ -7285,6 +7286,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Ignora', 'userErrors.action.openBilling': 'Apri fatturazione', 'userErrors.action.openProviderSettings': 'Impostazioni del provider', + 'userErrors.action.openEmbeddingsSettings': 'Configura gli embedding', 'userErrors.budgetExceeded.title': 'Budget gestito esaurito', 'userErrors.budgetExceeded.body': 'Il tuo budget IA gestito è esaurito. Aggiungi budget o cambia piano.', @@ -7299,6 +7301,17 @@ const messages: TranslationMap = { "Ollama non è raggiungibile sull'endpoint configurato, oppure il modello necessario non è installato lì. Avvia Ollama e scarica il modello su quell'endpoint, oppure sposta questo lavoro su un provider cloud.", 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Attività pianificata', + 'userErrors.scope.workspace': 'Spazio di lavoro', + 'userErrors.memoryBudgetExhausted.title': 'La memoria ha smesso di crescere', + 'userErrors.memoryBudgetExhausted.body': + 'Il tuo budget di embedding è esaurito, quindi i nuovi contenuti non vengono più aggiunti alla memoria. Configura embedding locali o aggiungi la tua chiave API per riprendere.', + 'memoryBudget.approachingTitle': 'La memoria si sta avvicinando al limite di embedding', + 'memoryBudget.approachingMessage': + 'Hai usato il {pct}% del tuo budget di embedding. Configura embedding locali o aggiungi la tua chiave API per far crescere la memoria senza interruzioni.', + 'memoryBudget.exhaustedTitle': 'La memoria ha smesso di crescere', + 'memoryBudget.exhaustedMessage': + 'Il tuo budget di embedding è esaurito, quindi i nuovi contenuti non vengono più aggiunti alla memoria. Configura embedding locali o aggiungi la tua chiave API per riprendere.', + 'memoryBudget.cta': 'Configura gli embedding', 'userErrors.scope.memory': 'Memoria', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Importo', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 931f9693c3..5ebf3d0962 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -6753,6 +6753,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': '비밀 저장 모드 및 키체인 상태', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': '저하됨', + 'memoryTree.status.statusBudgetExhausted': '일시 중지됨: 임베딩 예산 소진', 'memoryTree.status.degradedRecall': '의미 기반 검색 비활성화됨', 'memoryTree.status.degradedStructure': '위키 구조 불완전', 'memoryTree.status.extractionCoverage': '추출 범위: 청크의 {pct}%에 구조가 있음', @@ -7073,6 +7074,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': '닫기', 'userErrors.action.openBilling': '결제 열기', 'userErrors.action.openProviderSettings': '제공업체 설정', + 'userErrors.action.openEmbeddingsSettings': '임베딩 설정', 'userErrors.budgetExceeded.title': '관리형 예산 소진', 'userErrors.budgetExceeded.body': '관리형 AI 예산이 모두 소진되었습니다.', 'userErrors.insufficientCredits.title': '제공업체 크레딧 필요', @@ -7085,6 +7087,17 @@ const messages: TranslationMap = { '구성된 엔드포인트에서 Ollama에 연결할 수 없거나 필요한 모델이 그곳에 설치되어 있지 않습니다. Ollama를 실행하고 해당 엔드포인트에 모델을 내려받거나, 이 작업을 클라우드 제공업체로 전환하세요.', 'userErrors.scope.chat': '채팅', 'userErrors.scope.cron': '예약된 작업', + 'userErrors.scope.workspace': '작업 공간', + 'userErrors.memoryBudgetExhausted.title': '메모리가 더 이상 늘어나지 않습니다', + 'userErrors.memoryBudgetExhausted.body': + '임베딩 예산을 모두 사용해 새 콘텐츠가 메모리에 추가되지 않습니다. 로컬 임베딩을 설정하거나 본인의 API 키를 추가하면 다시 시작됩니다.', + 'memoryBudget.approachingTitle': '메모리가 임베딩 한도에 근접했습니다', + 'memoryBudget.approachingMessage': + '임베딩 예산의 {pct}%를 사용했습니다. 로컬 임베딩을 설정하거나 본인의 API 키를 추가하면 메모리가 끊김 없이 계속 쌓입니다.', + 'memoryBudget.exhaustedTitle': '메모리가 더 이상 늘어나지 않습니다', + 'memoryBudget.exhaustedMessage': + '임베딩 예산을 모두 사용해 새 콘텐츠가 메모리에 추가되지 않습니다. 로컬 임베딩을 설정하거나 본인의 API 키를 추가하면 다시 시작됩니다.', + 'memoryBudget.cta': '임베딩 설정', 'userErrors.scope.memory': '메모리', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '금액', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 9c2bfd9b93..70fc5cb121 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -6930,6 +6930,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Tryb przechowywania sekretów i stan pęku kluczy', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Ograniczony', + 'memoryTree.status.statusBudgetExhausted': 'Wstrzymano: wyczerpano budżet osadzeń', 'memoryTree.status.degradedRecall': 'Wyszukiwanie semantyczne wyłączone', 'memoryTree.status.degradedStructure': 'Struktura wiki niekompletna', 'memoryTree.status.extractionCoverage': 'Pokrycie ekstrakcji: {pct}% fragmentów ma strukturę', @@ -7257,6 +7258,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Odrzuć', 'userErrors.action.openBilling': 'Otwórz rozliczenia', 'userErrors.action.openProviderSettings': 'Ustawienia dostawcy', + 'userErrors.action.openEmbeddingsSettings': 'Skonfiguruj osadzenia', 'userErrors.budgetExceeded.title': 'Wyczerpano zarządzany budżet', 'userErrors.budgetExceeded.body': 'Twój zarządzany budżet AI został wyczerpany. Dodaj budżet lub zmień plan.', @@ -7271,6 +7273,17 @@ const messages: TranslationMap = { 'Ollama jest nieosiągalna pod skonfigurowanym punktem końcowym albo wymagany model nie jest tam zainstalowany. Uruchom Ollamę i pobierz model w tym punkcie końcowym lub przenieś tę pracę do dostawcy w chmurze.', 'userErrors.scope.chat': 'Czat', 'userErrors.scope.cron': 'Zaplanowane zadanie', + 'userErrors.scope.workspace': 'Obszar roboczy', + 'userErrors.memoryBudgetExhausted.title': 'Pamięć przestała rosnąć', + 'userErrors.memoryBudgetExhausted.body': + 'Twój budżet osadzeń został wyczerpany, więc nowe treści nie są już dodawane do pamięci. Skonfiguruj lokalne osadzenia lub dodaj własny klucz API, aby wznowić.', + 'memoryBudget.approachingTitle': 'Pamięć zbliża się do limitu osadzeń', + 'memoryBudget.approachingMessage': + 'Wykorzystano {pct}% budżetu osadzeń. Skonfiguruj lokalne osadzenia lub dodaj własny klucz API, aby pamięć rosła bez przerw.', + 'memoryBudget.exhaustedTitle': 'Pamięć przestała rosnąć', + 'memoryBudget.exhaustedMessage': + 'Twój budżet osadzeń został wyczerpany, więc nowe treści nie są już dodawane do pamięci. Skonfiguruj lokalne osadzenia lub dodaj własny klucz API, aby wznowić.', + 'memoryBudget.cta': 'Skonfiguruj osadzenia', 'userErrors.scope.memory': 'Pamięć', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Kwota', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 951b94d57e..03ff08f9d0 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -6933,6 +6933,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Modo de armazenamento de segredos e status do chaveiro', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Degradado', + 'memoryTree.status.statusBudgetExhausted': 'Em pausa: limite de embeddings atingido', 'memoryTree.status.degradedRecall': 'Recuperação semântica desativada', 'memoryTree.status.degradedStructure': 'Estrutura do wiki incompleta', 'memoryTree.status.extractionCoverage': @@ -7268,6 +7269,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Dispensar', 'userErrors.action.openBilling': 'Abrir faturamento', 'userErrors.action.openProviderSettings': 'Configurações do provedor', + 'userErrors.action.openEmbeddingsSettings': 'Configurar embeddings', 'userErrors.budgetExceeded.title': 'Orçamento gerenciado esgotado', 'userErrors.budgetExceeded.body': 'Seu orçamento de IA gerenciado acabou. Adicione orçamento ou altere seu plano.', @@ -7282,6 +7284,17 @@ const messages: TranslationMap = { 'O Ollama não está acessível no endpoint configurado, ou o modelo necessário não está instalado nele. Inicie o Ollama e baixe o modelo nesse endpoint, ou mude este trabalho para um provedor na nuvem.', 'userErrors.scope.chat': 'Chat', 'userErrors.scope.cron': 'Tarefa agendada', + 'userErrors.scope.workspace': 'Espaço de trabalho', + 'userErrors.memoryBudgetExhausted.title': 'A memória parou de crescer', + 'userErrors.memoryBudgetExhausted.body': + 'Seu orçamento de embeddings acabou, então novos conteúdos não estão mais sendo adicionados à memória. Configure embeddings locais ou adicione sua própria chave de API para retomar.', + 'memoryBudget.approachingTitle': 'A memória está chegando ao limite de embeddings', + 'memoryBudget.approachingMessage': + 'Você já usou {pct}% do seu orçamento de embeddings. Configure embeddings locais ou adicione sua própria chave de API para a memória continuar crescendo sem interrupção.', + 'memoryBudget.exhaustedTitle': 'A memória parou de crescer', + 'memoryBudget.exhaustedMessage': + 'Seu orçamento de embeddings acabou, então novos conteúdos não estão mais sendo adicionados à memória. Configure embeddings locais ou adicione sua própria chave de API para retomar.', + 'memoryBudget.cta': 'Configurar embeddings', 'userErrors.scope.memory': 'Memória', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Valor', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index 2c92fc1cd0..d0d8374e84 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -6900,6 +6900,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': 'Режим хранения секретов и статус связки ключей', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': 'Ухудшено', + 'memoryTree.status.statusBudgetExhausted': 'Приостановлено: бюджет эмбеддингов исчерпан', 'memoryTree.status.degradedRecall': 'Семантический поиск отключён', 'memoryTree.status.degradedStructure': 'Структура вики неполная', 'memoryTree.status.extractionCoverage': 'Охват извлечения: {pct}% фрагментов имеют структуру', @@ -7232,6 +7233,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': 'Отклонить', 'userErrors.action.openBilling': 'Открыть оплату', 'userErrors.action.openProviderSettings': 'Настройки провайдера', + 'userErrors.action.openEmbeddingsSettings': 'Настроить эмбеддинги', 'userErrors.budgetExceeded.title': 'Управляемый бюджет исчерпан', 'userErrors.budgetExceeded.body': 'Управляемый бюджет ИИ исчерпан. Измените план.', 'userErrors.insufficientCredits.title': 'Требуются кредиты провайдера', @@ -7244,6 +7246,17 @@ const messages: TranslationMap = { 'Ollama недоступен по настроенному адресу, либо нужная модель там не установлена. Запустите Ollama и загрузите модель по этому адресу или переведите эту работу на облачного провайдера.', 'userErrors.scope.chat': 'Чат', 'userErrors.scope.cron': 'Запланированная задача', + 'userErrors.scope.workspace': 'Рабочая область', + 'userErrors.memoryBudgetExhausted.title': 'Память перестала расти', + 'userErrors.memoryBudgetExhausted.body': + 'Бюджет эмбеддингов израсходован, поэтому новые данные больше не добавляются в память. Настройте локальные эмбеддинги или добавьте свой ключ API, чтобы продолжить.', + 'memoryBudget.approachingTitle': 'Память приближается к лимиту эмбеддингов', + 'memoryBudget.approachingMessage': + 'Вы израсходовали {pct}% бюджета эмбеддингов. Настройте локальные эмбеддинги или добавьте свой ключ API, чтобы память продолжала расти без перерывов.', + 'memoryBudget.exhaustedTitle': 'Память перестала расти', + 'memoryBudget.exhaustedMessage': + 'Бюджет эмбеддингов израсходован, поэтому новые данные больше не добавляются в память. Настройте локальные эмбеддинги или добавьте свой ключ API, чтобы продолжить.', + 'memoryBudget.cta': 'Настроить эмбеддинги', 'userErrors.scope.memory': 'Память', // Agent World: Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': 'Сумма', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 71be66104a..1de19f9b86 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -6465,6 +6465,7 @@ const messages: TranslationMap = { 'pages.settings.account.securityDesc': '密钥存储模式和密钥链状态', // #002 memory-pipeline-hardening: degraded badges + typed remediation. 'memoryTree.status.statusDegraded': '已降级', + 'memoryTree.status.statusBudgetExhausted': '已暂停：嵌入额度已用尽', 'memoryTree.status.degradedRecall': '语义召回已禁用', 'memoryTree.status.degradedStructure': 'Wiki 结构不完整', 'memoryTree.status.extractionCoverage': '提取覆盖率：{pct}% 的片段具有结构', @@ -6769,6 +6770,7 @@ const messages: TranslationMap = { 'userErrors.dismiss': '忽略', 'userErrors.action.openBilling': '打开账单', 'userErrors.action.openProviderSettings': '提供商设置', + 'userErrors.action.openEmbeddingsSettings': '设置嵌入', 'userErrors.budgetExceeded.title': '托管预算已用尽', 'userErrors.budgetExceeded.body': '托管 AI 预算已用尽，请增加预算或更改套餐。', 'userErrors.insufficientCredits.title': '需要提供商额度', @@ -6780,6 +6782,17 @@ const messages: TranslationMap = { '无法在配置的端点连接 Ollama，或所需模型未安装在该端点。请启动 Ollama 并在该端点拉取模型，或将此工作切换到云端提供商。', 'userErrors.scope.chat': '聊天', 'userErrors.scope.cron': '定时任务', + 'userErrors.scope.workspace': '工作区', + 'userErrors.memoryBudgetExhausted.title': '记忆已停止增长', + 'userErrors.memoryBudgetExhausted.body': + '你的嵌入额度已用尽，新内容不会再加入记忆。设置本地嵌入或添加你自己的 API 密钥即可恢复。', + 'memoryBudget.approachingTitle': '记忆即将达到嵌入额度上限', + 'memoryBudget.approachingMessage': + '你已使用 {pct}% 的嵌入额度。设置本地嵌入或添加你自己的 API 密钥，让记忆不中断地继续增长。', + 'memoryBudget.exhaustedTitle': '记忆已停止增长', + 'memoryBudget.exhaustedMessage': + '你的嵌入额度已用尽，新内容不会再加入记忆。设置本地嵌入或添加你自己的 API 密钥即可恢复。', + 'memoryBudget.cta': '设置嵌入', 'userErrors.scope.memory': '记忆', // Agent World：Identity trading (confirm-before-spend + balance gate) 'agentWorld.trading.amountLabel': '金额', diff --git a/app/src/lib/userErrors/__tests__/classify.test.ts b/app/src/lib/userErrors/__tests__/classify.test.ts index ca225456e3..6f0310d605 100644 --- a/app/src/lib/userErrors/__tests__/classify.test.ts +++ b/app/src/lib/userErrors/__tests__/classify.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { classifyUserActionableError, userErrorId } from '../classify'; +import { + classifyMemoryPipelineFailure, + classifyUserActionableError, + userErrorId, +} from '../classify'; const BUDGET_MSG = 'OpenHuman API error (400): Insufficient budget'; const CREDITS_MSG = 'OpenRouter: this request requires more credits'; @@ -119,3 +123,52 @@ describe('classifyUserActionableError', () => { expect(a?.id).toBe(userErrorId('insufficient_credits', 'chat', 'openrouter')); }); }); + +// ── #5324: memory pipeline budget exhaustion ──────────────────────────────── + +describe('classifyMemoryPipelineFailure', () => { + it('promotes a budget-exhausted memory pipeline to a user-actionable error', () => { + const d = classifyMemoryPipelineFailure('budget_exhausted'); + expect(d).not.toBeNull(); + expect(d!.kind).toBe('memory_budget_exhausted'); + expect(d!.scope).toBe('workspace'); + expect(d!.sourceDomain).toBe('memory_tree'); + }); + + it('routes the CTA to embeddings settings, not billing', () => { + // Adding credits does not fix a memory outage — pointing embeddings at a + // local or BYO provider does. Sending the user to billing would be a dead + // end. + expect(classifyMemoryPipelineFailure('budget_exhausted')!.action).toBe( + 'open_embeddings_settings' + ); + }); + + it('dedupes separately from the chat-scoped budget error', () => { + // One exhausted budget can break both chat and memory at once; they need + // different fixes, so they must not collapse into one panel entry. + const memory = classifyMemoryPipelineFailure('budget_exhausted')!; + const chat = classifyUserActionableError({ message: 'Insufficient budget' })!; + expect(memory.id).not.toBe(chat.id); + }); + + it('ignores every other failure code', () => { + for (const code of [ + 'auth_missing', + 'auth_invalid', + 'embeddings_unconfigured', + 'embedding_dim_mismatch', + 'local_model_unavailable', + 'extraction_timeout', + 'storage_unavailable', + 'transient', + ]) { + expect(classifyMemoryPipelineFailure(code)).toBeNull(); + } + }); + + it('is null-safe for an absent cause', () => { + expect(classifyMemoryPipelineFailure(null)).toBeNull(); + expect(classifyMemoryPipelineFailure(undefined)).toBeNull(); + }); +}); diff --git a/app/src/lib/userErrors/classify.ts b/app/src/lib/userErrors/classify.ts index 5629d11b80..bf3422db87 100644 --- a/app/src/lib/userErrors/classify.ts +++ b/app/src/lib/userErrors/classify.ts @@ -34,6 +34,39 @@ export interface RuntimeErrorSignal { provider?: string; } +/** + * #5324: the memory pipeline's typed `budget_exhausted` cause, promoted to a + * first-class user-actionable error. + * + * Unlike the classifiers below this takes the core's stable `FailureCode` + * directly rather than pattern-matching prose — the memory pipeline already + * emits a typed cause on `first_blocking_cause`, so there is nothing to guess. + * That is the end state the text matchers below are migrating toward. + * + * Scoped to `workspace` (not `chat`) so a memory outage and a chat outage + * dedupe as separate entries. They have different fixes, and a user can hit + * both at once off the same exhausted budget. + * + * @param failureCode The `first_blocking_cause.code` from + * `memory_tree_pipeline_status`. + * @returns A descriptor when the cause is user-actionable, else `null`. + */ +export function classifyMemoryPipelineFailure( + failureCode: string | null | undefined +): UserErrorDescriptor | null { + if (failureCode !== 'budget_exhausted') return null; + return { + id: userErrorId('memory_budget_exhausted', 'workspace'), + kind: 'memory_budget_exhausted', + severity: 'warning', + scope: 'workspace', + sourceDomain: 'memory_tree', + titleKey: 'userErrors.memoryBudgetExhausted.title', + bodyKey: 'userErrors.memoryBudgetExhausted.body', + action: 'open_embeddings_settings', + }; +} + /** Build the stable dedupe identity for an error. */ export function userErrorId( kind: UserErrorDescriptor['kind'], diff --git a/app/src/lib/userErrors/report.ts b/app/src/lib/userErrors/report.ts index 7cdc51008c..79aa6e04b6 100644 --- a/app/src/lib/userErrors/report.ts +++ b/app/src/lib/userErrors/report.ts @@ -11,7 +11,11 @@ import debug from 'debug'; import type { AppDispatch } from '../../store'; import { reportUserError } from '../../store/userErrorsSlice'; -import { classifyUserActionableError, type RuntimeErrorSignal } from './classify'; +import { + classifyMemoryPipelineFailure, + classifyUserActionableError, + type RuntimeErrorSignal, +} from './classify'; const log = debug('openhuman:user-errors'); @@ -41,3 +45,32 @@ export function ingestRuntimeErrorSignal( return false; } } + +/** + * #5324: promote the memory pipeline's typed blocking cause into the panel. + * + * Called from the Memory Tree status poll. The store dedupes on the + * descriptor id, so re-reporting the same cause on every poll bumps the + * recurrence count instead of stacking duplicate entries — which is what + * makes it safe to call unconditionally from a polling loop. + * + * Same defensive contract as {@link ingestRuntimeErrorSignal}: never throws, + * returns `false` for causes that are not user-actionable. + * + * @param failureCode `first_blocking_cause.code` from the status payload. + */ +export function reportMemoryPipelineFailure( + dispatch: AppDispatch, + failureCode: string | null | undefined +): boolean { + try { + const descriptor = classifyMemoryPipelineFailure(failureCode); + if (!descriptor) return false; + log('memory pipeline actionable kind=%s', descriptor.kind); + dispatch(reportUserError({ descriptor, at: Date.now() })); + return true; + } catch (err) { + log('memory pipeline ingest failed: %o', err); + return false; + } +} diff --git a/app/src/services/api/embeddingsApi.ts b/app/src/services/api/embeddingsApi.ts index f845c03aec..b01be31e5f 100644 --- a/app/src/services/api/embeddingsApi.ts +++ b/app/src/services/api/embeddingsApi.ts @@ -27,7 +27,20 @@ export interface EmbeddingProviderEntry { } export interface EmbeddingsSettings { + /** The picker's own setting (`config.memory.embedding_provider`). */ provider: string; + /** + * The embedder ingestion will **actually** use, resolved core-side by the + * memory-tree provider ladder (#5402). Ask this — not `provider` — when the + * question is "do these embeddings bill against the managed budget?": the + * Local AI "Memory embeddings" toggle and the `memory_tree.embedding_endpoint` + * override both route to local Ollama without rewriting `provider`. + * + * One of `ollama` | `custom` | `cloud` | `none` | `unconfigured` | `unknown`. + * Optional so an older core (or a stubbed test payload) degrades to `provider` + * rather than throwing. + */ + effective_provider?: string; model: string; dimensions: number; rate_limit_per_min: number; diff --git a/app/src/types/userError.ts b/app/src/types/userError.ts index 1ae50ecabc..86d5cb37a2 100644 --- a/app/src/types/userError.ts +++ b/app/src/types/userError.ts @@ -13,11 +13,22 @@ * type is shaped to accept that source without the panel changing. */ -/** Stable discriminator the UI branches on. Extend as new states are added. */ +/** + * Stable discriminator the UI branches on. Extend as new states are added. + * + * `memory_budget_exhausted` (#5324) is deliberately separate from + * `budget_exceeded` even though both originate in the same managed cycle + * budget: the consequence and the fix differ. Chat being gated is immediately + * visible and is fixed by adding credits; memory silently stopping is + * invisible and is fixed by pointing embeddings at local Ollama or a BYO key. + * Collapsing them would send memory users to the billing screen, which does + * not solve their problem. + */ export type UserErrorKind = | 'insufficient_credits' | 'budget_exceeded' | 'api_key_missing' + | 'memory_budget_exhausted' /** * The local model runtime a workload depends on is not usable — Ollama is * not running, or the configured model was never pulled (#5354). Mirrors the @@ -36,7 +47,11 @@ export type UserErrorScope = | 'memory'; /** Primary next-step the user can take. `dismiss` is always available too. */ -export type UserErrorAction = 'open_billing' | 'open_provider_settings' | 'dismiss'; +export type UserErrorAction = + | 'open_billing' + | 'open_provider_settings' + | 'open_embeddings_settings' + | 'dismiss'; export type UserErrorSeverity = 'warning' | 'error'; diff --git a/src/openhuman/config/ops/model.rs b/src/openhuman/config/ops/model.rs index 7b39fc8586..fbb751b8ea 100644 --- a/src/openhuman/config/ops/model.rs +++ b/src/openhuman/config/ops/model.rs @@ -99,6 +99,12 @@ pub async fn apply_model_settings( config: &mut Config, update: ModelSettingsPatch, ) -> Result, String> { + // #5324: snapshot the embedder selection BEFORE applying the patch so the + // failed-job un-park below only fires when the embedder actually changed. + // This path also saves chat/reasoning/vision/etc. providers; without this + // gate, saving an unrelated model setting would restart every terminally + // `unrecoverable` embedding job and re-run the same external failure. + let prev_embeddings_provider = config.embeddings_provider.clone(); if let Some(api_url) = update.api_url { config.api_url = if api_url.trim().is_empty() { None @@ -248,11 +254,29 @@ pub async fn apply_model_settings( // signature. Coverage-gated + non-fatal: if the active signature did // not actually change, this enqueues nothing. crate::openhuman::memory::queue::ensure_reembed_backfill(config); + // #5324: the embedder may have just moved off the exhausted managed + // budget onto local Ollama / a BYO provider. Give the jobs that parked as + // `unrecoverable` under the old provider a fresh attempt budget — but ONLY + // when the embedder selection actually changed, so a chat/vision/etc. model + // save leaves terminally-failed jobs parked instead of re-failing them. + let embedder_changed = config.embeddings_provider != prev_embeddings_provider; + // #5324: the save has already succeeded, so a failed un-park must NOT fail + // the RPC — but it must not be reported as `requeued_failed=0` either, which + // would read identically to "nothing was parked" and hide that the parked + // jobs are still stuck. Surface the error in the outcome line instead. + let requeued_note = if embedder_changed { + match crate::openhuman::memory::queue::requeue_failed_after_provider_change(config) { + Ok(n) => n.to_string(), + Err(e) => format!("error ({e})"), + } + } else { + "0".to_string() + }; let snapshot = snapshot_config_json(config)?; Ok(RpcOutcome::new( snapshot, vec![format!( - "model settings saved to {}", + "model settings saved to {} (requeued_failed={requeued_note})", config.config_path.display() )], )) @@ -271,6 +295,13 @@ pub async fn apply_memory_settings( config: &mut Config, update: MemorySettingsPatch, ) -> Result, String> { + // #5324: snapshot the embedding signature BEFORE applying the patch. This + // path also saves `backend` / `auto_save` / `memory_window`, none of which + // remediate a budget-exhausted embedder — so the failed-job un-park below + // must fire only when the provider/model/dimensions actually changed. + let prev_embedding_provider = config.memory.embedding_provider.clone(); + let prev_embedding_model = config.memory.embedding_model.clone(); + let prev_embedding_dimensions = config.memory.embedding_dimensions; if let Some(backend) = update.backend { config.memory.backend = backend; } @@ -318,11 +349,28 @@ pub async fn apply_memory_settings( // are logged, never fail the settings save). §7's migration is // one-shot so it does not cover a later switch — this does. crate::openhuman::memory::queue::ensure_reembed_backfill(config); + // #5324: same rationale as the model-settings path — a switch away from + // the exhausted managed budget must un-park the jobs that failed under it, + // but a `memory_window` / `auto_save` / `backend` save must not. Gate on a + // real embedder change (provider/model/dimensions). + let embedder_changed = config.memory.embedding_provider != prev_embedding_provider + || config.memory.embedding_model != prev_embedding_model + || config.memory.embedding_dimensions != prev_embedding_dimensions; + // #5324: same as the model-settings path — keep the save successful but + // report an un-park failure instead of a misleading `requeued_failed=0`. + let requeued_note = if embedder_changed { + match crate::openhuman::memory::queue::requeue_failed_after_provider_change(config) { + Ok(n) => n.to_string(), + Err(e) => format!("error ({e})"), + } + } else { + "0".to_string() + }; let snapshot = snapshot_config_json(config)?; Ok(RpcOutcome::new( snapshot, vec![format!( - "memory settings saved to {}", + "memory settings saved to {} (requeued_failed={requeued_note})", config.config_path.display() )], )) diff --git a/src/openhuman/config/ops_tests.rs b/src/openhuman/config/ops_tests.rs index 1c934f80d3..b8e845a9a8 100644 --- a/src/openhuman/config/ops_tests.rs +++ b/src/openhuman/config/ops_tests.rs @@ -468,6 +468,128 @@ async fn apply_model_settings_updates_fields_and_persists_snapshot() { ); } +/// #5324 (CodeRabbit): the failed-job un-park must be scoped to an embedder +/// change. Saving an unrelated model setting (temperature, chat model, …) +/// through this shared path must leave terminally-`failed` embedding jobs +/// parked, not restart them into the same external failure. Switching the +/// embeddings provider is what un-parks them. +#[tokio::test] +async fn apply_model_settings_requeues_failed_jobs_only_on_embedder_change() { + use crate::openhuman::memory::queue::store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, JobStatus, NewJob}; + use crate::openhuman::memory::tree::health::{FailureCode, PipelineFailure}; + + let tmp = tempdir().unwrap(); + let mut cfg = tmp_config(&tmp); + + // Park a job exactly the way an exhausted managed budget does. + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job).unwrap().expect("enqueue"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + store::mark_failed_typed( + &cfg, + &job, + "Insufficient budget", + Some(&PipelineFailure::new(FailureCode::BudgetExhausted)), + ) + .unwrap(); + assert_eq!(store::count_by_status(&cfg, JobStatus::Failed).unwrap(), 1); + + // Unrelated save (temperature only) — the job must stay parked. + let unrelated = ModelSettingsPatch { + default_temperature: Some(0.5), + ..Default::default() + }; + let outcome = apply_model_settings(&mut cfg, unrelated) + .await + .expect("apply"); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Failed).unwrap(), + 1, + "an unrelated model save must not un-park failed embedding jobs" + ); + assert!( + outcome.logs.iter().any(|m| m.contains("requeued_failed=0")), + "messages: {:?}", + outcome.logs + ); + + // Now change the embeddings provider — this is the remediation, so the + // parked job must be flipped back to `ready`. + let switch = ModelSettingsPatch { + embeddings_provider: Some("ollama:bge-m3".into()), + ..Default::default() + }; + let outcome = apply_model_settings(&mut cfg, switch).await.expect("apply"); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "switching the embeddings provider must un-park the failed job" + ); + assert_eq!(store::count_by_status(&cfg, JobStatus::Failed).unwrap(), 0); + assert!( + outcome.logs.iter().any(|m| m.contains("requeued_failed=1")), + "messages: {:?}", + outcome.logs + ); +} + +/// #5324 (CodeRabbit): mirror of the model-settings gate for the memory path. +/// A `memory_window` / `auto_save` / `backend` save shares this function but +/// does not remediate the embedder, so failed jobs must stay parked; changing +/// the embedding provider un-parks them. +#[tokio::test] +async fn apply_memory_settings_requeues_failed_jobs_only_on_embedder_change() { + use crate::openhuman::memory::queue::store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, JobStatus, NewJob}; + use crate::openhuman::memory::tree::health::{FailureCode, PipelineFailure}; + + let tmp = tempdir().unwrap(); + let mut cfg = tmp_config(&tmp); + + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job).unwrap().expect("enqueue"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + store::mark_failed_typed( + &cfg, + &job, + "Insufficient budget", + Some(&PipelineFailure::new(FailureCode::BudgetExhausted)), + ) + .unwrap(); + + // Unrelated save (memory window preset only) — job stays parked. + let unrelated = MemorySettingsPatch { + memory_window: Some("balanced".into()), + ..Default::default() + }; + let outcome = apply_memory_settings(&mut cfg, unrelated) + .await + .expect("apply"); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Failed).unwrap(), + 1, + "a memory-window save must not un-park failed embedding jobs" + ); + assert!(outcome.logs.iter().any(|m| m.contains("requeued_failed=0"))); + + // Change the embedding provider — un-parks. + let switch = MemorySettingsPatch { + embedding_provider: Some("ollama".into()), + ..Default::default() + }; + let outcome = apply_memory_settings(&mut cfg, switch) + .await + .expect("apply"); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "switching the embedding provider must un-park the failed job" + ); + assert_eq!(store::count_by_status(&cfg, JobStatus::Failed).unwrap(), 0); + assert!(outcome.logs.iter().any(|m| m.contains("requeued_failed=1"))); +} + #[tokio::test] async fn apply_search_settings_sets_and_clears_allowed_domains() { let tmp = tempdir().unwrap(); diff --git a/src/openhuman/inference/embeddings/rpc.rs b/src/openhuman/inference/embeddings/rpc.rs index e2a6197315..e3513ee10a 100644 --- a/src/openhuman/inference/embeddings/rpc.rs +++ b/src/openhuman/inference/embeddings/rpc.rs @@ -93,8 +93,19 @@ pub async fn get_settings(config: &Config) -> Result Result requeued_count.to_string(), + Some(e) => format!("error ({e})"), + }; + tracing::info!( provider = config.memory.embedding_provider.as_str(), model = config.memory.embedding_model.as_str(), dimensions = config.memory.embedding_dimensions, sig_changed, + requeued = requeued_count, + requeue_error = requeue_error.as_deref().unwrap_or(""), "{LOG_PREFIX} update_settings applied" ); @@ -381,12 +423,14 @@ pub async fn update_settings( "dimensions": config.memory.embedding_dimensions, "signature_changed": sig_changed, "new_signature": new_sig, + "requeued_failed_jobs": requeued_count, + "requeue_error": requeue_error, }); Ok(RpcOutcome::new( payload, vec![format!( - "embeddings settings updated (sig_changed={sig_changed})" + "embeddings settings updated (sig_changed={sig_changed} requeued_failed={requeued_note})" )], )) } @@ -409,11 +453,34 @@ pub async fn set_api_key( auth.store_provider_token(&cred_provider, "default", api_key, HashMap::new(), true) .map_err(|e| format!("failed to store embedding API key: {e}"))?; - tracing::info!(provider = provider_slug, "{LOG_PREFIX} set_api_key stored"); + // #5324: supplying a BYO key does NOT change the embedding signature, so + // `ensure_reembed_backfill` has nothing to enqueue — but it is precisely + // the action that unblocks jobs parked on `budget_exhausted` / + // `auth_missing`. Requeue them here or they stay dead until the user + // separately discovers the "Retry failed" button. A store failure is + // surfaced (not reported as `0`) so the key-stored response can't imply the + // parked queue was recovered when it wasn't. + let requeue_result = + crate::openhuman::memory::queue::requeue_failed_after_provider_change(config); + let requeued_count = *requeue_result.as_ref().unwrap_or(&0); + let requeue_error = requeue_result.as_ref().err().cloned(); + let requeued_note = match &requeue_error { + None => requeued_count.to_string(), + Some(e) => format!("error ({e})"), + }; + + tracing::info!( + provider = provider_slug, + requeued = requeued_count, + requeue_error = requeue_error.as_deref().unwrap_or(""), + "{LOG_PREFIX} set_api_key stored" + ); Ok(RpcOutcome::new( - serde_json::json!({ "stored": true, "provider": provider_slug }), - vec![format!("embedding API key stored for {provider_slug}")], + serde_json::json!({ "stored": true, "provider": provider_slug, "requeued_failed_jobs": requeued_count, "requeue_error": requeue_error }), + vec![format!( + "embedding API key stored for {provider_slug} (requeued_failed={requeued_note})" + )], )) } @@ -1021,6 +1088,38 @@ mod tests { assert_eq!(resolve_api_key(&config, "voyage"), ""); } + /// `get_settings` must report the embedder ingestion will **actually** use + /// alongside the picker's own setting (#5402). The two disagree whenever + /// the user enabled local embeddings through Local AI Settings: that path + /// never rewrites `memory.embedding_provider`, so `provider` still reads + /// `"cloud"` while nothing bills the managed budget. A consumer that gated + /// a "your memory has stopped growing" banner on `provider` would fire it + /// at a user whose memory is growing fine. + #[tokio::test] + async fn get_settings_reports_effective_provider_separately_from_the_setting() { + let tmp = TempDir::new().unwrap(); + let mut config = Config::default(); + config.config_path = tmp.path().join("config.toml"); + config.workspace_dir = tmp.path().to_path_buf(); + config.memory.embedding_provider = "cloud".to_string(); + // A managed session exists, so the ladder would resolve to cloud … + std::fs::write(tmp.path().join("auth-profiles.json"), "{}").unwrap(); + // … except the unified workload setting routes embeddings to Ollama. + config.embeddings_provider = Some("ollama:all-minilm:latest".into()); + + let out = get_settings(&config) + .await + .expect("get_settings must succeed"); + assert_eq!( + out.value["provider"], "cloud", + "the picker setting is unchanged" + ); + assert_eq!( + out.value["effective_provider"], "ollama", + "the effective embedder is local, so nothing bills the managed budget" + ); + } + /// `custom:` providers must look up under the `embeddings:custom` /// slug (the inline URL is not part of the credential key), mirroring the /// slug normalization in `embed`/`set_api_key`. diff --git a/src/openhuman/memory/queue/mod.rs b/src/openhuman/memory/queue/mod.rs index 13709cdf48..5fd1035c81 100644 --- a/src/openhuman/memory/queue/mod.rs +++ b/src/openhuman/memory/queue/mod.rs @@ -36,7 +36,10 @@ pub mod testing; pub mod types; pub(crate) mod worker; -pub use ops::{backfill_in_progress, ensure_reembed_backfill, set_backfill_in_progress}; +pub use ops::{ + backfill_in_progress, ensure_reembed_backfill, requeue_failed_after_provider_change, + set_backfill_in_progress, +}; pub use store::{ claim_next, count_by_status, count_total, enqueue, enqueue_tx, get_job, mark_deferred, mark_done, mark_failed, recover_stale_locks, DEFAULT_LOCK_DURATION_MS, diff --git a/src/openhuman/memory/queue/ops.rs b/src/openhuman/memory/queue/ops.rs index f31efcd123..a0ec93e51b 100644 --- a/src/openhuman/memory/queue/ops.rs +++ b/src/openhuman/memory/queue/ops.rs @@ -1,5 +1,5 @@ -//! Memory-queue operations: backfill-progress signalling and the re-embed -//! backfill switch-path trigger. +//! Memory-queue operations: backfill-progress signalling, the re-embed +//! backfill switch-path trigger, and the provider-change failed-job un-park. //! //! Split out of `mod.rs` so the module root stays export-focused. Public paths //! are preserved via re-exports in [`super`], so callers keep using @@ -39,3 +39,167 @@ pub fn ensure_reembed_backfill(config: &crate::openhuman::config::Config) { log::warn!("[memory::jobs] ensure_reembed_backfill failed: {error:#}"); } } + +/// #5324: un-park terminally-`failed` jobs after the user changes their +/// embedding provider or supplies a key. +/// +/// The motivating case is `budget_exhausted`: once the managed embedding +/// budget is spent, every embed job fails as `Unrecoverable` and is parked +/// forever. `Unrecoverable` is meant to read as "cannot succeed *without user +/// action*", not "will never be retried" — so the moment the user takes that +/// action (points embeddings at local Ollama, or pastes a BYO key) the parked +/// jobs must get a fresh attempt budget instead of waiting for the user to +/// find the "Retry failed" button in Memory Tree settings. +/// +/// Deliberately requeues **all** failed jobs, not just the budget-exhausted +/// ones: scoping by `failure_code` would need a new filtered query in the +/// vendored `tinycortex` crate, and the cost of the wider net is bounded — a +/// job that fails for an unrelated reason (dim mismatch, empty input) simply +/// fails once more and re-parks, and this only runs on an explicit user +/// config change, never on a timer or on login. +/// +/// Non-fatal to the settings save by design, but NOT silent: on a store +/// failure this returns `Err` rather than a `0` that reads identically to +/// "nothing to requeue". The caller keeps the save successful and surfaces the +/// recovery failure in its RPC outcome, so a queue that stayed parked is never +/// presented to the user as remediated. `Ok(n)` is the number of jobs flipped +/// back to `ready` (`Ok(0)` = nothing was parked). +pub fn requeue_failed_after_provider_change( + config: &crate::openhuman::config::Config, +) -> Result { + // Entry record (see AGENTS.md "Debug logging"): state-transition op, so log + // entry + every branch + outcome. Prefix matches this module's sibling + // `ensure_reembed_backfill` (`[memory::jobs]`) — a stable, grep-friendly + // domain prefix. + log::debug!("[memory::jobs] provider change: evaluating parked failed jobs for requeue"); + match super::store::requeue_failed(config) { + Ok(0) => { + log::debug!("[memory::jobs] provider change: no failed jobs to requeue"); + Ok(0) + } + Ok(requeued) => { + log::info!( + "[memory::jobs] provider change: requeued {requeued} failed job(s) for a fresh attempt" + ); + // Wake the worker pool so the un-parked jobs are picked up + // promptly rather than at the next scheduled flush window. + super::wake_workers(); + Ok(requeued) + } + Err(error) => { + log::warn!("[memory::jobs] provider change: requeue_failed failed: {error:#}"); + Err(format!("{error:#}")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::config::Config; + use crate::openhuman::memory::tree::health::{FailureCode, PipelineFailure}; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + /// Nothing parked ⇒ nothing to un-park. Must not error, and must not wake + /// the worker pool for no reason. + #[test] + fn requeue_after_provider_change_is_zero_on_an_empty_queue() { + let (_tmp, cfg) = test_config(); + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 0); + } + + /// The #5324 case: jobs parked as `budget_exhausted` (unrecoverable, so + /// the periodic transient-requeue deliberately leaves them alone) must be + /// flipped back to `ready` when the user changes their embedding provider. + #[tokio::test] + async fn requeue_after_provider_change_unparks_budget_exhausted_jobs() { + use crate::openhuman::memory::queue::store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, JobStatus, NewJob}; + + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue job"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + + // Park it exactly the way an exhausted managed budget does. + let failure = PipelineFailure::new(FailureCode::BudgetExhausted); + assert!( + failure.is_unrecoverable(), + "precondition: parked, not retried" + ); + store::mark_failed_typed(&cfg, &job, "Insufficient budget", Some(&failure)).unwrap(); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Failed).unwrap(), + 1, + "precondition: the job is parked" + ); + assert_eq!( + store::count_failed_unrecoverable(&cfg).unwrap(), + 1, + "precondition: parked as unrecoverable, so periodic retry skips it" + ); + + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 1); + + assert_eq!( + store::count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "the job must be retryable again after the user fixes their provider" + ); + assert_eq!(store::count_by_status(&cfg, JobStatus::Failed).unwrap(), 0); + } + + /// Idempotent: calling it again once the queue is drained of failures is a + /// no-op, so re-saving settings repeatedly cannot spam the worker pool. + #[tokio::test] + async fn requeue_after_provider_change_is_idempotent() { + use crate::openhuman::memory::queue::store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, NewJob}; + + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue job"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + store::mark_failed_typed( + &cfg, + &job, + "Insufficient budget", + Some(&PipelineFailure::new(FailureCode::BudgetExhausted)), + ) + .unwrap(); + + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 1); + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 0); + } + + /// CodeRabbit (#5324): a store failure must SURFACE as `Err`, not collapse + /// into a `0` that reads identically to "nothing to requeue" and makes a + /// still-parked queue look remediated. + #[test] + fn requeue_after_provider_change_surfaces_store_errors() { + let tmp = TempDir::new().unwrap(); + // Point workspace_dir at a regular file, so the queue DB underneath it + // cannot be opened (ENOTDIR). The failure must propagate to the caller. + let as_file = tmp.path().join("workspace-is-a-file"); + std::fs::write(&as_file, b"not a directory").unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = as_file; + + let out = requeue_failed_after_provider_change(&cfg); + assert!( + out.is_err(), + "a store failure must surface as Err, not a misleading Ok(0): {out:?}" + ); + } +} diff --git a/src/openhuman/memory/tree/score/embed/factory.rs b/src/openhuman/memory/tree/score/embed/factory.rs index d15b8ae274..8f1faed015 100644 --- a/src/openhuman/memory/tree/score/embed/factory.rs +++ b/src/openhuman/memory/tree/score/embed/factory.rs @@ -250,6 +250,94 @@ pub fn build_write_embedder(config: &Config) -> Result> }) } +/// Render a ladder-resolution error safely for a log line. +/// +/// The only user-controlled values these errors interpolate are the configured +/// provider string and model (see `openai_compat::try_from_config`), and one of +/// them is an endpoint: in its `custom:` form `memory.embedding_provider` +/// *is* a URL, which may carry `user:pass@` userinfo. Configured +/// `cloud_providers` endpoints can reach the message the same way through the +/// underlying constructor's own context. +/// +/// So rather than dropping the reason — which would cost the diagnostic that +/// makes this log worth having ("dimension mismatch", "build failed") — replace +/// each known endpoint substring with its [`redact_endpoint`] form. Scrubbing +/// the exact strings we already hold is precise, where a generic URL-matching +/// pass over free text would be guesswork (CodeRabbit, #5402 / CWE-532). +fn redact_ladder_error(config: &Config, err: &anyhow::Error) -> String { + use crate::openhuman::memory::util::redact::redact_endpoint; + + // Candidates: the inline `custom:` endpoint (when that is the + // configured form) plus every configured OpenAI-compatible endpoint + // (LM Studio, vLLM, …), any of which the ladder may have been resolving. + let mut endpoints: Vec<&str> = config + .memory + .embedding_provider + .trim() + .strip_prefix("custom:") + .into_iter() + .chain(config.cloud_providers.iter().map(|e| e.endpoint.as_str())) + .map(str::trim) + .filter(|e| !e.is_empty()) + .collect(); + + // Longest first. Substring replacement is order-sensitive: if a short + // endpoint is a strict prefix of a longer one (`https://host` vs + // `https://host/v1?key=…`), scrubbing the short one first rewrites the + // longer one's prefix, so its own replacement no longer matches and the + // credential-bearing suffix survives in the log (CodeRabbit, #5402). + endpoints.sort_by_key(|e| std::cmp::Reverse(e.len())); + endpoints.dedup(); + + let mut msg = format!("{err:#}"); + for endpoint in endpoints { + msg = msg.replace(endpoint, &redact_endpoint(endpoint)); + } + msg +} + +/// Slug naming the embedder ingestion will **actually** use, walking the same +/// [`resolve_embedder_choice`] ladder the read and write factories walk. +/// +/// This exists because `config.memory.embedding_provider` is *not* authoritative +/// for how embeddings are funded, and reading it as if it were produces a false +/// alarm. The ladder resolves local Ollama from `memory_tree.embedding_endpoint` +/// or from the unified `workload_local_model("embeddings")` setting (the "Memory +/// embeddings" toggle in Local AI Settings), and **neither path rewrites +/// `memory.embedding_provider`** — so a user running fully local still reads as +/// `"cloud"` there. Any surface that asks "do these embeddings bill against the +/// managed budget?" must ask this function, not that field (reviewer M3gA-Mind, +/// #5402: otherwise a local-embeddings user whose *chat* budget crosses 90% is +/// told memory has stopped growing while it is growing fine). +/// +/// Slugs are stable wire values consumed by the frontend: +/// - `"ollama"` — local daemon; user-funded. +/// - `"custom"` — user's own OpenAI-compatible endpoint / key; user-funded. +/// - `"cloud"` — managed OpenHuman backend; **bills the managed cycle budget**. +/// - `"none"` — deliberate opt-out (`embeddings_provider = "none"`). +/// - `"unconfigured"` — no usable provider (signed out); nothing is billed. +/// - `"unknown"` — the ladder itself failed to resolve. Deliberately not +/// `"cloud"`: an unresolvable config must never manufacture a budget warning. +pub fn effective_embedder_slug(config: &Config) -> &'static str { + let slug = match resolve_embedder_choice(config) { + Ok(EmbedderChoice::Ollama { .. }) => "ollama", + Ok(EmbedderChoice::OptOut) => "none", + Ok(EmbedderChoice::OpenAiCompat(_)) => "custom", + Ok(EmbedderChoice::Cloud) => "cloud", + Ok(EmbedderChoice::NoProvider) => "unconfigured", + Err(err) => { + log::warn!( + "[memory_tree::embed::factory] effective_embedder_slug: ladder failed to \ + resolve ({}) — reporting 'unknown' (treated as NOT managed)", + redact_ladder_error(config, &err) + ); + "unknown" + } + }; + log::debug!("[memory_tree::embed::factory] effective_embedder_slug → {slug}"); + slug +} + fn build_ollama_embedder(endpoint: &str, model: &str, timeout_ms: u64) -> Result { let timeout = Duration::from_millis(if timeout_ms == 0 { 10_000 } else { timeout_ms }); let client = reqwest::Client::builder() @@ -606,4 +694,171 @@ mod tests { let e = build_embedder_from_config(&cfg).expect("override path should build"); assert_eq!(e.name(), "ollama"); } + + /// The regression this whole helper exists for (reviewer M3gA-Mind, #5402): + /// a user who enabled local embeddings through Local AI Settings still has + /// `memory.embedding_provider == "cloud"` (nothing rewrites it), so any + /// surface reading that field concludes they bill against the managed + /// budget and warns them their memory has stopped growing — while it is + /// growing fine, fully locally, costing them nothing. + #[test] + fn effective_slug_reports_ollama_when_local_ai_overrides_cloud_setting() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + cfg.embeddings_provider = Some("ollama:all-minilm:latest".into()); + cfg.local_ai.runtime_enabled = true; + cfg.local_ai.embedding_model_id = "all-minilm:latest".to_string(); + touch_auth_profile(&cfg); + + // The stale per-section field still says cloud … + assert_eq!(cfg.memory.embedding_provider, "cloud"); + // … but the ladder — and therefore the wire field — says local. + assert_eq!(effective_embedder_slug(&cfg), "ollama"); + } + + #[test] + fn effective_slug_reports_ollama_for_explicit_endpoint_override() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + cfg.memory_tree.embedding_endpoint = Some("http://localhost:11434".into()); + cfg.memory_tree.embedding_model = Some("bge-m3".into()); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "ollama"); + } + + #[test] + fn effective_slug_reports_cloud_only_for_a_real_managed_session() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "cloud"); + } + + #[test] + fn effective_slug_reports_unconfigured_without_a_session() { + // No auth-profiles.json → nothing is billed, so this must not read as + // managed even though the per-section field defaults to cloud. + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + assert_eq!(effective_embedder_slug(&cfg), "unconfigured"); + } + + #[test] + fn effective_slug_reports_none_for_deliberate_opt_out() { + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "none"); + } + + /// The ladder error quotes `memory.embedding_provider` verbatim, and in the + /// `custom:` form that string is a full endpoint URL — potentially with + /// `user:pass@` userinfo. Logging it raw would write credentials to disk + /// (CodeRabbit, #5402 / CWE-532). Scrub the endpoint, keep the reason. + #[test] + fn ladder_error_log_redacts_custom_endpoint_credentials() { + let (_tmp, mut cfg) = test_config(); + // No model + a non-tree dimension → `try_from_config` bails, and its + // message interpolates the provider string. + cfg.memory.embedding_provider = "custom:https://user:pass@embed.example.com/v1".to_string(); + cfg.memory.embedding_model = String::new(); + cfg.memory.embedding_dimensions = 512; + + // `EmbedderChoice` is not `Debug` (it holds a live embedder), so unwrap + // the error by hand rather than via `expect_err`. + let err = match resolve_embedder_choice(&cfg) { + Err(e) => e, + Ok(_) => panic!("a non-tree dimension with no model must fail to resolve"), + }; + let raw = format!("{err:#}"); + assert!( + raw.contains("user:pass"), + "precondition: the unredacted error really does carry the credentials — \ + otherwise this test proves nothing. Got: {raw}" + ); + + let rendered = redact_ladder_error(&cfg, &err); + assert!( + !rendered.contains("user:pass"), + "userinfo must not reach the log: {rendered}" + ); + assert!( + !rendered.contains("/v1"), + "path must not reach the log: {rendered}" + ); + assert!( + rendered.contains("embed.example.com"), + "host is kept so the line stays diagnosable: {rendered}" + ); + assert!( + rendered.contains("1024"), + "the failure reason must survive redaction: {rendered}" + ); + + // And the caller degrades to not-managed rather than to `cloud`. + assert_eq!(effective_embedder_slug(&cfg), "unknown"); + } + + /// Substring replacement is order-sensitive. With a short endpoint that is a + /// strict prefix of the long one, scrubbing shortest-first rewrites the long + /// endpoint's prefix, its own replacement then fails to match, and the + /// credential-bearing suffix survives in the log. Longest-first is the fix + /// (CodeRabbit, #5402). + #[test] + fn ladder_error_redaction_handles_prefix_overlapping_endpoints() { + use crate::openhuman::config::schema::cloud_providers::CloudProviderCreds; + let (_tmp, mut cfg) = test_config(); + // The SHORT endpoint is the one the old code scrubbed first (the inline + // `custom:` form led the list), and it is a strict prefix of the long + // one. That ordering is what let the long endpoint's secret survive: + // scrubbing `https://embed.example.com` first rewrote the long string's + // prefix, so the long string's own replacement no longer matched. + cfg.memory.embedding_provider = "custom:https://embed.example.com".to_string(); + cfg.cloud_providers = vec![CloudProviderCreds { + id: "p_long".to_string(), + slug: "longpfx".to_string(), + endpoint: "https://embed.example.com/v1?key=super-secret".to_string(), + ..Default::default() + }]; + + // Synthesize the error rather than driving the ladder: this pins the + // redaction function's ordering contract for ANY message carrying both + // endpoints, which is the property at risk. Which ladder branch happens + // to surface a `cloud_providers` endpoint today is beside the point. + let err = anyhow::anyhow!( + "build custom embedder failed (provider='custom:https://embed.example.com', \ + endpoint='https://embed.example.com/v1?key=super-secret')" + ); + let rendered = redact_ladder_error(&cfg, &err); + + assert!( + !rendered.contains("super-secret"), + "the long endpoint's query must not survive the short endpoint's scrub: {rendered}" + ); + assert!( + !rendered.contains("/v1"), + "the long endpoint's path must not survive either: {rendered}" + ); + assert!( + rendered.contains("embed.example.com"), + "host is still kept: {rendered}" + ); + } + + #[test] + fn effective_slug_reports_custom_for_byo_openai_compatible() { + use crate::openhuman::config::schema::cloud_providers::CloudProviderCreds; + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = None; + cfg.memory.embedding_provider = "lmstudio".to_string(); + cfg.memory.embedding_model = "bge-m3".to_string(); + cfg.cloud_providers = vec![CloudProviderCreds { + id: "p_lmstudio".to_string(), + slug: "lmstudio".to_string(), + endpoint: "http://localhost:1234/v1".to_string(), + ..Default::default() + }]; + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "custom"); + } } diff --git a/src/openhuman/memory/tree/score/embed/mod.rs b/src/openhuman/memory/tree/score/embed/mod.rs index f92b7c3389..b0950c055f 100644 --- a/src/openhuman/memory/tree/score/embed/mod.rs +++ b/src/openhuman/memory/tree/score/embed/mod.rs @@ -33,7 +33,7 @@ pub mod factory; pub mod inert; pub mod openai_compat; -pub use factory::{build_embedder_from_config, build_write_embedder}; +pub use factory::{build_embedder_from_config, build_write_embedder, effective_embedder_slug}; pub use inert::InertEmbedder; pub use openai_compat::OpenAiCompatEmbedder; diff --git a/src/openhuman/memory/tree/tree/rpc.rs b/src/openhuman/memory/tree/tree/rpc.rs index ff47b7e0a6..933bb3105b 100644 --- a/src/openhuman/memory/tree/tree/rpc.rs +++ b/src/openhuman/memory/tree/tree/rpc.rs @@ -436,9 +436,17 @@ pub async fn pipeline_status_rpc( // #3365 left-right split: of the failed jobs, how many are the hard, // user-actionable kind (`failure_class = 'unrecoverable'`) vs transient ones // that self-heal via auto-requeue. Only the former escalates to `error`. + // + // #5324 rides along in the same blocking task: `oldest_ready_age_ms` is the + // stall signal (queued work that never drains). Kept here rather than in + // its own `spawn_blocking` so a polled status call still costs one + // blocking-pool dispatch for all queue reads. Best-effort — a read error + // degrades to `None` (no stall claimed) instead of failing the RPC, so a + // broken measurement path can never manufacture a `degraded` verdict. let cfg_for_jobs = config.clone(); - let (pipeline_jobs, failed_unrecoverable) = - tokio::task::spawn_blocking(move || -> Result<(PipelineJobCounts, u64), String> { + let now_ms = chrono::Utc::now().timestamp_millis(); + let (pipeline_jobs, failed_unrecoverable, queue_idle_ms) = tokio::task::spawn_blocking( + move || -> Result<(PipelineJobCounts, u64, Option), String> { let ready = queue_store::count_by_status(&cfg_for_jobs, JobStatus::Ready) .map_err(|e| format!("count_by_status(ready): {e:#}"))?; let running = queue_store::count_by_status(&cfg_for_jobs, JobStatus::Running) @@ -447,6 +455,10 @@ pub async fn pipeline_status_rpc( .map_err(|e| format!("count_by_status(failed): {e:#}"))?; let failed_unrecoverable = queue_store::count_failed_unrecoverable(&cfg_for_jobs) .map_err(|e| format!("count_failed_unrecoverable: {e:#}"))?; + let queue_idle_ms = queue_idle_ms(&cfg_for_jobs, now_ms).unwrap_or_else(|e| { + log::warn!("[memory-tree][rpc] pipeline_status: queue_idle_ms read failed: {e}"); + None + }); Ok(( PipelineJobCounts { ready, @@ -454,14 +466,16 @@ pub async fn pipeline_status_rpc( failed, }, failed_unrecoverable, + queue_idle_ms, )) - }) - .await - .map_err(|e| { - let msg = format!("pipeline_status job-count join error: {e}"); - log::warn!("[memory-tree][rpc] pipeline_status: {msg}"); - msg - })??; + }, + ) + .await + .map_err(|e| { + let msg = format!("pipeline_status job-count join error: {e}"); + log::warn!("[memory-tree][rpc] pipeline_status: {msg}"); + msg + })??; // Disk size — best-effort. Permission errors etc. degrade to 0 with a // warn log rather than failing the whole RPC. Scoped to the `wiki/` @@ -498,6 +512,7 @@ pub async fn pipeline_status_rpc( failed_unrecoverable, total_chunks, °raded, + queue_idle_ms, ); // #002: both of these touch SQLite, so run them off the async runtime @@ -672,6 +687,84 @@ fn latest_failed_job_failure( Ok(Some(failure)) } +/// #5324: how long the queue has been sitting on eligible work without +/// finishing anything, or `None` when there is no eligible work waiting. +/// +/// This is the "queued but never processed" signal. Getting the predicate +/// right matters more than it looks, because the naive versions produce false +/// alarms for exactly the heavy users this issue is about: +/// +/// - **Not** `MIN(created_at_ms)` over all `ready` rows. `mark_deferred` parks +/// a backing-off job by leaving `status = 'ready'` and pushing +/// `available_at_ms` forward, so deferred work would count as waiting when +/// it is deliberately asleep. +/// - **Not** the age of the oldest eligible row either. A re-embed backfill +/// enqueues thousands of rows in one burst; six hours into a perfectly +/// healthy drain of a 68k-chunk workspace, the oldest un-drained row is by +/// definition hours old. That would flag the exact case the issue's reporter +/// was in — a big, slow, *working* backfill — as broken. +/// +/// So the measure is **idle time, not backlog age**: how long since the queue +/// last settled *any* job. A pipeline making progress refreshes +/// `completed_at_ms` continuously no matter how deep the backlog is, while a +/// pipeline whose jobs all fail unrecoverably or whose worker never runs goes +/// quiet. `completed_at_ms` is stamped on failure as well as success, so a +/// fast-failing pipeline reports `error` (via `failed_unrecoverable`) rather +/// than being mislabelled as stalled. +/// +/// Returns `Some(idle_ms)` only when eligible work is actually waiting — an +/// idle queue with nothing to do is not stalled, it is done. When nothing has +/// ever settled (fresh workspace whose worker has never run), idle time falls +/// back to how long the oldest eligible job has been waiting. +/// +/// Best-effort like its siblings — a DB error degrades to `Ok(None)` at the +/// call site rather than failing the polled status RPC. +fn queue_idle_ms(config: &Config, now_ms: i64) -> Result, String> { + let row: Option<(i64, Option, Option)> = + chunk_store::with_connection(config, |conn| { + conn.query_row( + "SELECT + (SELECT COUNT(*) FROM mem_tree_jobs + WHERE status = 'ready' AND available_at_ms <= ?1), + (SELECT MAX(completed_at_ms) FROM mem_tree_jobs), + (SELECT MIN(available_at_ms) FROM mem_tree_jobs + WHERE status = 'ready' AND available_at_ms <= ?1)", + [now_ms], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional() + .map_err(Into::into) + }) + .map_err(|e| format!("queue_idle_ms: {e:#}"))?; + + let Some((eligible_ready, last_settled_ms, oldest_eligible_ms)) = row else { + return Ok(None); + }; + // Nothing eligible is waiting ⇒ nothing is being held up. + if eligible_ready <= 0 { + return Ok(None); + } + // Idle time is "how long since the queue last made progress on the work + // that is waiting *now*" — so start the clock at the LATER of the last + // settle and the oldest eligible job's arrival. Using `last_settled_ms` + // alone (`.or`) mis-reads a real shape: if the queue drained everything, + // sat empty for days, then a fresh job arrives, the stale completion is + // hours/days old while the new work is seconds old. Taking the max means + // freshly-enqueued work starts its own idle window instead of inheriting + // an ancient completion, so a just-arrived job can't be flagged `degraded` + // before the worker has had a chance to touch it. Fall back to the oldest + // eligible job's wait when the queue has never settled a job at all. + let reference_ms = match (last_settled_ms, oldest_eligible_ms) { + (Some(last_settled), Some(oldest_eligible)) => Some(last_settled.max(oldest_eligible)), + (Some(last_settled), None) => Some(last_settled), + (None, Some(oldest_eligible)) => Some(oldest_eligible), + (None, None) => None, + }; + // Clamp at zero: clock skew / a future-dated row must read as "just now", + // never as a negative age. + Ok(reference_ms.map(|since| (now_ms - since).max(0))) +} + /// Recursive byte-count of files under `root`. Returns `0` when the root /// does not exist or any traversal error occurs (best-effort; the status /// panel is a UI convenience, not an audit surface). @@ -706,9 +799,24 @@ fn compute_dir_size_bytes(root: &std::path::Path) -> u64 { total } +/// #5324: how long the queue may hold eligible work without settling a single +/// job before the pipeline is reported as `degraded` rather than +/// `running`/`idle`. +/// +/// A working pipeline settles jobs continuously, so this is idle time, not +/// backlog depth — a deep-but-draining backfill never approaches it. Six hours +/// is far outside a normal flush window (minutes) yet well inside the "broken +/// for a month" window the issue describes, so it cannot fire on a busy +/// machine or a laptop that was asleep for an hour. +pub(crate) const QUEUE_STALL_THRESHOLD_MS: i64 = 6 * 60 * 60 * 1000; + /// Pure derivation of `(status, reason)` from raw signals. Split out so the /// unit tests can exercise the precedence rules without spinning up a /// store. +/// +/// `queue_idle_ms` is how long the queue has held eligible work without +/// settling any job, or `None` when no eligible work is waiting (or the +/// metric could not be read). fn derive_pipeline_status( is_paused: bool, mode: crate::openhuman::config::SchedulerGateMode, @@ -717,6 +825,7 @@ fn derive_pipeline_status( failed_unrecoverable: u64, total_chunks: u64, degraded: &crate::openhuman::memory::tree::health::DegradedState, + queue_idle_ms: Option, ) -> (String, Option) { if is_paused { return ( @@ -751,6 +860,27 @@ fn derive_pipeline_status( )), ); } + // #5324: the queue is accepting work but not draining it. This is the + // "silently broken for a month" shape — new files keep getting detected + // and queued, health checks keep reporting `ok` because the process is + // alive, and nothing ever becomes searchable memory. Liveness is not + // output, so a queue whose oldest ready job has been waiting past the + // threshold reports `degraded`, never `running`/`idle`. + // + // Sits below `error` (a typed unrecoverable failure is the more specific + // diagnosis and carries its own remediation) and above the recall/structure + // degradation, and is deliberately NOT gated on `total_chunks` — a queue + // that never drained has no chunks to gate on, which is exactly the case + // that must not read as `idle`. + if queue_idle_ms.is_some_and(|idle| idle >= QUEUE_STALL_THRESHOLD_MS) { + let hours = queue_idle_ms.unwrap_or(0) / (60 * 60 * 1000); + return ( + "degraded".to_string(), + Some(format!( + "queue has not completed any job in {hours}h — memory is not growing" + )), + ); + } // #002 (FR-005): "degraded" sits below error but above syncing/running — // the pipeline is making progress, but recall/structure is reduced (or some // jobs failed transiently and are retrying) and the user should be told why. @@ -1270,7 +1400,7 @@ mod tests { }; // Args: (is_paused, mode, is_syncing, failed, failed_unrecoverable, - // total_chunks, °raded). + // total_chunks, °raded, queue_idle_ms). // paused beats everything else (even degradation) let (s, reason) = derive_pipeline_status( @@ -1281,6 +1411,7 @@ mod tests { 5, 100, &recall_degraded, + None, ); assert_eq!(s, "paused"); assert!(reason.unwrap().contains("off")); @@ -1295,6 +1426,7 @@ mod tests { 0, 0, &storage_degraded, + None, ); assert_eq!(s, "paused", "paused beats storage"); @@ -1309,6 +1441,7 @@ mod tests { 0, 0, // no chunks — must still surface &storage_degraded, + None, ); assert_eq!( s, "error", @@ -1325,6 +1458,7 @@ mod tests { 0, 100, &storage_degraded, + None, ); assert_eq!(s, "error", "storage beats transient-degraded"); @@ -1338,6 +1472,7 @@ mod tests { 2, // both failures unrecoverable 100, &recall_degraded, + None, ); assert_eq!(s, "error"); assert!(reason.unwrap().contains("unrecoverable")); @@ -1345,8 +1480,16 @@ mod tests { // #3365: transient-only failures (failed > 0, none unrecoverable) do NOT // escalate to error — they self-heal via auto-requeue, so they surface // as `degraded` ("retrying"), beating syncing/running. - let (s, reason) = - derive_pipeline_status(false, SchedulerGateMode::Auto, true, 3, 0, 100, &healthy); + let (s, reason) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + true, + 3, + 0, + 100, + &healthy, + None, + ); assert_eq!(s, "degraded", "transient failures must not read as error"); assert!(reason.unwrap().contains("3 job(s) failed, retrying")); @@ -1359,6 +1502,7 @@ mod tests { 0, 100, &recall_degraded, + None, ); assert_eq!(s, "degraded", "degraded must beat syncing"); assert!(reason.unwrap().contains("semantic recall disabled")); @@ -1371,28 +1515,296 @@ mod tests { 0, 100, &structure_degraded, + None, ); assert_eq!(s, "degraded"); assert!(reason.unwrap().contains("wiki structure incomplete")); // syncing beats running / idle (when healthy) - let (s, reason) = - derive_pipeline_status(false, SchedulerGateMode::Auto, true, 0, 0, 100, &healthy); + let (s, reason) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + true, + 0, + 0, + 100, + &healthy, + None, + ); assert_eq!(s, "syncing"); assert!(reason.is_none()); // running when chunks exist but nothing in flight - let (s, _) = - derive_pipeline_status(false, SchedulerGateMode::Auto, false, 0, 0, 100, &healthy); + let (s, _) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 0, + 0, + 100, + &healthy, + None, + ); assert_eq!(s, "running"); // idle when the store is empty and nothing is in flight (transient // failures with no content don't manufacture a `degraded`). - let (s, _) = - derive_pipeline_status(false, SchedulerGateMode::Auto, false, 2, 0, 0, &healthy); + let (s, _) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 2, + 0, + 0, + &healthy, + None, + ); assert_eq!(s, "idle"); } + /// #5324: a queue that accepts work but never drains it must report + /// `degraded`, not the `running`/`idle` that made a month-long outage look + /// healthy. Pins the threshold boundary and the full precedence chain. + #[test] + fn stalled_queue_degrades_instead_of_reading_healthy() { + use crate::openhuman::config::SchedulerGateMode; + use crate::openhuman::memory::tree::health::{DegradedState, FailureCode, PipelineFailure}; + + let healthy = DegradedState::default(); + let stalled = Some(QUEUE_STALL_THRESHOLD_MS); + let just_under = Some(QUEUE_STALL_THRESHOLD_MS - 1); + + // The regression itself: chunks exist, nothing failed, nothing running + // — previously "running", which is what let the outage hide. + let (s, reason) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 0, + 0, + 100, + &healthy, + stalled, + ); + assert_eq!(s, "degraded", "a stalled queue must not read as running"); + assert!(reason.unwrap().contains("has not completed any job")); + + // NOT gated on total_chunks: a queue that never drained has no chunks, + // and that case must not read as `idle`. + let (s, _) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 0, + 0, + 0, + &healthy, + stalled, + ); + assert_eq!(s, "degraded", "empty-but-stalled must not read as idle"); + + // Boundary: one millisecond under the threshold is still healthy, so a + // merely slow flush window can't trip it. + let (s, _) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 0, + 0, + 100, + &healthy, + just_under, + ); + assert_eq!(s, "running", "under the threshold stays healthy"); + + // `None` (no ready jobs, or an unreadable metric) never manufactures a + // degraded verdict. + let (s, _) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 0, + 0, + 100, + &healthy, + None, + ); + assert_eq!(s, "running", "absent metric must not claim a stall"); + + // Precedence: paused and error both outrank the stall — a typed + // unrecoverable failure is the more specific, more actionable answer. + let (s, _) = derive_pipeline_status( + true, + SchedulerGateMode::Off, + false, + 0, + 0, + 100, + &healthy, + stalled, + ); + assert_eq!(s, "paused", "paused beats stalled"); + + let (s, reason) = derive_pipeline_status( + false, + SchedulerGateMode::Auto, + false, + 1, + 1, + 100, + &healthy, + stalled, + ); + assert_eq!(s, "error", "unrecoverable failure beats stalled"); + assert!(reason.unwrap().contains("unrecoverable")); + + // Sanity: the budget-exhausted failure this issue is about is indeed + // classified unrecoverable, so it lands in the `error` branch above and + // carries its own remediation key. + let budget = PipelineFailure::new(FailureCode::BudgetExhausted); + assert!(budget.is_unrecoverable()); + assert_eq!( + budget.remediation_key, + "memory.health.remediation.budget_exhausted" + ); + } + + /// #5324: `queue_idle_ms` measures idle time, not backlog depth. Pins the + /// two shapes that must NOT be reported as stalled, both of which a + /// backlog-age metric would have flagged — and both of which describe the + /// heavy users this issue is about. + #[tokio::test] + async fn queue_idle_ms_ignores_deep_but_draining_and_deferred_backlogs() { + use crate::openhuman::memory::queue::store as queue_store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, NewJob}; + + let (_tmp, cfg) = test_config(); + let now = 1_800_000_000_000_i64; + let long_ago = now - 48 * 60 * 60 * 1000; + + // Nothing queued at all ⇒ not stalled (an empty queue is done, not stuck). + assert_eq!(queue_idle_ms(&cfg, now).unwrap(), None); + + // A deep backlog whose oldest row was enqueued 48h ago. A naive + // MIN(created_at_ms) reads 48h and cries "stalled"; the pipeline is in + // fact draining, which we simulate by settling one job just now. + for i in 0..3 { + let job = + NewJob::flush_stale(&FlushStalePayload::default(), &format!("2026-08-0{i}"), 3) + .unwrap(); + queue_store::enqueue(&cfg, &job).unwrap(); + } + chunk_store::with_connection(&cfg, |conn| { + conn.execute( + "UPDATE mem_tree_jobs SET created_at_ms = ?1, available_at_ms = ?1", + [long_ago], + )?; + Ok(()) + }) + .unwrap(); + + // Never settled anything yet ⇒ falls back to the oldest eligible wait, + // which is the genuine "worker has never run" case. + assert!( + queue_idle_ms(&cfg, now).unwrap().unwrap() >= QUEUE_STALL_THRESHOLD_MS, + "a queue that has never settled a job IS stalled" + ); + + // Now mark one job as settled a minute ago — the pipeline is draining. + chunk_store::with_connection(&cfg, |conn| { + conn.execute( + "UPDATE mem_tree_jobs SET status = 'done', completed_at_ms = ?1 + WHERE id = (SELECT id FROM mem_tree_jobs LIMIT 1)", + [now - 60_000], + )?; + Ok(()) + }) + .unwrap(); + let idle = queue_idle_ms(&cfg, now) + .unwrap() + .expect("work still queued"); + assert!( + idle < QUEUE_STALL_THRESHOLD_MS, + "a deep but draining backlog must not read as stalled (idle={idle}ms)" + ); + + // Deferred work: `mark_deferred` leaves status='ready' and pushes + // available_at_ms into the future. Those rows are asleep on purpose and + // must not count as eligible waiting work. + chunk_store::with_connection(&cfg, |conn| { + conn.execute( + "UPDATE mem_tree_jobs SET status = 'ready', available_at_ms = ?1", + [now + 60 * 60 * 1000], + )?; + Ok(()) + }) + .unwrap(); + assert_eq!( + queue_idle_ms(&cfg, now).unwrap(), + None, + "wholly-deferred work is asleep, not stalled" + ); + } + + /// #5324 regression (CodeRabbit/Codex): a queue that drained everything, + /// sat quiet for two days, then received one fresh eligible job must start + /// the idle clock at the NEW job's arrival — not inherit the ancient + /// completion. The prior `last_settled_ms.or(oldest_eligible_ms)` picked + /// the stale 48h-old settle and reported `degraded` the instant new work + /// appeared, before the worker had any chance to touch it. + #[tokio::test] + async fn queue_idle_ms_starts_from_fresh_work_not_ancient_completion() { + use crate::openhuman::memory::queue::store as queue_store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, NewJob}; + + let (_tmp, cfg) = test_config(); + let now = 1_800_000_000_000_i64; + let long_ago = now - 48 * 60 * 60 * 1000; + let just_now = now - 60_000; + + // Job A: the last thing the queue settled, 48h ago, then it went quiet. + let job_a = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-01", 3).unwrap(); + let id_a = queue_store::enqueue(&cfg, &job_a) + .unwrap() + .expect("enqueue A"); + // Job B: a brand-new eligible job that arrived a minute ago. + let job_b = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-02", 3).unwrap(); + let id_b = queue_store::enqueue(&cfg, &job_b) + .unwrap() + .expect("enqueue B"); + + chunk_store::with_connection(&cfg, |conn| { + conn.execute( + "UPDATE mem_tree_jobs + SET status = 'done', completed_at_ms = ?2, available_at_ms = ?2 + WHERE id = ?1", + rusqlite::params![id_a, long_ago], + )?; + conn.execute( + "UPDATE mem_tree_jobs + SET status = 'ready', completed_at_ms = NULL, available_at_ms = ?2 + WHERE id = ?1", + rusqlite::params![id_b, just_now], + )?; + Ok(()) + }) + .unwrap(); + + let idle = queue_idle_ms(&cfg, now) + .unwrap() + .expect("fresh work is waiting"); + assert!( + idle < QUEUE_STALL_THRESHOLD_MS, + "freshly-enqueued work must start its own idle window, not inherit a 48h-old \ + completion (idle={idle}ms)" + ); + assert_eq!( + idle, + now - just_now, + "the idle clock starts at the new job's arrival, not the stale settle" + ); + } + /// On a fresh workspace the panel must report `idle` with zero /// counters — the UI uses this to swap the loading skeleton for a /// "no memory yet" state. diff --git a/tests/embeddings_rpc_e2e.rs b/tests/embeddings_rpc_e2e.rs index 444e726581..435a37da50 100644 --- a/tests/embeddings_rpc_e2e.rs +++ b/tests/embeddings_rpc_e2e.rs @@ -294,6 +294,28 @@ async fn embeddings_get_settings_returns_catalog() { "get_settings result missing 'dimensions': {inner}" ); + // `effective_provider` must survive serialization to the wire, not just + // exist on the handler's return value (#5402). The frontend budget warning + // gates on this field: if it silently stopped being emitted, the hook would + // fall back to `provider` — which is exactly the stale value that produced + // the false "memory has stopped growing" alarm for local-embeddings users. + let effective = inner + .get("effective_provider") + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("get_settings result missing 'effective_provider': {inner}")); + assert!( + [ + "ollama", + "custom", + "cloud", + "none", + "unconfigured", + "unknown" + ] + .contains(&effective), + "effective_provider must be one of the documented slugs, got {effective:?}" + ); + // Provider catalog let providers = inner .get("providers")