diff --git a/.gitignore b/.gitignore
index ab13dec04..e9116a86c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,5 +86,10 @@ fastlane/*.p8
!.yarn/sdks
!.yarn/versions
docs/TRACTION_KNOWLEDGE_BASE.md
+# Dev-only insights sim harness + private real recordings (never commit)
+/sim/
+# Debug screenshots (scratch)
+/image.png
+/image-*.png
# Local marketing drafts (not part of the app)
marketing/
diff --git a/App.tsx b/App.tsx
index fd5968276..3560ffb51 100644
--- a/App.tsx
+++ b/App.tsx
@@ -29,6 +29,7 @@ import { LockScreen } from './src/screens';
import { useAppState } from './src/hooks/useAppState';
import { useDownloadStore } from './src/stores/downloadStore';
import { ErrorBoundary } from './src/components/ErrorBoundary';
+import { Toast } from './src/components';
LogBox.ignoreAllLogs(); // Suppress all logs
@@ -351,6 +352,7 @@ function App() {
>
+
);
diff --git a/__tests__/integration/locket/compressedTranscription.test.ts b/__tests__/integration/locket/compressedTranscription.test.ts
deleted file mode 100644
index 76e3bfbb3..000000000
--- a/__tests__/integration/locket/compressedTranscription.test.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Integration test: compressed (.m4a) recordings transcribe correctly.
- *
- * The real bug class this guards: transcription slices audio with
- * extractWavSlice, which RIFF-parses a WAV header. A compressed .m4a has no such
- * header, so without the decode step it would slice garbage. This test wires the
- * REAL transcribeChunked + the REAL resolveToWav (recordingCompression) together,
- * mocking only the native/whisper leaves, and asserts:
- * - an .m4a source is decoded ONCE (normalizeToWav16kMono) before slicing,
- * - extractWavSlice is called on the DECODED wav, never on the .m4a,
- * - the decoded temp file is cleaned up afterwards,
- * - a plain .wav source is NEVER decoded (the no-op fast path).
- */
-
-const mockExtractWavSlice = jest.fn();
-const mockNormalize = jest.fn();
-
-jest.mock('react-native', () => ({
- NativeModules: {
- AudioNormalizer: {
- extractWavSlice: (s: string, a: number, d: number) => mockExtractWavSlice(s, a, d),
- normalizeToWav16kMono: (i: string, o: string) => mockNormalize(i, o),
- },
- },
-}));
-
-const mockUnlink = jest.fn().mockResolvedValue(undefined);
-jest.mock('react-native-fs', () => ({
- CachesDirectoryPath: '/caches',
- exists: jest.fn().mockResolvedValue(true),
- unlink: (p: string) => mockUnlink(p),
-}));
-
-jest.mock('@offgrid/core/services/whisperService', () => ({
- whisperService: {
- transcribeFile: jest.fn().mockResolvedValue('hello world'),
- },
-}));
-
-jest.mock('@offgrid/core/utils/memorySnapshot', () => ({ logMemory: jest.fn().mockResolvedValue(undefined) }));
-jest.mock('@offgrid/core/utils/logger', () => ({
- __esModule: true,
- default: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
-}));
-jest.mock('../../../pro/locket/services/transcriptionForeground', () => ({
- transcriptionForeground: { start: jest.fn().mockResolvedValue(undefined), stop: jest.fn().mockResolvedValue(undefined) },
-}));
-
-// Minimal in-memory store: transcribeChunked reads checkpoint/segments and writes back.
-const state: Record = {};
-jest.mock('../../../pro/locket/stores', () => ({
- useRecordingsStore: {
- getState: () => ({
- recordings: [{ id: 'rec-1', ...state }],
- updateRecording: (_id: string, patch: Record) => Object.assign(state, patch),
- }),
- },
-}));
-// recordingCompression imports the store from '../stores/recordingsStore' - mock that path too.
-jest.mock('../../../pro/locket/stores/recordingsStore', () => ({
- useRecordingsStore: { getState: () => ({ updateRecording: jest.fn() }) },
-}));
-
-import { transcribeChunked } from '../../../pro/locket/services/transcribeChunked';
-
-beforeEach(() => {
- jest.clearAllMocks();
- for (const k of Object.keys(state)) delete state[k];
- mockExtractWavSlice.mockImplementation((_s, _a, _d) => Promise.resolve('/tmp/slice.wav'));
- mockNormalize.mockResolvedValue('/docs/.decode-rec-100.wav');
-});
-
-it('decodes a compressed .m4a once, then slices the DECODED wav (never the m4a)', async () => {
- await transcribeChunked({ id: 'rec-1', path: '/docs/rec-100.m4a', durationMs: 60_000 });
-
- // decoded exactly once, from the .m4a, into the caches dir with a unique name
- expect(mockNormalize).toHaveBeenCalledTimes(1);
- expect(mockNormalize).toHaveBeenCalledWith('/docs/rec-100.m4a', expect.stringMatching(/^\/caches\/decode-rec-100-[^/]+\.wav$/));
- const decodedWav = mockNormalize.mock.calls[0][1];
-
- // every slice targets the DECODED wav, and NEVER the .m4a
- expect(mockExtractWavSlice).toHaveBeenCalled();
- for (const call of mockExtractWavSlice.mock.calls) {
- expect(call[0]).toBe(decodedWav);
- }
- expect(mockExtractWavSlice).not.toHaveBeenCalledWith('/docs/rec-100.m4a', expect.anything(), expect.anything());
-
- // temp decoded wav cleaned up
- expect(mockUnlink).toHaveBeenCalledWith(decodedWav);
-});
-
-it('never decodes a plain .wav source (no-op fast path)', async () => {
- await transcribeChunked({ id: 'rec-1', path: '/docs/rec-100.wav', durationMs: 60_000 });
-
- expect(mockNormalize).not.toHaveBeenCalled();
- // slices go straight against the original wav
- for (const call of mockExtractWavSlice.mock.calls) {
- expect(call[0]).toBe('/docs/rec-100.wav');
- }
-});
diff --git a/__tests__/unit/locket/dayTimeline.test.ts b/__tests__/unit/locket/dayTimeline.test.ts
deleted file mode 100644
index 7a550d99e..000000000
--- a/__tests__/unit/locket/dayTimeline.test.ts
+++ /dev/null
@@ -1,99 +0,0 @@
-import {
- buildDayTimeline,
- timeBucket,
- clipState,
- recordingsForDay,
-} from '../../../pro/locket/utils/dayTimeline';
-import type { Recording } from '../../../pro/locket/stores/recordingsStore';
-
-// Build a minimal Recording on 2026-07-08 at a given hour/min.
-const mk = (id: string, hour: number, min: number, opts: Partial = {}): Recording => {
- const startedAt = new Date(2026, 6, 8, hour, min, 0).getTime();
- return {
- id,
- path: `/rec/${id}.wav`,
- startedAt,
- endedAt: startedAt + 30_000,
- durationMs: 30_000,
- sizeBytes: 1000,
- ...opts,
- } as Recording;
-};
-
-describe('timeBucket (fixed clock boundaries)', () => {
- const at = (h: number) => timeBucket(new Date(2026, 6, 8, h, 0).getTime());
- it('maps hours to fixed buckets', () => {
- expect(at(6)).toBe('Morning'); // 5-12
- expect(at(11)).toBe('Morning');
- expect(at(12)).toBe('Afternoon'); // 12-17
- expect(at(16)).toBe('Afternoon');
- expect(at(17)).toBe('Evening'); // 17-21
- expect(at(20)).toBe('Evening');
- expect(at(21)).toBe('Night'); // 21-5
- expect(at(3)).toBe('Night');
- });
-});
-
-describe('clipState (three states, never blurred)', () => {
- it('raw when not transcribed', () => {
- expect(clipState(mk('a', 9, 0))).toBe('raw');
- });
- it('text when transcript has content', () => {
- expect(clipState(mk('a', 9, 0, { transcript: 'hello there' }))).toBe('text');
- });
- it('nospeech when transcribed but empty', () => {
- expect(clipState(mk('a', 9, 0, { transcript: ' ', transcriptStatus: 'done' }))).toBe('nospeech');
- });
-});
-
-describe('buildDayTimeline', () => {
- it('groups loose clips under time-of-day headers', () => {
- const items = buildDayTimeline([mk('a', 8, 0), mk('b', 13, 0)]);
- expect(items.map((i) => i.kind)).toEqual(['timeHeader', 'clip', 'timeHeader', 'clip']);
- expect((items[0] as any).label).toBe('Morning');
- expect((items[2] as any).label).toBe('Afternoon');
- });
-
- it('collapses same-eventId clips into one meeting block', () => {
- const items = buildDayTimeline([
- mk('a', 9, 0, { eventId: 'E1', eventTitle: 'Standup' }),
- mk('b', 9, 10, { eventId: 'E1', eventTitle: 'Standup' }),
- ]);
- const meetings = items.filter((i) => i.kind === 'meeting');
- expect(meetings).toHaveLength(1);
- expect((meetings[0] as any).title).toBe('Standup');
- expect((meetings[0] as any).clips).toHaveLength(2);
- });
-
- it('does not repeat a time header after a meeting (once per bucket)', () => {
- const items = buildDayTimeline([
- mk('a', 8, 0), // loose morning
- mk('b', 9, 0, { eventId: 'E1', eventTitle: 'Standup' }), // meeting
- mk('c', 9, 40), // loose morning again - no second Morning header
- ]);
- expect(items.map((i) => i.kind)).toEqual([
- 'timeHeader', // Morning (once)
- 'clip', // a
- 'meeting', // Standup
- 'clip', // c
- ]);
- expect(items.filter((i) => i.kind === 'timeHeader')).toHaveLength(1);
- });
-
- it('orders everything by start time', () => {
- const items = buildDayTimeline([mk('late', 15, 0), mk('early', 7, 0)]);
- const clips = items.filter((i) => i.kind === 'clip') as any[];
- expect(clips[0].clip.id).toBe('early');
- expect(clips[1].clip.id).toBe('late');
- });
-});
-
-describe('recordingsForDay', () => {
- it('keeps only the same local calendar day', () => {
- const today = mk('t', 10, 0).startedAt;
- const other = new Date(2026, 6, 7, 10, 0).getTime();
- const list = [mk('t', 10, 0), { ...mk('o', 10, 0), startedAt: other } as Recording];
- const kept = recordingsForDay(list, today);
- expect(kept.map((r) => r.id)).toEqual(['t']);
- });
-});
diff --git a/__tests__/unit/locket/meetingSchedule.test.ts b/__tests__/unit/locket/meetingSchedule.test.ts
deleted file mode 100644
index da2af2ee3..000000000
--- a/__tests__/unit/locket/meetingSchedule.test.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Unit tests for the pure meeting-reminder scheduling math (selection window,
- * fire time, notification ids). No notifee / calendar / native imports.
- */
-import {
- selectUpcomingMeetings,
- reminderFireTime,
- reminderNotificationId,
- isReminderId,
-} from '../../../pro/locket/utils/meetingSchedule';
-import type { MatchEvent } from '../../../pro/locket/utils/calendarMatch';
-
-const iso = (ms: number) => new Date(ms).toISOString();
-const NOW = 1_000_000_000_000; // fixed "now"
-const WINDOW = 24 * 60 * 60 * 1000; // 24h
-const MIN = 60_000;
-
-const ev = (over: Partial): MatchEvent => ({
- id: 'e',
- title: 'Standup',
- startDate: iso(NOW + 30 * MIN),
- endDate: iso(NOW + 60 * MIN),
- ...over,
-});
-
-describe('selectUpcomingMeetings', () => {
- it('keeps a timed meeting starting inside the window', () => {
- const out = selectUpcomingMeetings([ev({})], NOW, WINDOW);
- expect(out).toHaveLength(1);
- expect(out[0]).toMatchObject({ id: 'e', title: 'Standup', startMs: NOW + 30 * MIN, endMs: NOW + 60 * MIN });
- });
-
- it('drops meetings that already started (in the past)', () => {
- expect(selectUpcomingMeetings([ev({ id: 'p', startDate: iso(NOW - MIN) })], NOW, WINDOW)).toEqual([]);
- });
-
- it('drops meetings beyond the window', () => {
- expect(selectUpcomingMeetings([ev({ id: 'far', startDate: iso(NOW + WINDOW + MIN) })], NOW, WINDOW)).toEqual([]);
- });
-
- it('skips all-day events (they would match the whole day)', () => {
- expect(selectUpcomingMeetings([ev({ allDay: true })], NOW, WINDOW)).toEqual([]);
- });
-
- it('skips events with an unparseable start', () => {
- expect(selectUpcomingMeetings([ev({ startDate: 'not-a-date' })], NOW, WINDOW)).toEqual([]);
- });
-
- it('dedupes by event id', () => {
- const out = selectUpcomingMeetings([ev({}), ev({})], NOW, WINDOW);
- expect(out).toHaveLength(1);
- });
-
- it('sorts by start time', () => {
- const a = ev({ id: 'a', startDate: iso(NOW + 3 * 60 * MIN) });
- const b = ev({ id: 'b', startDate: iso(NOW + 30 * MIN) });
- expect(selectUpcomingMeetings([a, b], NOW, WINDOW).map((m) => m.id)).toEqual(['b', 'a']);
- });
-
- it('defaults a missing/invalid end to start + 1h, and a blank title to "Meeting"', () => {
- const out = selectUpcomingMeetings([ev({ endDate: undefined, title: ' ' })], NOW, WINDOW);
- expect(out[0].endMs).toBe(NOW + 30 * MIN + 60 * MIN);
- expect(out[0].title).toBe('Meeting');
- });
-});
-
-describe('reminderFireTime', () => {
- it('fires leadMs before the start', () => {
- expect(reminderFireTime(NOW + 30 * MIN, 2 * MIN, NOW)).toBe(NOW + 28 * MIN);
- });
-
- it('never returns a time in the past when the lead exceeds the time-to-start', () => {
- // Meeting in 1 min, lead 5 min -> would be in the past; clamps to ~now.
- expect(reminderFireTime(NOW + MIN, 5 * MIN, NOW)).toBe(NOW + 1000);
- });
-});
-
-describe('reminderNotificationId / isReminderId', () => {
- it('round-trips a stable, recognizable id', () => {
- const id = reminderNotificationId('event-42');
- expect(id).toBe('meeting-reminder-event-42');
- expect(isReminderId(id)).toBe(true);
- expect(isReminderId('some-other-notification')).toBe(false);
- });
-});
diff --git a/__tests__/unit/locket/recordingCompression.test.ts b/__tests__/unit/locket/recordingCompression.test.ts
deleted file mode 100644
index fd71f6468..000000000
--- a/__tests__/unit/locket/recordingCompression.test.ts
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Unit tests for manual recording compression (recordingCompression.ts).
- *
- * Covers the safety contract: encode -> verify -> only THEN drop the raw; a
- * failed/empty/too-large encode leaves the original untouched; already-compressed
- * is a no-op; and resolveToWav decodes .m4a but passes .wav through untouched.
- * Native AudioNormalizer, RNFS, and the store are mocked so we exercise only the
- * decision logic (no real audio work).
- */
-
-const mockCompressToAac = jest.fn();
-const mockNormalize = jest.fn();
-
-jest.mock('react-native', () => ({
- NativeModules: {
- AudioNormalizer: {
- compressToAac: (s: string, o: string) => mockCompressToAac(s, o),
- normalizeToWav16kMono: (i: string, o: string) => mockNormalize(i, o),
- },
- },
-}));
-
-const mockRNFS = {
- exists: jest.fn(),
- unlink: jest.fn(),
-};
-jest.mock('react-native-fs', () => ({
- CachesDirectoryPath: '/caches',
- exists: (p: string) => mockRNFS.exists(p),
- unlink: (p: string) => mockRNFS.unlink(p),
-}));
-
-const mockUpdate = jest.fn();
-jest.mock('../../../pro/locket/stores/recordingsStore', () => ({
- useRecordingsStore: { getState: () => ({ updateRecording: mockUpdate }) },
-}));
-
-jest.mock('@offgrid/core/utils/logger', () => ({
- __esModule: true,
- default: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
-}));
-
-import {
- compressRecording,
- isCompressed,
- resolveToWav,
-} from '../../../pro/locket/services/recordingCompression';
-import type { Recording } from '../../../pro/locket/stores/recordingsStore';
-
-const rec = (over: Partial = {}): Recording => ({
- id: 'rec-1',
- path: '/docs/Music/Recordings/rec-100.wav',
- startedAt: 100,
- endedAt: 100000,
- durationMs: 99900,
- sizeBytes: 1_000_000,
- ...over,
-} as Recording);
-
-beforeEach(() => {
- jest.clearAllMocks();
- mockRNFS.exists.mockResolvedValue(true);
- mockRNFS.unlink.mockResolvedValue(undefined);
-});
-
-describe('isCompressed', () => {
- it('is false for .wav, true for .m4a', () => {
- expect(isCompressed({ path: '/a/rec-1.wav' })).toBe(false);
- expect(isCompressed({ path: '/a/rec-1.m4a' })).toBe(true);
- expect(isCompressed({ path: '/a/rec-1.WAV' })).toBe(false); // case-insensitive
- });
-});
-
-describe('compressRecording - happy path', () => {
- it('encodes, verifies, repoints the store, then drops the raw', async () => {
- mockCompressToAac.mockResolvedValue({ path: '/docs/Music/Recordings/rec-100.m4a', sizeBytes: 90_000 });
- const r = rec();
- const res = await compressRecording(r);
- expect(res).toEqual({ ok: true, savedBytes: 910_000, newSizeBytes: 90_000 });
- // store repointed to the .m4a with the new size
- expect(mockUpdate).toHaveBeenCalledWith('rec-1', {
- path: '/docs/Music/Recordings/rec-100.m4a',
- sizeBytes: 90_000,
- });
- // raw dropped (the .wav), AFTER the update
- expect(mockRNFS.unlink).toHaveBeenCalledWith('/docs/Music/Recordings/rec-100.wav');
- });
-});
-
-describe('compressRecording - safety guards (raw never lost)', () => {
- it('no-ops when already compressed', async () => {
- const res = await compressRecording(rec({ path: '/docs/rec-100.m4a' }));
- expect(res).toEqual({ ok: false, reason: 'already-compressed' });
- expect(mockCompressToAac).not.toHaveBeenCalled();
- expect(mockUpdate).not.toHaveBeenCalled();
- });
-
- it('keeps the raw when the source file is missing', async () => {
- mockRNFS.exists.mockResolvedValue(false);
- const res = await compressRecording(rec());
- expect(res).toEqual({ ok: false, reason: 'file-missing' });
- expect(mockCompressToAac).not.toHaveBeenCalled();
- });
-
- it('keeps the raw when the encoder throws', async () => {
- mockCompressToAac.mockRejectedValue(new Error('AVAssetWriter failed'));
- const res = await compressRecording(rec());
- expect(res).toEqual({ ok: false, reason: 'encode-failed' });
- expect(mockUpdate).not.toHaveBeenCalled();
- // never unlinked the raw
- expect(mockRNFS.unlink).not.toHaveBeenCalledWith('/docs/Music/Recordings/rec-100.wav');
- });
-
- it('keeps the raw when the output verifies as empty', async () => {
- mockCompressToAac.mockResolvedValue({ path: '/docs/Music/Recordings/rec-100.m4a', sizeBytes: 10 });
- const res = await compressRecording(rec());
- expect(res).toEqual({ ok: false, reason: 'verify-failed' });
- expect(mockUpdate).not.toHaveBeenCalled();
- });
-
- it('keeps the raw when the output is not actually smaller', async () => {
- mockCompressToAac.mockResolvedValue({ path: '/docs/Music/Recordings/rec-100.m4a', sizeBytes: 1_200_000 });
- const res = await compressRecording(rec({ sizeBytes: 1_000_000 }));
- expect(res).toEqual({ ok: false, reason: 'not-smaller' });
- expect(mockUpdate).not.toHaveBeenCalled();
- // the bogus larger output is cleaned up
- expect(mockRNFS.unlink).toHaveBeenCalledWith('/docs/Music/Recordings/rec-100.m4a');
- });
-});
-
-describe('resolveToWav', () => {
- it('passes a .wav through untouched with a no-op cleanup (no decode)', async () => {
- const { wavPath, cleanup } = await resolveToWav('/docs/rec-1.wav');
- expect(wavPath).toBe('/docs/rec-1.wav');
- expect(mockNormalize).not.toHaveBeenCalled();
- await cleanup();
- expect(mockRNFS.unlink).not.toHaveBeenCalled();
- });
-
- it('decodes a .m4a to a temp WAV in the caches dir (not recordings) and cleanup removes it', async () => {
- mockNormalize.mockResolvedValue('ok');
- const { wavPath, cleanup } = await resolveToWav('/docs/rec-1.m4a');
- // Temp lands in the CACHES dir (not the recordings dir, so recovery can't
- // surface it) with a unique name (so concurrent jobs don't collide).
- expect(mockNormalize).toHaveBeenCalledWith('/docs/rec-1.m4a', expect.stringMatching(/^\/caches\/decode-rec-1-[^/]+\.wav$/));
- expect(wavPath).toMatch(/^\/caches\/decode-rec-1-[^/]+\.wav$/);
- await cleanup();
- expect(mockRNFS.unlink).toHaveBeenCalledWith(wavPath);
- });
-});
diff --git a/__tests__/unit/locket/recordingSearch.test.ts b/__tests__/unit/locket/recordingSearch.test.ts
deleted file mode 100644
index 8e7819ff1..000000000
--- a/__tests__/unit/locket/recordingSearch.test.ts
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Unit tests for transcript-aware recording search (recordingSearch.ts).
- */
-import {
- countOccurrences,
- extractSnippet,
- matchRecording,
- searchRecordings,
- highlightSegments,
- findSegmentSeekMs,
-} from '../../../pro/locket/utils/recordingSearch';
-import type { Recording } from '../../../pro/locket/stores/recordingsStore';
-
-function rec(partial: Partial): Recording {
- return {
- id: partial.id ?? 'r1',
- path: partial.path ?? '/x/rec-1.wav',
- startedAt: partial.startedAt ?? 1_000,
- endedAt: partial.endedAt ?? 2_000,
- durationMs: partial.durationMs ?? 1_000,
- sizeBytes: partial.sizeBytes ?? 1_000,
- ...partial,
- } as Recording;
-}
-
-describe('countOccurrences', () => {
- it('counts non-overlapping hits', () => {
- expect(countOccurrences('the budget and the budget again', 'budget')).toBe(2);
- });
- it('returns 0 for no hit and for empty needle', () => {
- expect(countOccurrences('nothing here', 'budget')).toBe(0);
- expect(countOccurrences('anything', '')).toBe(0);
- });
-});
-
-describe('extractSnippet', () => {
- it('adds ellipses when trimmed on both sides', () => {
- const text = `${'a'.repeat(60)}budget${'b'.repeat(60)}`;
- const idx = text.toLowerCase().indexOf('budget');
- const s = extractSnippet(text, idx, 'budget'.length);
- expect(s.startsWith('...')).toBe(true);
- expect(s.endsWith('...')).toBe(true);
- expect(s).toContain('budget');
- });
- it('omits leading ellipsis when the hit is near the start', () => {
- const s = extractSnippet('budget talk here', 0, 'budget'.length);
- expect(s.startsWith('...')).toBe(false);
- expect(s).toContain('budget');
- });
-});
-
-describe('matchRecording', () => {
- it('matches transcript and returns a snippet + hit count, ranked as transcript', () => {
- const r = rec({ transcript: 'we should finalize the budget before the budget review' });
- const m = matchRecording(r, 'budget');
- expect(m?.field).toBe('transcript');
- expect(m?.transcriptHits).toBe(2);
- expect(m?.snippet).toContain('budget');
- });
- it('matches title with no snippet when transcript does not hit', () => {
- const r = rec({ name: 'Budget meeting', transcript: 'unrelated words' });
- const m = matchRecording(r, 'budget');
- expect(m?.field).toBe('title');
- expect(m?.snippet).toBeNull();
- expect(m?.transcriptHits).toBe(0);
- });
- it('matches people when neither transcript nor title hit', () => {
- const r = rec({
- transcript: 'nope',
- attendees: [{ name: 'Sam Rivera', email: 'sam@x.com' }],
- });
- const m = matchRecording(r, 'rivera');
- expect(m?.field).toBe('people');
- });
- it('returns null when nothing matches', () => {
- expect(matchRecording(rec({ transcript: 'hello' }), 'budget')).toBeNull();
- });
-});
-
-describe('searchRecordings', () => {
- it('ranks transcript matches before title/people matches', () => {
- const titleHit = rec({ id: 'title', name: 'Budget sync', transcript: 'x' });
- const transcriptHit = rec({ id: 'tr', transcript: 'the budget is set' });
- const results = searchRecordings([titleHit, transcriptHit], 'budget');
- expect(results.map((m) => m.recording.id)).toEqual(['tr', 'title']);
- });
- it('is case-insensitive and returns [] for empty query', () => {
- const r = rec({ transcript: 'The BUDGET' });
- expect(searchRecordings([r], 'budget')).toHaveLength(1);
- expect(searchRecordings([r], ' ')).toHaveLength(0);
- });
-});
-
-describe('findSegmentSeekMs', () => {
- const segs = [
- { text: 'welcome everyone', startMs: 0, endMs: 2000 },
- { text: 'lets discuss the budget', startMs: 2000, endMs: 5000 },
- { text: 'and the budget again', startMs: 5000, endMs: 8000 },
- ];
- it('returns the startMs of the first segment containing the query', () => {
- expect(findSegmentSeekMs(segs, 'budget')).toBe(2000);
- });
- it('is case-insensitive', () => {
- expect(findSegmentSeekMs(segs, 'BUDGET')).toBe(2000);
- });
- it('returns null when no segment matches or no segments given', () => {
- expect(findSegmentSeekMs(segs, 'nothere')).toBeNull();
- expect(findSegmentSeekMs(undefined, 'budget')).toBeNull();
- expect(findSegmentSeekMs([], 'budget')).toBeNull();
- });
-});
-
-describe('matchRecording - seekMs', () => {
- it('sets seekMs from the matching segment on a transcript hit', () => {
- const r = rec({
- transcript: 'lets discuss the budget now',
- transcriptSegments: [
- { text: 'lets discuss the budget now', startMs: 4200, endMs: 9000 },
- ],
- });
- expect(matchRecording(r, 'budget')?.seekMs).toBe(4200);
- });
- it('leaves seekMs null when the transcript has no timed segments', () => {
- const r = rec({ transcript: 'the budget is set' });
- const m = matchRecording(r, 'budget');
- expect(m?.field).toBe('transcript');
- expect(m?.seekMs).toBeNull();
- });
-});
-
-describe('highlightSegments', () => {
- it('splits into match/non-match runs preserving original casing', () => {
- const segs = highlightSegments('The Budget is the Budget', 'budget');
- const matched = segs.filter((s) => s.match).map((s) => s.text);
- expect(matched).toEqual(['Budget', 'Budget']);
- // Reassembling yields the original text.
- expect(segs.map((s) => s.text).join('')).toBe('The Budget is the Budget');
- });
- it('returns the whole string as a single non-match run for empty query', () => {
- expect(highlightSegments('hello', '')).toEqual([{ text: 'hello', match: false }]);
- });
-});
diff --git a/__tests__/unit/locket/recordingSplit.test.ts b/__tests__/unit/locket/recordingSplit.test.ts
deleted file mode 100644
index 1668bca95..000000000
--- a/__tests__/unit/locket/recordingSplit.test.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Unit tests for split planning (recordingSplit.ts) - pure logic over a VAD map.
- */
-import { planSplits, countSplits, SPLIT_GAP_DEFAULT_MS } from '../../../pro/locket/services/recordingSplit';
-import type { VadResult } from '../../../pro/locket/services/vadDetect';
-
-// Build a VadResult from speech segments; gaps are the inverse within totalMs.
-function vad(totalMs: number, speech: [number, number][]): VadResult {
- const seg = speech.map(([s, e]) => ({ startMs: s, endMs: e }));
- const gaps: { startMs: number; endMs: number }[] = [];
- let cur = 0;
- for (const s of seg) {
- if (s.startMs > cur) gaps.push({ startMs: cur, endMs: s.startMs });
- cur = Math.max(cur, s.endMs);
- }
- if (cur < totalMs) gaps.push({ startMs: cur, endMs: totalMs });
- const speechMs = seg.reduce((a, s) => a + (s.endMs - s.startMs), 0);
- return { speech: seg, gaps, totalMs, speechMs, speechPct: Math.round((speechMs / totalMs) * 100), wallMs: 0 };
-}
-
-describe('planSplits', () => {
- it('does not split when no gap exceeds the threshold', () => {
- // speech with only short (5s) gaps, threshold 30s -> one piece
- const v = vad(120_000, [[0, 40_000], [45_000, 80_000], [85_000, 120_000]]);
- const pieces = planSplits(v, SPLIT_GAP_DEFAULT_MS);
- expect(pieces.length).toBe(1);
- expect(pieces[0]).toMatchObject({ startMs: 0, endMs: 120_000 });
- });
-
- it('splits at a long gap (midpoint) and folds the pieces', () => {
- // 40s speech, 60s gap (>30s), 40s speech -> 2 pieces, cut at gap midpoint
- const v = vad(140_000, [[0, 40_000], [100_000, 140_000]]);
- const pieces = planSplits(v, 30_000);
- expect(pieces.length).toBe(2);
- // divider = midpoint of 40k-100k gap = 70k
- expect(pieces[0]).toMatchObject({ startMs: 0, endMs: 70_000 });
- expect(pieces[1]).toMatchObject({ startMs: 70_000, endMs: 140_000 });
- });
-
- it('a higher threshold yields fewer pieces', () => {
- // speech runs to the very end so there's no trailing gap; two internal gaps.
- const v = vad(180_000, [[0, 30_000], [60_000, 90_000], [150_000, 180_000]]);
- // internal gaps: 30k-60k (30s), 90k-150k (60s)
- expect(countSplits(v, 20_000)).toBe(3); // both gaps split -> 3 pieces
- expect(countSplits(v, 45_000)).toBe(2); // only the 60s gap splits -> 2 pieces
- expect(countSplits(v, 90_000)).toBe(1); // neither splits -> 1 piece
- });
-
- it('reports speech duration within each piece', () => {
- const v = vad(140_000, [[0, 40_000], [100_000, 140_000]]);
- const pieces = planSplits(v, 30_000);
- expect(pieces[0].speechMs).toBe(40_000);
- expect(pieces[1].speechMs).toBe(40_000);
- });
-
- it('folds a too-short trailing piece into the previous one', () => {
- // a long gap right near the end would make a 1s final piece -> folded
- const v = vad(100_000, [[0, 40_000], [98_000, 99_000]]);
- const pieces = planSplits(v, 30_000, 3_000);
- // the tiny tail piece is absorbed, so we still get sensible pieces
- expect(pieces.every((p) => p.endMs - p.startMs >= 3_000)).toBe(true);
- });
-
- it('returns [] for an empty/zero-length recording', () => {
- expect(planSplits(vad(0, []), 30_000)).toEqual([]);
- });
-});
diff --git a/__tests__/unit/locket/recordingsRecovery.test.ts b/__tests__/unit/locket/recordingsRecovery.test.ts
deleted file mode 100644
index ff7dce30a..000000000
--- a/__tests__/unit/locket/recordingsRecovery.test.ts
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Unit tests for locket orphan recovery (recordingsRecovery.ts).
- *
- * Covers the by-directory (not by-filename) scoping, the conditional grace
- * window (only while the recorder is running), and the epoch/mtime startedAt
- * fallback. RNFS and the recordings store are mocked so the test exercises
- * only the recovery decision logic.
- */
-
-const SAMPLE_RATE = 16000;
-const BYTES_PER_SECOND = SAMPLE_RATE * 1 * 2; // 16k mono 16-bit
-const HEADER = 44;
-// A comfortably-recoverable size: header + 5s of PCM.
-const OK_SIZE = HEADER + BYTES_PER_SECOND * 5;
-
-// ---- Mocks -----------------------------------------------------------------
-
-jest.mock('react-native-fs', () => ({
- ExternalDirectoryPath: '/ext',
- DocumentDirectoryPath: '/docs',
- exists: jest.fn(),
- readDir: jest.fn(),
- stat: jest.fn(),
- read: jest.fn(),
-}));
-
-// `mock`-prefixed so babel allows referencing them inside the hoisted factory.
-const mockAddRecoveredBatch = jest.fn((recs: unknown[]) => (recs as unknown[]).length);
-const mockStore: {
- currentFilePath: string | null;
- isRunning: boolean;
- recordings: { path: string; startedAt?: number; sizeBytes?: number }[];
-} = {
- currentFilePath: null,
- isRunning: false,
- recordings: [],
-};
-
-jest.mock('../../../pro/locket/stores/recordingsStore', () => ({
- useRecordingsStore: {
- getState: () => ({
- currentFilePath: mockStore.currentFilePath,
- isRunning: mockStore.isRunning,
- recordings: mockStore.recordings,
- addRecoveredBatch: mockAddRecoveredBatch,
- }),
- },
-}));
-
-jest.mock('@offgrid/core/utils/logger', () => ({
- __esModule: true,
- default: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
-}));
-
-import RNFS from 'react-native-fs';
-import {
- recoverOrphans,
- _resetRecoveryGuardForTesting,
-} from '../../../pro/locket/services/recordingsRecovery';
-
-const mockRNFS = RNFS as unknown as {
- exists: jest.Mock;
- readDir: jest.Mock;
- stat: jest.Mock;
- read: jest.Mock;
-};
-
-// Build a healthy WAV header (declared data size == fileSize - 44) in base64,
-// so readDeclaredDataSize sees a healthy header unless we say otherwise.
-function wavHeaderB64(dataSize: number): string {
- const b = Buffer.alloc(44);
- b.write('RIFF', 0, 'ascii');
- b.writeUInt32LE(36 + dataSize, 4);
- b.write('WAVE', 8, 'ascii');
- b.write('data', 36, 'ascii');
- b.writeUInt32LE(dataSize, 40);
- return b.toString('base64');
-}
-
-function entry(name: string) {
- return { name, path: `/ext/Music/Recordings/${name}`, isFile: () => true };
-}
-
-beforeEach(() => {
- _resetRecoveryGuardForTesting();
- mockStore.currentFilePath = null;
- mockStore.isRunning = false;
- mockStore.recordings = [];
- mockAddRecoveredBatch.mockClear();
- mockRNFS.exists.mockResolvedValue(true);
- // Default: healthy header, recent-ish mtime, OK size.
- mockRNFS.stat.mockResolvedValue({ size: OK_SIZE, mtime: 1_000_000 });
- mockRNFS.read.mockResolvedValue(wavHeaderB64(OK_SIZE - HEADER));
-});
-
-describe('recoverOrphans - by-directory scoping (Gap 4 fix)', () => {
- it('recovers a .wav that does NOT match the rec- name', async () => {
- mockRNFS.readDir.mockResolvedValue([entry('imported-thing.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(1);
- expect(mockAddRecoveredBatch).toHaveBeenCalledTimes(1);
- });
-
- it('still recovers a rec-.wav file', async () => {
- mockRNFS.readDir.mockResolvedValue([entry('rec-1720000000000.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(1);
- });
-
- it('ignores non-audio files in the directory', async () => {
- mockRNFS.readDir.mockResolvedValue([entry('notes.txt'), entry('cover.jpg')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.skippedBadName).toBe(2);
- });
-
- it('never recovers a stray backup-*.m4a as a recording (backups live in Backups/)', async () => {
- // Backups normally live in a sibling Backups/ dir, but a stray/old-layout
- // one in Recordings/ must never be surfaced as a recording (it's a restore
- // copy, not a recording).
- mockRNFS.stat.mockResolvedValue({ size: 300_000, mtime: 1_000_000 });
- mockRNFS.readDir.mockResolvedValue([entry('backup-rec-1720000000000.m4a')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.skippedBadName).toBe(1);
- });
-
- it('recovers a compressed .m4a recording (bug #3: was dropped after store wipe)', async () => {
- // A recording the user compressed becomes rec-.m4a with the .wav
- // deleted. Recovery must find it or it vanishes from the archive on a wipe.
- mockRNFS.stat.mockResolvedValue({ size: 300_000, mtime: 1_000_000 });
- mockRNFS.readDir.mockResolvedValue([entry('rec-1720000000000.m4a')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(1);
- // No WAV header on an .m4a, so it is never flagged "header damaged".
- expect(report.staleHeaderDetected).toBe(0);
- const queued = mockAddRecoveredBatch.mock.calls[0][0] as { name: string; durationMs: number }[];
- expect(queued[0].name).toBe('Recovered');
- // Duration is ESTIMATED from the AAC bitrate (24 kbps mono = 3000 B/s), not
- // the WAV PCM-size math: 300000 / 3000 * 1000 = 100000 ms.
- expect(queued[0].durationMs).toBe(100_000);
- });
-});
-
-describe('recoverOrphans - conditional grace window (Gap 2 fix)', () => {
- it('recovers a freshly-modified file when the recorder is NOT running', async () => {
- mockStore.isRunning = false;
- // mtime = "now" so it is within any grace window.
- const now = 5_000_000;
- jest.spyOn(Date, 'now').mockReturnValue(now);
- mockRNFS.stat.mockResolvedValue({ size: OK_SIZE, mtime: now });
- mockRNFS.readDir.mockResolvedValue([entry('rec-1.wav')]);
-
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(1);
- expect(report.skippedTooNew).toBe(0);
- (Date.now as jest.Mock).mockRestore?.();
- });
-
- it('skips a freshly-modified file WHILE the recorder is running', async () => {
- mockStore.isRunning = true;
- const now = 5_000_000;
- jest.spyOn(Date, 'now').mockReturnValue(now);
- mockRNFS.stat.mockResolvedValue({ size: OK_SIZE, mtime: now });
- mockRNFS.readDir.mockResolvedValue([entry('rec-1.wav')]);
-
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.skippedTooNew).toBe(1);
- (Date.now as jest.Mock).mockRestore?.();
- });
-});
-
-describe('recoverOrphans - safety guards preserved', () => {
- it('never recovers the currently-recording file', async () => {
- mockStore.currentFilePath = '/ext/Music/Recordings/rec-active.wav';
- mockRNFS.readDir.mockResolvedValue([entry('rec-active.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.skippedActive).toBe(1);
- });
-
- it('skips files below the minimum recoverable size', async () => {
- mockRNFS.stat.mockResolvedValue({ size: HEADER + 10, mtime: 1_000_000 });
- mockRNFS.readDir.mockResolvedValue([entry('rec-tiny.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.skippedTooSmall).toBe(1);
- });
-
- it('does not re-add a recording already in the store', async () => {
- mockStore.recordings = [{ path: '/ext/Music/Recordings/rec-known.wav' }];
- mockRNFS.readDir.mockResolvedValue([entry('rec-known.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.alreadyInStore).toBe(1);
- });
-
- it('dedups by content (same size + close startedAt) despite a different path', async () => {
- // Simulates iOS container rotation: the file on disk has a NEW path, but the
- // store holds the same recording under an OLD path. Basename also differs.
- // startedAt within 5s + identical size => same recording, must not duplicate.
- mockStore.recordings = [
- { path: '/OLD-UUID/Music/Recordings/rec-1720000000000.wav', startedAt: 1720000000000, sizeBytes: OK_SIZE },
- ] as unknown as typeof mockStore.recordings;
- // On-disk file: different basename/path, epoch 2s later, same size.
- mockRNFS.readDir.mockResolvedValue([entry('rec-1720000002000.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.alreadyInStore).toBe(1);
- });
-
- it('dedups a .wav orphan against a compressed .m4a store entry by epoch (killed mid-compression)', async () => {
- // Compression converts rec-.wav -> rec-.m4a and deletes the wav.
- // If the app is killed between the store swap and the wav delete, the store
- // points at the .m4a (small) while the orphan .wav (big) is still on disk.
- // Same epoch => same recording; recovery must skip it, not add a duplicate -
- // even though extension AND size differ (so content dedup can't catch it).
- mockStore.recordings = [
- { path: '/ext/Music/Recordings/rec-1720000000000.m4a', startedAt: 1720000000000, sizeBytes: 20_000 },
- ] as unknown as typeof mockStore.recordings;
- mockRNFS.stat.mockResolvedValue({ size: OK_SIZE, mtime: 1_000_000 }); // big raw wav
- mockRNFS.readDir.mockResolvedValue([entry('rec-1720000000000.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(0);
- expect(report.alreadyInStore).toBe(1);
- });
-
- it('labels a zeroed-header file as damaged', async () => {
- // Header declares 0 data bytes but the file has real PCM -> damaged.
- mockRNFS.read.mockResolvedValue(wavHeaderB64(0));
- mockRNFS.readDir.mockResolvedValue([entry('rec-damaged.wav')]);
- const report = await recoverOrphans({ force: true });
- expect(report.added).toBe(1);
- expect(report.staleHeaderDetected).toBe(1);
- const queued = mockAddRecoveredBatch.mock.calls[0][0] as { name: string }[];
- expect(queued[0].name).toBe('Recovered (header damaged)');
- });
-});
diff --git a/__tests__/unit/locket/speechCleanup.test.ts b/__tests__/unit/locket/speechCleanup.test.ts
deleted file mode 100644
index a97d53c16..000000000
--- a/__tests__/unit/locket/speechCleanup.test.ts
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Unit tests for the pure range math in speechCleanup - the safety-critical
- * logic (delete/keep composition + transcript remap after a cut). No I/O.
- */
-import {
- mergeRanges,
- subtractRanges,
- computeKeptRanges,
- remapSegments,
- remapSegmentsToFull,
- mergeWithinGap,
- type Range,
-} from '../../../pro/locket/services/speechCleanup';
-
-const r = (startMs: number, endMs: number): Range => ({ startMs, endMs });
-const seg = (startMs: number, endMs: number, text = 'x') => ({ text, startMs, endMs });
-
-describe('mergeRanges', () => {
- it('merges overlapping and adjacent, sorts', () => {
- expect(mergeRanges([r(10, 20), r(5, 12), r(30, 40)])).toEqual([r(5, 20), r(30, 40)]);
- expect(mergeRanges([r(20, 30), r(30, 40)])).toEqual([r(20, 40)]); // touching merges
- });
- it('returns [] for empty', () => {
- expect(mergeRanges([])).toEqual([]);
- });
-});
-
-describe('subtractRanges / computeKeptRanges', () => {
- it('removes a middle chunk, splitting the range', () => {
- expect(subtractRanges([r(0, 100)], [r(40, 60)])).toEqual([r(0, 40), r(60, 100)]);
- });
- it('trims edges', () => {
- expect(subtractRanges([r(0, 100)], [r(0, 20), r(90, 100)])).toEqual([r(20, 90)]);
- });
- it('fully removed range disappears', () => {
- expect(subtractRanges([r(10, 20)], [r(0, 100)])).toEqual([]);
- });
- it('no overlap leaves base intact (merged)', () => {
- expect(subtractRanges([r(0, 10), r(20, 30)], [r(12, 15)])).toEqual([r(0, 10), r(20, 30)]);
- });
- it('computeKeptRanges = speech minus deleted', () => {
- expect(computeKeptRanges([r(0, 40), r(60, 100)], [r(70, 80)]))
- .toEqual([r(0, 40), r(60, 70), r(80, 100)]);
- });
- it('empty deleted returns speech merged', () => {
- expect(computeKeptRanges([r(0, 40)], undefined)).toEqual([r(0, 40)]);
- });
-});
-
-describe('remapSegments', () => {
- // kept = [0-40s] + [60-100s] (a 20s gap removed). New timeline: 0-40 keeps,
- // then 60-100 shifts down by 20s -> 40-80.
- const kept = [r(0, 40_000), r(60_000, 100_000)];
-
- it('shifts a segment in the second kept range down by the removed length', () => {
- const out = remapSegments([{ text: 'a', startMs: 70_000, endMs: 90_000 }], kept);
- expect(out).toEqual([{ text: 'a', startMs: 50_000, endMs: 70_000 }]);
- });
- it('keeps a segment in the first kept range unchanged', () => {
- const out = remapSegments([{ text: 'b', startMs: 10_000, endMs: 20_000 }], kept);
- expect(out).toEqual([{ text: 'b', startMs: 10_000, endMs: 20_000 }]);
- });
- it('drops a segment entirely inside a removed gap', () => {
- expect(remapSegments([{ text: 'gap', startMs: 45_000, endMs: 55_000 }], kept)).toEqual([]);
- });
- it('splits a segment straddling a removed boundary into per-kept-range pieces', () => {
- // 30s-70s straddles the removed 40-60 gap: kept parts 30-40 (->30-40) and
- // 60-70 (->40-50).
- const out = remapSegments([{ text: 's', startMs: 30_000, endMs: 70_000 }], kept);
- expect(out).toEqual([
- { text: 's', startMs: 30_000, endMs: 40_000 },
- { text: 's', startMs: 40_000, endMs: 50_000 },
- ]);
- });
- it('never produces negative or overlapping-with-self lengths', () => {
- const out = remapSegments([{ text: 'x', startMs: 0, endMs: 100_000 }], kept);
- expect(out.every((s) => s.endMs > s.startMs)).toBe(true);
- });
- it('empty in -> empty out', () => {
- expect(remapSegments(undefined, kept)).toEqual([]);
- });
-});
-
-describe('remapSegmentsToFull (inverse - restore)', () => {
- // Same scenario: kept = [0-40s] + [60-100s], 40-60s removed. Compacted
- // timeline is 0-80s; the second range occupies compacted 40-80s.
- const kept = [r(0, 40_000), r(60_000, 100_000)];
-
- it('maps a compacted second-range segment back up by the removed length', () => {
- // compacted 50-70s -> full 70-90s (add back the 20s gap).
- expect(remapSegmentsToFull([seg(50_000, 70_000, 'a')], kept)).toEqual([seg(70_000, 90_000, 'a')]);
- });
-
- it('leaves a first-range segment unchanged', () => {
- expect(remapSegmentsToFull([seg(10_000, 20_000, 'b')], kept)).toEqual([seg(10_000, 20_000, 'b')]);
- });
-
- it('round-trips: forward then inverse restores a segment that did not straddle a boundary', () => {
- const original = [seg(10_000, 20_000, 'p'), seg(70_000, 90_000, 'q')];
- const compacted = remapSegments(original, kept);
- const restored = remapSegmentsToFull(compacted, kept);
- expect(restored).toEqual(original);
- });
-
- it('clamps a time past the end of kept audio into the last range', () => {
- // compacted 80s = end; maps to full 100s (end of the second kept range).
- expect(remapSegmentsToFull([seg(79_000, 80_000, 'z')], kept)).toEqual([seg(99_000, 100_000, 'z')]);
- });
-
- it('empty / no kept ranges -> empty out', () => {
- expect(remapSegmentsToFull(undefined, kept)).toEqual([]);
- expect(remapSegmentsToFull([seg(0, 1000)], [])).toEqual([]);
- });
-});
-
-describe('mergeWithinGap (auto-prune: keep short pauses, drop long dead-air)', () => {
- const GAP = 60_000; // 60s threshold
-
- it('keeps a short (< 60s) gap inside one range', () => {
- // speech 0-10s, 30s pause, speech 40-50s -> one range 0-50s (pause kept).
- expect(mergeWithinGap([r(0, 10_000), r(40_000, 50_000)], GAP)).toEqual([r(0, 50_000)]);
- });
-
- it('splits on a long (> 60s) gap so it gets dropped', () => {
- // speech 0-10s, 90s gap, speech 100-110s -> two ranges; the 90s gap is
- // between them and will be removed by the compaction.
- expect(mergeWithinGap([r(0, 10_000), r(100_000, 110_000)], GAP)).toEqual([
- r(0, 10_000),
- r(100_000, 110_000),
- ]);
- });
-
- it('a gap exactly at the threshold is kept (<=)', () => {
- expect(mergeWithinGap([r(0, 10_000), r(70_000, 80_000)], GAP)).toEqual([r(0, 80_000)]);
- });
-
- it('empty in -> empty out', () => {
- expect(mergeWithinGap([], GAP)).toEqual([]);
- });
-});
diff --git a/__tests__/unit/locket/transcribeWindows.test.ts b/__tests__/unit/locket/transcribeWindows.test.ts
deleted file mode 100644
index dabace865..000000000
--- a/__tests__/unit/locket/transcribeWindows.test.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Unit tests for buildWorkWindows - the pure logic behind VAD-gated
- * transcription (which windows get transcribed). No I/O.
- */
-import { buildWorkWindows } from '../../../pro/locket/services/transcribeWindows';
-
-describe('buildWorkWindows', () => {
- const MAX = 60_000; // 1 min windows for readable cases
-
- it('returns [] for a zero/negative duration', () => {
- expect(buildWorkWindows(0, MAX)).toEqual([]);
- expect(buildWorkWindows(-1, MAX)).toEqual([]);
- });
-
- it('with no speech ranges, tiles the whole file into max-sized windows', () => {
- expect(buildWorkWindows(120_000, MAX)).toEqual([
- { startMs: 0, endMs: 60_000 },
- { startMs: 60_000, endMs: 120_000 },
- ]);
- });
-
- it('with no speech ranges, a short file is one window', () => {
- expect(buildWorkWindows(40_000, MAX)).toEqual([{ startMs: 0, endMs: 40_000 }]);
- });
-
- it('an empty speech-range array falls back to the full file', () => {
- expect(buildWorkWindows(50_000, MAX, { speechRanges: [] })).toEqual([{ startMs: 0, endMs: 50_000 }]);
- });
-
- it('with speech ranges, transcribes only the speech and skips the silence between', () => {
- const windows = buildWorkWindows(120_000, MAX, {
- speechRanges: [{ startMs: 0, endMs: 40_000 }, { startMs: 100_000, endMs: 120_000 }],
- });
- expect(windows).toEqual([
- { startMs: 0, endMs: 40_000 },
- { startMs: 100_000, endMs: 120_000 },
- ]);
- // The silence 40s-100s is never emitted.
- expect(windows.some((w) => w.startMs >= 40_000 && w.endMs <= 100_000)).toBe(false);
- });
-
- it('pads each range and merges ranges that overlap after padding', () => {
- // Two ranges 500ms apart; a 1s pad makes them overlap -> one window.
- const windows = buildWorkWindows(60_000, MAX, {
- speechRanges: [{ startMs: 10_000, endMs: 20_000 }, { startMs: 20_500, endMs: 30_000 }],
- padMs: 1_000,
- });
- expect(windows).toEqual([{ startMs: 9_000, endMs: 31_000 }]);
- });
-
- it('splits a long merged speech range into max-sized sub-windows', () => {
- const windows = buildWorkWindows(200_000, MAX, { speechRanges: [{ startMs: 0, endMs: 200_000 }] });
- expect(windows).toEqual([
- { startMs: 0, endMs: 60_000 },
- { startMs: 60_000, endMs: 120_000 },
- { startMs: 120_000, endMs: 180_000 },
- { startMs: 180_000, endMs: 200_000 },
- ]);
- });
-
- it('clamps padded ranges to [0, durationMs]', () => {
- const windows = buildWorkWindows(100_000, MAX, {
- speechRanges: [{ startMs: 2_000, endMs: 98_000 }],
- padMs: 5_000, // pad would push to -3000 / 103000
- });
- expect(windows[0].startMs).toBe(0);
- expect(windows[windows.length - 1].endMs).toBe(100_000);
- });
-});
diff --git a/__tests__/unit/services/selectTextModel.test.ts b/__tests__/unit/services/selectTextModel.test.ts
new file mode 100644
index 000000000..190183f19
--- /dev/null
+++ b/__tests__/unit/services/selectTextModel.test.ts
@@ -0,0 +1,63 @@
+import { selectTextModelToLoad, fitsBudget } from '../../../src/services/selectTextModel';
+import type { DownloadedModel } from '../../../src/types';
+
+const MB = 1024 * 1024;
+
+function model(id: string, fileSizeMB: number): DownloadedModel {
+ return {
+ id,
+ name: id,
+ author: 'test',
+ filePath: `/models/${id}`,
+ fileName: `${id}.gguf`,
+ fileSize: fileSizeMB * MB,
+ quantization: 'Q4',
+ downloadedAt: '2026-07-13',
+ engine: 'llama',
+ };
+}
+
+// Footprint = fileSize in MB (1x) — the selection logic is independent of the
+// multiplier; the real caller passes hardwareService.estimateModelRam.
+const footprint = (m: DownloadedModel) => (m.fileSize || 0) / MB;
+
+const small = model('small', 500);
+const medium = model('medium', 1000);
+const large = model('large', 3000);
+
+describe('fitsBudget', () => {
+ it('fits when footprint <= budget, not otherwise', () => {
+ expect(fitsBudget(1000, 1000)).toBe(true); // exactly fits
+ expect(fitsBudget(1001, 1000)).toBe(false);
+ });
+});
+
+describe('selectTextModelToLoad', () => {
+ it('returns null when nothing is downloaded', () => {
+ expect(selectTextModelToLoad([], 4000, { activeId: null, footprintMB: footprint })).toBeNull();
+ expect(selectTextModelToLoad([], 4000, { activeId: 'medium', footprintMB: footprint })).toBeNull();
+ });
+
+ it('uses the active model when it fits the budget', () => {
+ expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: 'small', footprintMB: footprint })?.id).toBe('small');
+ });
+
+ it('ignores the active model when it does NOT fit, and picks the largest that fits', () => {
+ // budget 2000: large(3000) does not fit -> largest fitting is medium(1000)
+ expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: 'large', footprintMB: footprint })?.id).toBe('medium');
+ });
+
+ it('with no active id, picks the largest model that fits (best quality within RAM)', () => {
+ expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: null, footprintMB: footprint })?.id).toBe('medium');
+ expect(selectTextModelToLoad([small, medium, large], 4000, { activeId: null, footprintMB: footprint })?.id).toBe('large');
+ });
+
+ it('falls back to the SMALLEST when nothing fits (run something, not an OOM)', () => {
+ // budget 400: smallest is small(500) > 400, nothing fits -> smallest
+ expect(selectTextModelToLoad([small, medium, large], 400, { activeId: 'large', footprintMB: footprint })?.id).toBe('small');
+ });
+
+ it('ignores an active id that is not among the downloaded models', () => {
+ expect(selectTextModelToLoad([small, medium], 2000, { activeId: 'ghost', footprintMB: footprint })?.id).toBe('medium');
+ });
+});
diff --git a/__tests__/unit/services/whisperService.test.ts b/__tests__/unit/services/whisperService.test.ts
index 25ce7b7f2..abdc6350f 100644
--- a/__tests__/unit/services/whisperService.test.ts
+++ b/__tests__/unit/services/whisperService.test.ts
@@ -315,7 +315,9 @@ describe('WhisperService', () => {
filePath: '/path/to/model.bin',
useGpu: false,
useFlashAttn: false,
- useCoreMLIos: false,
+ // The test's RNFS.exists mock reports the CoreML encoder present, so
+ // loadModel auto-enables ANE CoreML on iOS.
+ useCoreMLIos: true,
});
expect(whisperService.isModelLoaded()).toBe(true);
expect(whisperService.getLoadedModelPath()).toBe('/path/to/model.bin');
diff --git a/__tests__/unit/stores/whisperStore.test.ts b/__tests__/unit/stores/whisperStore.test.ts
index eb33dff83..66b74d6ae 100644
--- a/__tests__/unit/stores/whisperStore.test.ts
+++ b/__tests__/unit/stores/whisperStore.test.ts
@@ -15,6 +15,7 @@ jest.mock('../../../src/services', () => ({
unloadModel: jest.fn(),
deleteModel: jest.fn(),
isModelDownloaded: jest.fn(),
+ listDownloadedModels: jest.fn(),
},
WHISPER_MODELS: [{ id: 'tiny', size: 75 }, { id: 'base', size: 142 }],
}));
@@ -39,10 +40,16 @@ const mockResidency = modelResidencyManager as jest.Mocked useWhisperStore.getState();
+// Build the on-disk model list shape whisperService.listDownloadedModels returns.
+const onDisk = (...ids: string[]) =>
+ ids.map((modelId) => ({ modelId, fileName: `ggml-${modelId}.bin`, sizeBytes: 1, filePath: `/models/ggml-${modelId}.bin` }));
+
describe('whisperStore', () => {
beforeEach(() => {
resetWhisperStore();
jest.clearAllMocks();
+ // Default: nothing else on disk (delete flows fall back to no model).
+ mockWhisperService.listDownloadedModels.mockResolvedValue(onDisk());
});
// ============================================================================
@@ -384,7 +391,10 @@ describe('whisperStore', () => {
const result = await getState().loadModel();
- expect(mockWhisperService.loadModel).toHaveBeenCalledWith('/models/ggml-tiny');
+ // loadModel() forwards its (here undefined) options arg to the service, so
+ // the recorded call is (path, undefined) - assert both, since jest's
+ // toHaveBeenCalledWith does not match a trailing undefined against one arg.
+ expect(mockWhisperService.loadModel).toHaveBeenCalledWith('/models/ggml-tiny', undefined);
expect(mockResidency.register).toHaveBeenCalled();
expect(getState().isModelLoaded).toBe(true);
expect(result).toBe('loaded');
@@ -564,17 +574,19 @@ describe('whisperStore', () => {
expect(mockWhisperService.downloadModel).not.toHaveBeenCalled();
});
- it('deleteModelById removes the file and clears active when it was active', async () => {
+ it('deleteModelById falls back to another on-disk model when the active one is deleted', async () => {
useWhisperStore.setState({ presentModelIds: ['tiny', 'base'], downloadedModelId: 'base', isModelLoaded: true });
+ mockWhisperService.listDownloadedModels.mockResolvedValue(onDisk('tiny'));
await getState().deleteModelById('base');
expect(mockWhisperService.deleteModel).toHaveBeenCalledWith('base');
expect(getState().presentModelIds).toEqual(['tiny']);
- expect(getState().downloadedModelId).toBeNull();
+ expect(getState().downloadedModelId).toBe('tiny');
expect(getState().isModelLoaded).toBe(false);
});
it('deleteModelById keeps the active model when deleting a different one', async () => {
useWhisperStore.setState({ presentModelIds: ['tiny', 'base'], downloadedModelId: 'base' });
+ mockWhisperService.listDownloadedModels.mockResolvedValue(onDisk('base'));
await getState().deleteModelById('tiny');
expect(getState().presentModelIds).toEqual(['base']);
expect(getState().downloadedModelId).toBe('base');
diff --git a/android/app/src/main/assets/whisper-vad/ggml-silero-v5.1.2.bin b/android/app/src/main/assets/whisper-vad/ggml-silero-v5.1.2.bin
new file mode 100644
index 000000000..c5ddfb537
Binary files /dev/null and b/android/app/src/main/assets/whisper-vad/ggml-silero-v5.1.2.bin differ
diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt
index 64d5ebf85..66a00e017 100644
--- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt
+++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt
@@ -13,6 +13,7 @@ import com.google.ai.edge.litertlm.BenchmarkInfo
import com.google.ai.edge.litertlm.ConversationConfig
import com.google.ai.edge.litertlm.Engine
import com.google.ai.edge.litertlm.EngineConfig
+import com.google.ai.edge.litertlm.ExperimentalFlags
import com.google.ai.edge.litertlm.Content
import com.google.ai.edge.litertlm.Contents
import com.google.ai.edge.litertlm.ExperimentalApi
@@ -107,8 +108,30 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
private val pendingToolCalls = ConcurrentHashMap>()
private var configuredMaxTokens: Int = 4096
+ // DEV-only constrained decoding (LLGuidance: json_schema / lark / regex).
+ // Set from JS via setConstrainedDecoding() before resetConversation. The map
+ // contract below is UNVERIFIED - every use is wrapped so a wrong shape logs
+ // and falls back to unconstrained generation, never crashing the chat path.
+ @Volatile private var constrainedEnabled = false
+ @Volatile private var constraintType = ""
+ @Volatile private var constraintString = ""
+
override fun getName(): String = "LiteRTModule"
+ // -------------------------------------------------------------------------
+ // setConstrainedDecoding (DEV) — arm/disarm an LLGuidance constraint that
+ // resetConversation + sendMessage will apply. type = json_schema|lark|regex.
+ // -------------------------------------------------------------------------
+
+ @ReactMethod
+ fun setConstrainedDecoding(enabled: Boolean, type: String, constraint: String, promise: Promise) {
+ constrainedEnabled = enabled && constraint.isNotEmpty()
+ constraintType = type
+ constraintString = constraint
+ Log.i(TAG, "[DevGrammar-LiteRT] setConstrainedDecoding enabled=$constrainedEnabled type=$type len=${constraint.length}")
+ promise.resolve(null)
+ }
+
// -------------------------------------------------------------------------
// loadModel
// -------------------------------------------------------------------------
@@ -238,6 +261,7 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
// resetConversation — closes and recreates Conversation only, Engine stays
// -------------------------------------------------------------------------
+ @OptIn(ExperimentalApi::class)
@ReactMethod
fun resetConversation(systemPrompt: String, temperature: Double, topK: Int, topP: Double, toolsJson: String, historyJson: String, promise: Promise) {
val safe = SafePromise(promise, TAG)
@@ -268,6 +292,16 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
)
}
+ // DEV: constrained decoding is a per-conversation experimental flag,
+ // so it must be set before createConversation. Guarded - a missing/renamed
+ // API in a future SDK must not break conversation setup.
+ try {
+ ExperimentalFlags.enableConversationConstrainedDecoding = constrainedEnabled
+ if (constrainedEnabled) debugLog("[DevGrammar-LiteRT] enableConversationConstrainedDecoding=true (type=$constraintType len=${constraintString.length})")
+ } catch (e: Throwable) {
+ Log.w(TAG, "[DevGrammar-LiteRT] could not set constrained-decoding flag: ${e.message}")
+ }
+
val toolProviders = buildToolProviders(toolsJson)
val initialMessages = parseHistoryMessages(historyJson)
debugLog("ConversationConfig — historyTurns=${initialMessages.size} tools=${toolProviders.size} maxTokenBudget=$configuredMaxTokens autoToolCalling=${toolProviders.isNotEmpty()}")
@@ -409,16 +443,37 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
safe.reject("LITERT_NO_CONV", "No conversation. Call resetConversation first.", null)
return@launch
}
-
currentJob = launch {
try {
val contents = buildSendContents(imageUris, audioUris, text, safe) ?: return@launch
+ // DEV: attach an LLGuidance constraint via OptionalArgs. The exact
+ // map shape is UNVERIFIED (C++ docs only), so if building/starting the
+ // constrained flow throws we log and fall back to an unconstrained send
+ // - a probe that can reveal the contract from logs without breaking chat.
+ val flow = if (constrainedEnabled && constraintString.isNotEmpty()) {
+ try {
+ val optionalArgs = mapOf(
+ "decoding_constraint" to mapOf(
+ "constraint_type" to constraintType,
+ "constraint_string" to constraintString,
+ ),
+ )
+ Log.i(TAG, "[DevGrammar-LiteRT] sending WITH decoding_constraint type=$constraintType len=${constraintString.length}")
+ conv.sendMessageAsync(contents, optionalArgs)
+ } catch (e: Throwable) {
+ Log.w(TAG, "[DevGrammar-LiteRT] constrained send failed to start (${e.message}); falling back unconstrained")
+ conv.sendMessageAsync(contents)
+ }
+ } else {
+ conv.sendMessageAsync(contents)
+ }
+
var tokenCount = 0
- conv.sendMessageAsync(contents)
+ flow
.collect { message ->
tokenCount++
- if (tokenCount == 1) Log.i(TAG, "sendMessage — first message from model (audio=${audioUris.size} image=${imageUris.size})")
+ if (tokenCount == 1) Log.i(TAG, "sendMessage — first message from model (audio=${audioUris.size} image=${imageUris.size} constrained=${constrainedEnabled && constraintString.isNotEmpty()})")
dispatchStreamToken(message)
}
@@ -543,10 +598,34 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
}
try {
conv.close()
- Log.d(TAG, "closeConversationSafely — closed")
+ Log.d(TAG, "closeConversationSafely — closed id=${System.identityHashCode(conv)}")
} catch (e: Exception) {
Log.w(TAG, "closeConversationSafely — error: ${e.message}")
}
+
+ // litert-uaf mitigation (crash observed on the fbjni "HybridData Dest"
+ // GC thread during insight generation). Insight runs churn conversations
+ // hard: reset -> close -> recreate, back to back. Each finished generation
+ // leaves fbjni HybridData peers (event/bridge objects) waiting to be
+ // reclaimed on the GC finalizer thread. Under that churn the reclaim can
+ // fire LATER, in the middle of the NEXT conversation's active decode, and
+ // dereference memory that is already gone -> SIGSEGV (fault 0x0101..).
+ // We are at a quiescent point here: the previous generation is cancelled
+ // and joined (above) and the next one has not started, so drain the
+ // reference queue NOW, off the decode path, so those reclaims do not
+ // overlap live native work. This is a mitigation aimed at the observed
+ // timing, not a proven root-cause fix - verify on-device before relying on it.
+ try {
+ System.gc()
+ System.runFinalization()
+ // gc() only ENQUEUES fbjni's phantom-ref reclaims; the "HybridData
+ // Dest" thread drains them asynchronously. Yield briefly (non-blocking,
+ // we are in a suspend fun) so that thread runs the reclaims before the
+ // caller creates the next conversation and starts a fresh decode.
+ delay(16)
+ } catch (e: Throwable) {
+ Log.w(TAG, "closeConversationSafely — gc drain skipped: ${e.message}")
+ }
}
private suspend fun cleanupEngine() {
diff --git a/docs/plans/audio-mode-progress-captions.md b/docs/plans/audio-mode-progress-captions.md
deleted file mode 100644
index cf64fee15..000000000
--- a/docs/plans/audio-mode-progress-captions.md
+++ /dev/null
@@ -1,131 +0,0 @@
-# Plan: Phase-aware progress captions for audio mode (no "is it broken?" gaps)
-
-## Goal
-When a user sends a voice message, several silent gaps occur (prefill, thinking,
-TTS synthesis). Today all of them show the same pulsing dots, so the user can't
-tell working-but-slow from stuck. Add a short status caption that names the phase,
-designed so it can never flicker, reverse, or get stranded.
-
-## Non-goal
-Do NOT build a new independent state machine or timers. The caption must be a pure
-function of state that already exists, gated by the same condition that mounts the
-in-progress bubble, so it cannot outlive or contradict that bubble.
-
----
-
-## The gaps (current behavior)
-
-| Phase | Trigger | Shown now |
-|---|---|---|
-| A. Prefill | sent → before first token (~3-5s, audio TTFT) | pulsing dots, silent |
-| B. Thinking | reasoning tokens stream (thinking enabled); never spoken | same dots, silent |
-| C. Synthesis | answer streaming; Kokoro synthesizing first sentence | play button + tiny spinner, silent |
-| D. Playing | audio plays | waveform animates (good) |
-
-Streaming TTS (`pro/audio/index.ts` `audio.onStreamingToken` → `feedStreamingText`)
-speaks sentence-by-sentence, so C and the tail of answer-generation OVERLAP. The
-design must tolerate overlap rather than force a linear phase.
-
----
-
-## Design: monotonic phase, pure-derived, playback wins
-
-### Signal sources (read-only; do not add new state)
-- Message: `msg.isThinking`, `msg.reasoningContent`, `msg.content`.
-- TTS store (`pro/audio/ttsStore.ts`): `playbackStatus`, `currentMessageId`, derived `isSpeaking`/`isLoading`.
-- The bubble already mounts on `isStreamingThis || isThinkingItem`
- (`pro/audio/ui/MessageAudioMode.tsx` `renderAudioInProgress`).
-
-### Phase ladder (advance-only)
-```
-0 waiting → "Processing your message…" (in-progress, no reasoning, no answer yet)
-1 thinking → "Thinking…" (reasoningContent growing AND content empty)
-2 answering → "Preparing audio…" (content non-empty, TTS not yet playing THIS msg)
-3 playing → (no caption; waveform owns UI)(currentMessageId===msg.id && playing)
-```
-Rules that kill the failure modes:
-1. **Monotonic.** Keep a ref of the highest phase reached for this messageId; never
- render a lower phase even if a signal momentarily regresses. (Prevents flicker /
- backward "Preparing→Thinking".)
-2. **Playback wins.** If `ttsStore.currentMessageId === msg.id` and status is
- playing/processing → phase 3, caption empty, waveform shows progress. Never draw a
- caption over real audio.
-3. **Cross-message gate.** Only read `playbackStatus` when
- `currentMessageId === msg.id`; otherwise treat as not-playing. (Prevents a prior
- message's TTS state bleeding onto this bubble.)
-4. **Lifetime = bubble.** Caption is rendered only inside the in-progress bubble
- (gated on `isStreamingThis || isThinkingItem`). On error/abort/finalize the bubble
- unmounts → caption gone. It can never strand on "Thinking…".
-5. **Graceful "thinking" detection.** Only show "Thinking…" when reasoning is actively
- growing AND answer still empty. If the model's reasoning format is unparseable
- (see `buildAudioBubbleProps` note), fall back to "Responding…" rather than assert a
- phase. Never block the answer on a wrong thinking guess.
-
-### Why this avoids the documented risks
-- Flicker/reverse → blocked by monotonic ref (rule 1).
-- Overlap of generate+speak → playback-wins (rule 2) yields to the waveform.
-- Singleton bleed → message-id gate (rule 3).
-- Stuck terminal state → lifetime tied to bubble (rule 4); no independent state to leak.
-- Parser fragility → graceful fallback (rule 5).
-
----
-
-## Implementation
-
-### 1. New hook: `useAudioProgressPhase(msg)` — `pro/audio/ui/AudioMessageBubble/useAudioProgressPhase.ts`
-- Subscribe narrowly to `ttsStore`: `currentMessageId`, `playbackStatus` (select only
- these two to limit re-renders).
-- Compute raw phase from the ladder above using `msg` + gated TTS read.
-- Hold `useRef(maxPhase)` keyed by `msg.id`; clamp raw phase up to it (reset the ref
- when `msg.id` changes).
-- Return `{ phase, caption }` where caption is '' for phase 3.
-- Copy (brand voice — plain, no exclamation, no em dash):
- - 0 → `Processing your message…`
- - 1 → `Thinking…`
- - 2 → `Preparing audio…`
- - fallback → `Responding…`
-
-### 2. `AudioMessageBubble/index.tsx`
-- Accept optional `statusCaption?: string` (or call the hook directly when `isLoading`).
-- In the `isLoading && !isUser` branch, render the caption as a META-styled `Text`
- beside/under `ThinkingDots`. Keep dots animating (moving indicator + label).
-- Use `TYPOGRAPHY.meta` + `colors.textMuted`. No new colors, no hardcoded sizes.
-
-### 3. `MessageAudioMode.tsx`
-- In `renderAudioInProgress`, pass the computed caption into `AudioMessageBubble`
- (or let the bubble call the hook; either is fine since it's pure).
-
-### 4. Instant bubble on send (close Gap A's "nothing on screen")
-- Verify the assistant in-progress placeholder (`msg.isThinking` item or streaming
- message) is added to the store IMMEDIATELY on send, before prefill completes. If
- there's a delay, the user sees only their own bubble + silence during prefill.
-- Trace: core generation path that creates the thinking/streaming placeholder
- (`src/services/generationService*`, `useChatGenerationActions`). If the placeholder
- is created only on first token, add it at generation start for audio mode. Confirm
- on-device that the dots+caption appear within ~100ms of sending.
-
----
-
-## Tests (both unit + integration, per project rules)
-
-Unit (`__tests__/unit/...`):
-- `useAudioProgressPhase`: returns correct caption per signal combo.
-- Monotonic clamp: feed phase 2 then a regressing signal → still phase 2.
-- Cross-message gate: `currentMessageId` = other id → playback ignored.
-- Playback wins: this-id + playing → phase 3, empty caption.
-- Unparseable thinking → "Responding…", never blocks.
-- Reset on `msg.id` change → maxPhase ref resets.
-
-Integration (`__tests__/integration/...`):
-- Simulated send → prefill → thinking → answer → TTS preparing → playing: caption
- advances 0→1→2→'' and never regresses.
-- Error mid-thinking: in-progress bubble unmounts, no stranded caption.
-
----
-
-## Risk register (carry into review)
-- Re-render cost: select only 2 TTS fields; memoize caption. Verify no per-token
- re-render storm of all bubbles.
-- Streaming vs fallback TTS: "Preparing audio…" may flash briefly (streaming) or
- linger (fallback) — acceptable; both read as "working".
-- Copy review: run the brand-voice checklist on the four strings before commit.
diff --git a/docs/plans/desktop-parity-roadmap.md b/docs/plans/desktop-parity-roadmap.md
deleted file mode 100644
index 500c54b17..000000000
--- a/docs/plans/desktop-parity-roadmap.md
+++ /dev/null
@@ -1,103 +0,0 @@
-# Mobile <- Desktop Parity Roadmap
-
-Status: Planning
-Goal: Bring the mobile app to feature parity with the desktop app, plus mobile's own
-native bet (offline recordings). This doc is the full picture across epics; sprint
-assignment and estimates are decided separately in Jira.
-
-## Two tracks
-
-1. Mobile-native bet: Offline Recordings & Transcriptions
- (see `offline-recordings.md`). Mobile's analog of the desktop meeting recorder.
-2. Desktop parity: close the capability gaps where mobile lags desktop.
-
-## Parity status (full map)
-
-### Already at parity - no work
-Chat + vision, image generation, voice STT (Whisper), tools/MCP, projects + RAG,
-model catalog/downloads, Keygen licensing.
-
-TTS / Audio Mode - BUILT and shipped in the `mobile-pro` submodule
-(`pro/audio/`): Kokoro + OuteTTS + Qwen3 engines behind an EngineRegistry, full
-`ttsService` lifecycle (download/load/generate/save/speak/stop/cache), streaming
-playback, waveforms, and complete audio-mode UI. At polish stage (recent iOS playback
-fix). NOTE: the public `TTS_IMPLEMENTATION_PLAN.md` is stale and says "NOT STARTED" -
-trust the pro code. Only follow-up: the audio module has no tests yet (quality task,
-not a feature gap).
-
-### Gaps - candidate epics
-| Desktop capability | Mobile today | Epic |
-|--------------------|--------------|------|
-| Meeting recorder | none | A: Offline Recordings (planned) |
-| Personas (assistants w/ memory + integrations) | none in pro yet | C: Personas |
-| Artifacts / canvas (HTML/React/SVG/Mermaid render) | none | D: Artifacts |
-| Local OpenAI-compatible gateway (serve over LAN) | none | E: On-phone server (legacy SCRUM-150, SCRUM-157) |
-| Clipboard manager | basic copy only | F: Clipboard (low priority) |
-
-### OS-constrained desktop features - kept in-scope for now
-Rely on capabilities macOS exposes but iOS/Android restrict. Treated as doable for
-planning; the OS constraint is a risk to resolve (research spike) before build, not a
-hard exclusion yet.
-| Desktop capability | Constraint | Epic |
-|--------------------|-----------|------|
-| Screen capture -> OCR -> entities | No continuous background screenshot on iOS/Android | G: Capture loop (spike first) |
-| Day / Replay / Reflect | Fed by capture; data model ports, input source does not | H: Memory/Reflect (depends on G) |
-| System / call-audio capture | OS-restricted; mobile recordings are mic-only | folded into Recordings risk notes |
-
-## Build sequence (active order; not sprint-bound)
-
-A -> G -> H -> C -> D -> E -> F
-
-(B / TTS is already shipped, so it is not in the build sequence - it sits under
-"already at parity".)
-
-1. A - Offline Recordings & Transcriptions (mobile-native; in flight / next)
-2. G - Capture loop (research spike first to define mobile feasibility)
-3. H - Memory / Day / Replay / Reflect (depends on G's outcome)
-4. C - Personas
-5. D - Artifacts / Canvas
-6. E - On-phone OpenAI-compatible server
-7. F - Clipboard manager (low priority)
-
-## Epics overview
-
-### Epic A - Offline Recordings & Transcriptions (mobile-native)
-Detailed in `offline-recordings.md`. Record (fg + bg) -> chunked Whisper transcription
--> LLM summary + action items. Pro-gated. Reuse note: `pro/audio/recordBridge.ts`
-already bridges mic input for audio mode.
-
-### Epic G - Capture loop (research spike first)
-Screen/context capture -> OCR -> observations + entities. Desktop-style "sees your
-work" loop. Mobile feasibility uncertain (no background screenshot API); start with a
-spike to define what is achievable (share-sheet capture, manual screenshots,
-accessibility APIs) before committing build stories.
-
-### Epic H - Memory / Day / Replay / Reflect
-Journal (Day), timeline (Replay), analytics (Reflect). Data structures port directly
-from desktop; the input source depends on Epic G. Sequenced after G.
-
-### Epic C - Personas
-Named assistants with system prompt, memory (cross-conversation RAG), capabilities
-(text/voice/vision/image/RAG), and skills/integrations. Plan exists
-(`PERSONAS_IMPLEMENTATION_PLAN.md`). No personas module in pro yet - genuine gap.
-
-### Epic D - Artifacts / Canvas
-Render model output as HTML / React-JSX / SVG / Mermaid / Markdown in a sandboxed
-webview. Pure RN, highly portable from desktop.
-
-### Epic E - On-phone OpenAI-compatible server
-Expose local models as an OpenAI-compatible API over the home network so other
-devices/apps can use the phone's models. Parity with desktop gateway. Maps to legacy
-tickets SCRUM-150 (Android server) and SCRUM-157 (OpenAI-compatible API).
-
-### Epic F - Clipboard manager (low priority)
-Searchable on-device clipboard history. Desktop has a `@offgrid/clipboard` engine to
-reuse. Lower user value on mobile; parked low.
-
-## Notes for the team
-- Pro-gating reuses `proLicenseService` (shared Keygen account with desktop; one
- license already spans platforms).
-- Reuse-first: desktop packages (`@offgrid/rag`, `@offgrid/models`, `@offgrid/clipboard`),
- the `mobile-pro` audio module, and desktop plan docs are the reference; do not fork.
-- Epics G/H carry real OS-feasibility risk - resolve via spike before sizing build work.
-- TTS audio module needs test coverage added (repo mandates unit + integration tests).
diff --git a/docs/plans/offline-recordings.md b/docs/plans/offline-recordings.md
deleted file mode 100644
index 4b283c0f2..000000000
--- a/docs/plans/offline-recordings.md
+++ /dev/null
@@ -1,116 +0,0 @@
-# Offline Recordings & Transcriptions
-
-Status: Planning
-Epic: Offline Recordings & Transcriptions (single epic, all stories inside)
-Tier: Pro-gated
-
-## What this is
-
-A Pro surface in the mobile app to record audio on-device, transcribe it locally
-with Whisper, and generate an LLM title, summary, and action items. Recording works
-in the foreground and in the background (lock screen). Nothing leaves the phone.
-
-This is mobile's native analog of the desktop meeting recorder. The desktop recorder
-relies on macOS ScreenCaptureKit + system-audio loopback, which iOS and Android do not
-expose. Mobile therefore captures microphone audio rather than call/system audio.
-
-## Why this is mostly orchestration, not new capability
-
-The core primitives already exist in the codebase:
-
-- Recording: `src/services/audioRecorderService.ts` records 16 kHz mono WAV to disk
- (foreground today).
-- Transcription: `src/services/whisperService.ts` `transcribeFile(path)` transcribes
- any audio file with progress callbacks.
-- Summarization: `src/services/generationService.ts` runs the active LLM; reuse it with
- a summary prompt.
-- Persistence/metadata: `chatStore` already models audio fields (audioPath,
- waveformData, audioDurationSeconds); Whisper model download/management is solved.
-
-The genuinely new work: the Recordings product surface, the record -> transcribe ->
-summarize orchestration and persistence, background capture (the heavy native piece),
-Pro-gating, and storage management.
-
-## Decisions locked
-
-- Transcription trigger: automatic after recording stops, with a setting to disable
- (auto-with-toggle).
-- Long audio: chunked transcription with a progress indicator (handles long meetings).
- Confirm whisper.rn practical segment limits during story 4.
-- Summary depth: structured summary + extracted action items (title, TL;DR, bullets,
- action items).
-- Background recording: in scope (iOS background-audio + AVAudioSession; Android
- foreground service + mic notification).
-
-## Stories (single epic - no sprint assignment yet)
-
-1. Recordings data layer
- `recordingStore` (Zustand) + SQLite table:
- `id, title, audioPath, durationSeconds, createdAt, transcript, summary,
- actionItems, status`. Foundation for all other stories.
-
-2. Record screen (foreground)
- Start/stop, elapsed timer, live amplitude meter (react-native-audio-api),
- save WAV to `Documents/recordings/`.
-
-3. Recordings list + playback
- New tab. List by date, play/pause/seek, delete a recording.
-
-4. Transcription orchestration
- On stop (when auto enabled) -> `transcribeFile` with progress UI. Chunk long audio
- into sequential segments. Persist transcript. Handle model-not-loaded.
-
-5. LLM summary + action items
- Transcript -> summary prompt -> structured title / TL;DR / bullets / action items.
- Re-run summary on demand. Reuses `generationService`.
-
-6. Recording detail screen
- Tabbed Transcript / Summary view, copy, export as text.
-
-7. Background recording (iOS + Android)
- iOS background-audio mode + AVAudioSession; Android foreground service with a
- persistent mic notification (FOREGROUND_SERVICE_MICROPHONE on Android 14+).
- Interruption handling (incoming call, Siri, app kill/restart).
- Heaviest, highest-risk story. Native work on both platforms.
-
-8. Pro-gating
- Gate the whole surface behind the Pro license via `proLicenseService`; upsell entry
- point for non-Pro users. Land before story 7 so native work isn't redone.
-
-9. Storage management
- Size display, bulk delete, quota warning. Mirrors desktop retention behavior.
-
-10. Settings + auto-transcribe toggle
- Setting to enable/disable auto transcription; transcription language; clear-cache.
-
-11. QA, edge cases, polish
- Interrupted-recording recovery, permissions flows, no-model prompt, brand-voice
- copy pass, design-token compliance, unit + integration tests per repo conventions
- (eslint + tsc + tests; Gemini/Codecov/Sonar gates green).
-
-## Dependencies and sequencing notes
-
-- Story 1 (data layer) blocks everything.
-- Stories 2 -> 3 -> 4 -> 5 -> 6 form the core foreground happy path.
-- Story 8 (Pro-gating) should land before story 7 (background recording) so the
- expensive native work is not restructured for gating later.
-- Story 7 is ~the largest effort and carries App Store / Android-policy review risk.
- Even inside one epic, treat it as separable: stories 1-6 + 8-11 deliver a complete,
- shippable foreground feature without it.
-
-## Out of scope (flag for stakeholders)
-
-- System / call-audio capture (OS-restricted on iOS and Android).
-- Speaker diarization (who-said-what).
-- Cross-device sync of recordings (would route through Off Grid Sync later).
-
-## Reuse map (build on, do not fork)
-
-| Need | Existing |
-|------|----------|
-| Record WAV | `audioRecorderService.ts` |
-| Transcribe file | `whisperService.transcribeFile()` |
-| Summarize | `generationService.ts` |
-| Audio metadata shape | `chatStore` audio fields |
-| Whisper model mgmt | existing model manager + whisper store |
-| Pro license check | `proLicenseService.ts` |
diff --git a/pro b/pro
index 5f0997019..10f49a18d 160000
--- a/pro
+++ b/pro
@@ -1 +1 @@
-Subproject commit 5f09970196554f69cbbdb936426e8827bc1b3cc4
+Subproject commit 10f49a18d830713de8fb8ff4bf2e68e57d04470d
diff --git a/src/components/DevGrammarModal.tsx b/src/components/DevGrammarModal.tsx
new file mode 100644
index 000000000..97e8ea14d
--- /dev/null
+++ b/src/components/DevGrammarModal.tsx
@@ -0,0 +1,271 @@
+import React, { useEffect, useState } from 'react';
+import {
+ Modal,
+ View,
+ Text,
+ TextInput,
+ TouchableOpacity,
+ Switch,
+ ScrollView,
+ Pressable,
+} from 'react-native';
+import Icon from 'react-native-vector-icons/Feather';
+import { useTheme, useThemedStyles } from '../theme';
+import type { ThemeColors, ThemeShadows } from '../theme';
+import { SPACING, TYPOGRAPHY, FONTS } from '../constants';
+import { useDevInferenceStore } from '../stores/devInferenceStore';
+import logger from '../utils/logger';
+
+const STARTER_GRAMMAR = `root ::= "TITLE: " line "\\nSUMMARY: " line "\\nACTIONS:\\n" acts
+acts ::= "none\\n" | item+
+item ::= "- " line "\\n"
+line ::= [^\\n]+`;
+
+interface DevGrammarModalProps {
+ visible: boolean;
+ onClose: () => void;
+}
+
+/**
+ * DEV-ONLY test harness: paste a GBNF grammar (plus optional temperature /
+ * assistant prefill) and apply it to the next chat completion(s). Lets us test
+ * grammar-constrained / prefill / temp=0 output on the real on-device model
+ * without leaving the app. Only mounted behind `__DEV__`.
+ */
+export const DevGrammarModal: React.FC = ({ visible, onClose }) => {
+ const styles = useThemedStyles(createStyles);
+ const { colors } = useTheme();
+ const store = useDevInferenceStore();
+
+ // Local drafts so edits aren't live until Apply. Seed from the store on open.
+ const [grammar, setGrammar] = useState('');
+ const [temperature, setTemperature] = useState('');
+ const [prefix, setPrefix] = useState('');
+ const [maxWords, setMaxWords] = useState('');
+ const [litertType, setLitertType] = useState<'json_schema' | 'lark' | 'regex'>('json_schema');
+ const [litertConstraint, setLitertConstraint] = useState('');
+ const [enabled, setEnabled] = useState(false);
+
+ useEffect(() => {
+ if (!visible) return;
+ setGrammar(store.grammar);
+ setTemperature(store.temperature != null ? String(store.temperature) : '');
+ setPrefix(store.assistantPrefix);
+ setMaxWords(store.maxWords != null ? String(store.maxWords) : '');
+ setLitertType(store.litertConstraintType);
+ setLitertConstraint(store.litertConstraintString);
+ setEnabled(store.enabled);
+ // Seed once per open; store fields are intentionally not deps.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [visible]);
+
+ const apply = () => {
+ const t = temperature.trim();
+ const parsedTemp = t.length > 0 ? Number(t) : NaN;
+ const w = maxWords.trim();
+ const parsedWords = w.length > 0 ? Math.round(Number(w)) : NaN;
+ store.setGrammar(grammar);
+ store.setTemperature(Number.isFinite(parsedTemp) ? parsedTemp : undefined);
+ store.setAssistantPrefix(prefix);
+ store.setMaxWords(Number.isFinite(parsedWords) && parsedWords > 0 ? parsedWords : undefined);
+ store.setLitertConstraintType(litertType);
+ store.setLitertConstraintString(litertConstraint);
+ store.setLastError(undefined);
+ store.setEnabled(enabled);
+ logger.log(
+ `[DevGrammar] ARMED enabled=${enabled} grammarLen=${grammar.trim().length} ` +
+ `temp=${Number.isFinite(parsedTemp) ? parsedTemp : 'default'} prefill=${prefix ? JSON.stringify(prefix) : 'none'} ` +
+ `maxWords=${Number.isFinite(parsedWords) && parsedWords > 0 ? parsedWords : 'none'}`,
+ );
+ onClose();
+ };
+
+ const clearAll = () => {
+ store.clear();
+ setGrammar('');
+ setTemperature('');
+ setPrefix('');
+ setMaxWords('');
+ setLitertType('json_schema');
+ setLitertConstraint('');
+ setEnabled(false);
+ };
+
+ return (
+
+
+
+
+
+
+ Grammar test harness
+ DEV
+
+
+
+
+
+
+
+ GBNF grammar
+
+ setGrammar(STARTER_GRAMMAR)}>
+ Insert starter grammar
+
+
+
+
+ Temperature
+
+
+
+ Max words
+
+
+
+
+ Assistant prefill
+
+
+
+ LiteRT constraint (LLGuidance)
+ Used when a LiteRT model is active. Not GBNF - pick a format below.
+
+ {(['json_schema', 'lark', 'regex'] as const).map((t) => (
+ setLitertType(t)}
+ >
+ {t}
+
+ ))}
+
+
+
+
+
+ Enable override
+ GBNF applies on llama.cpp; the LiteRT constraint applies on LiteRT. Tools off while a grammar is active.
+
+ { logger.log(`[DevGrammar] enable toggle -> ${v}`); setEnabled(v); }}
+ />
+
+
+ {store.lastError ? (
+ Grammar error: {store.lastError}
+ ) : null}
+
+
+
+
+ Clear
+
+
+ Apply
+
+
+
+
+
+ );
+};
+
+DevGrammarModal.displayName = 'DevGrammarModal';
+
+const createStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({
+ backdrop: { ...StyleSheetAbsolute, backgroundColor: 'rgba(0,0,0,0.5)' },
+ centerWrap: { flex: 1, alignItems: 'center' as const, justifyContent: 'center' as const, padding: SPACING.lg },
+ card: {
+ width: '100%' as const,
+ maxWidth: 460,
+ maxHeight: '85%' as const,
+ backgroundColor: colors.surface,
+ borderRadius: 14,
+ padding: SPACING.lg,
+ ...shadows.medium,
+ },
+ headerRow: { flexDirection: 'row' as const, alignItems: 'center' as const, gap: SPACING.sm, marginBottom: SPACING.md },
+ title: { ...TYPOGRAPHY.h3, color: colors.text },
+ devBadge: { backgroundColor: `${colors.primary}22`, borderRadius: 5, paddingHorizontal: 5, paddingVertical: 1 },
+ devBadgeText: { ...TYPOGRAPHY.labelSmall, color: colors.primary },
+ flex: { flex: 1 },
+ body: { flexGrow: 0 },
+ label: { ...TYPOGRAPHY.label, color: colors.textSecondary, marginBottom: SPACING.xs, marginTop: SPACING.sm },
+ input: {
+ borderWidth: 1,
+ borderColor: colors.border,
+ borderRadius: 8,
+ paddingHorizontal: SPACING.sm,
+ paddingVertical: SPACING.sm,
+ color: colors.text,
+ backgroundColor: colors.background,
+ ...TYPOGRAPHY.bodySmall,
+ },
+ grammarInput: { minHeight: 120, maxHeight: 220, fontFamily: FONTS.mono, textAlignVertical: 'top' as const },
+ starterLink: { ...TYPOGRAPHY.meta, color: colors.primary, marginTop: SPACING.xs },
+ twoCol: { flexDirection: 'row' as const, gap: SPACING.md },
+ col: { flex: 1 },
+ divider: { height: 1, backgroundColor: colors.border, marginTop: SPACING.lg, marginBottom: SPACING.xs },
+ sectionLabel: { ...TYPOGRAPHY.label, color: colors.textSecondary, marginTop: SPACING.sm },
+ typeRow: { flexDirection: 'row' as const, gap: SPACING.xs, marginTop: SPACING.sm, marginBottom: SPACING.xs },
+ typeChip: { paddingHorizontal: SPACING.sm, paddingVertical: 5, borderRadius: 7, borderWidth: 1, borderColor: colors.border },
+ typeChipOn: { backgroundColor: `${colors.primary}22`, borderColor: colors.primary },
+ typeChipText: { ...TYPOGRAPHY.meta, color: colors.textMuted },
+ typeChipTextOn: { color: colors.primary },
+ enableRow: { flexDirection: 'row' as const, alignItems: 'center' as const, gap: SPACING.md, marginTop: SPACING.md },
+ enableLabel: { ...TYPOGRAPHY.body, color: colors.text },
+ hint: { ...TYPOGRAPHY.meta, color: colors.textMuted, marginTop: 2 },
+ error: { ...TYPOGRAPHY.bodySmall, color: colors.error, marginTop: SPACING.md },
+ actions: { flexDirection: 'row' as const, justifyContent: 'flex-end' as const, gap: SPACING.sm, marginTop: SPACING.lg },
+ secondaryBtn: { paddingHorizontal: SPACING.lg, paddingVertical: SPACING.sm, borderRadius: 8, borderWidth: 1, borderColor: colors.border },
+ secondaryText: { ...TYPOGRAPHY.body, color: colors.textSecondary },
+ primaryBtn: { paddingHorizontal: SPACING.lg, paddingVertical: SPACING.sm, borderRadius: 8, backgroundColor: colors.primary },
+ primaryText: { ...TYPOGRAPHY.body, color: colors.background },
+});
+
+const StyleSheetAbsolute = { position: 'absolute' as const, top: 0, left: 0, right: 0, bottom: 0 };
diff --git a/src/components/Toast.tsx b/src/components/Toast.tsx
new file mode 100644
index 000000000..020c9a486
--- /dev/null
+++ b/src/components/Toast.tsx
@@ -0,0 +1,144 @@
+import React, { useEffect, useRef } from 'react';
+import { Animated, Text, TouchableOpacity, StyleSheet } from 'react-native';
+import { useSafeAreaInsets } from 'react-native-safe-area-context';
+import Icon from 'react-native-vector-icons/Feather';
+import { create } from 'zustand';
+import { useTheme, useThemedStyles } from '../theme';
+import type { ThemeColors, ThemeShadows } from '../theme';
+import { SPACING, TYPOGRAPHY } from '../constants';
+
+/**
+ * One cross-platform toast (not ToastAndroid, which is Android-only): a brief,
+ * non-blocking message that slides up from the bottom and auto-dismisses. A
+ * single host is mounted once at the app root; anywhere in the app (screens or
+ * services) calls `showToast(message)` imperatively - no per-screen wiring.
+ */
+export interface ToastOptions {
+ /** Optional leading Feather icon name. */
+ icon?: string;
+ /** Auto-dismiss delay in ms (default 2600). */
+ durationMs?: number;
+}
+
+interface ToastState {
+ visible: boolean;
+ message: string;
+ icon?: string;
+ durationMs: number;
+ /** Bumped on every show so the host restarts its timer even for the same text. */
+ nonce: number;
+ show: (message: string, opts?: ToastOptions) => void;
+ hide: () => void;
+}
+
+const DEFAULT_DURATION_MS = 2600;
+
+const useToastStore = create((set) => ({
+ visible: false,
+ message: '',
+ icon: undefined,
+ durationMs: DEFAULT_DURATION_MS,
+ nonce: 0,
+ show: (message, opts) =>
+ set((s) => ({
+ visible: true,
+ message,
+ icon: opts?.icon,
+ durationMs: opts?.durationMs ?? DEFAULT_DURATION_MS,
+ nonce: s.nonce + 1,
+ })),
+ hide: () => set({ visible: false }),
+}));
+
+/** Show a toast from anywhere (screens or services). */
+export const showToast = (message: string, opts?: ToastOptions): void =>
+ useToastStore.getState().show(message, opts);
+
+/** Hide the current toast early. */
+export const hideToast = (): void => useToastStore.getState().hide();
+
+/**
+ * The toast host. Mount exactly once near the app root (inside SafeAreaProvider).
+ * Renders nothing until a toast is shown.
+ */
+export const Toast: React.FC = () => {
+ const { colors } = useTheme();
+ const styles = useThemedStyles(createStyles);
+ const insets = useSafeAreaInsets();
+
+ const visible = useToastStore((s) => s.visible);
+ const message = useToastStore((s) => s.message);
+ const icon = useToastStore((s) => s.icon);
+ const durationMs = useToastStore((s) => s.durationMs);
+ const nonce = useToastStore((s) => s.nonce);
+ const hide = useToastStore((s) => s.hide);
+
+ const opacity = useRef(new Animated.Value(0)).current;
+ const translateY = useRef(new Animated.Value(20)).current;
+ const timer = useRef | null>(null);
+ // Keep the last message on screen through the fade-out so it doesn't blank mid-animation.
+ const [shown, setShown] = React.useState(false);
+
+ useEffect(() => {
+ if (timer.current) { clearTimeout(timer.current); timer.current = null; }
+ if (visible) {
+ setShown(true);
+ Animated.parallel([
+ Animated.timing(opacity, { toValue: 1, duration: 180, useNativeDriver: true }),
+ Animated.timing(translateY, { toValue: 0, duration: 180, useNativeDriver: true }),
+ ]).start();
+ timer.current = setTimeout(hide, durationMs);
+ } else {
+ Animated.parallel([
+ Animated.timing(opacity, { toValue: 0, duration: 160, useNativeDriver: true }),
+ Animated.timing(translateY, { toValue: 20, duration: 160, useNativeDriver: true }),
+ ]).start(({ finished }) => { if (finished) setShown(false); });
+ }
+ return () => { if (timer.current) { clearTimeout(timer.current); timer.current = null; } };
+ // nonce forces re-run (and timer reset) even when message text is unchanged.
+ }, [visible, nonce, durationMs, hide, opacity, translateY]);
+
+ if (!shown) return null;
+
+ return (
+
+
+ {icon ? : null}
+ {message}
+
+
+ );
+};
+
+Toast.displayName = 'Toast';
+
+const createStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({
+ wrap: {
+ position: 'absolute' as const,
+ left: SPACING.lg,
+ right: SPACING.lg,
+ alignItems: 'center' as const,
+ },
+ toast: {
+ flexDirection: 'row' as const,
+ alignItems: 'center' as const,
+ maxWidth: '100%' as const,
+ paddingHorizontal: SPACING.lg,
+ paddingVertical: SPACING.md,
+ borderRadius: 12,
+ backgroundColor: colors.surface,
+ borderWidth: StyleSheet.hairlineWidth,
+ borderColor: colors.border,
+ ...shadows.small,
+ },
+ icon: { marginRight: SPACING.sm },
+ text: { ...TYPOGRAPHY.bodySmall, color: colors.text, flexShrink: 1 },
+});
diff --git a/src/components/index.ts b/src/components/index.ts
index 7e21a867d..3982b338d 100644
--- a/src/components/index.ts
+++ b/src/components/index.ts
@@ -8,6 +8,8 @@ export { ChatInput } from './ChatInput';
export { VoiceRecordButton } from './VoiceRecordButton';
export { ModelSelectorModal } from './ModelSelectorModal';
export { GenerationSettingsModal } from './GenerationSettingsModal';
+export { Toast, showToast, hideToast } from './Toast';
+export type { ToastOptions } from './Toast';
export { CustomAlert, showAlert, hideAlert, initialAlertState } from './CustomAlert';
export type { AlertButton, AlertState, CustomAlertProps } from './CustomAlert';
export { CenteredAlert } from './CenteredAlert';
diff --git a/src/screens/ChatScreen/ChatScreenComponents.tsx b/src/screens/ChatScreen/ChatScreenComponents.tsx
index 1f1de19ce..0ec78104a 100644
--- a/src/screens/ChatScreen/ChatScreenComponents.tsx
+++ b/src/screens/ChatScreen/ChatScreenComponents.tsx
@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useState } from 'react';
import {
View,
Text,
@@ -11,8 +11,11 @@ import Icon from 'react-native-vector-icons/Feather';
import { SafeAreaView } from 'react-native-safe-area-context';
import { AttachStep } from 'react-native-spotlight-tour';
import { ModelSelectorModal } from '../../components';
+import { DevGrammarModal } from '../../components/DevGrammarModal';
import { AnimatedEntry } from '../../components/AnimatedEntry';
import { llmService } from '../../services';
+import { useDevInferenceStore } from '../../stores/devInferenceStore';
+import logger from '../../utils/logger';
import { createStyles } from './styles';
import { useTheme } from '../../theme';
import { getSlot, SLOTS } from '../../bootstrap/slotRegistry';
@@ -94,7 +97,13 @@ export const ChatHeader: React.FC<{
setShowSettingsPanel: (v: boolean) => void;
setShowProjectSelector: (v: boolean) => void;
isRemote?: boolean;
-}> = ({ styles, colors, activeConversation, activeProject, navigation, onOpenModels, setShowSettingsPanel, setShowProjectSelector, isRemote }) => (
+}> = ({ styles, colors, activeConversation, activeProject, navigation, onOpenModels, setShowSettingsPanel, setShowProjectSelector, isRemote }) => {
+ // DEV-only grammar test harness (see DevGrammarModal). devActive lights the
+ // header icon when a custom grammar is armed so it's obvious the next reply
+ // is constrained.
+ const [devOpen, setDevOpen] = useState(false);
+ const devActive = useDevInferenceStore((s) => s.enabled && s.grammar.trim().length > 0);
+ return (
navigation.goBack()}>
@@ -128,6 +137,15 @@ export const ChatHeader: React.FC<{
+ {__DEV__ && (
+ { logger.log(`[DevGrammar] header button tapped - opening modal (currently armed=${devActive})`); setDevOpen(true); }}
+ testID="chat-dev-grammar"
+ >
+
+
+ )}
setShowSettingsPanel(true)} testID="chat-settings-icon">
@@ -135,8 +153,10 @@ export const ChatHeader: React.FC<{
+ {__DEV__ && setDevOpen(false)} />}
-);
+ );
+};
export const EmptyChat: React.FC<{
styles: StylesType;
diff --git a/src/services/activeModelService/index.ts b/src/services/activeModelService/index.ts
index 4be27ad44..09a10ca1f 100644
--- a/src/services/activeModelService/index.ts
+++ b/src/services/activeModelService/index.ts
@@ -1,3 +1,7 @@
+/* eslint-disable max-lines -- 505 lines. This is the single owning service for
+ loading/unloading every model (text/LiteRT/image) through the residency lock;
+ splitting the one gateway would scatter the load lifecycle it exists to unify.
+ The recent overflow is the text-only (skip-mmproj) load option. */
// ActiveModelService — THE ONLY PLACE models should be loaded/unloaded from.
import { llmService } from '../llm';
import { liteRTService } from '../litert';
@@ -115,7 +119,7 @@ class ActiveModelService {
async loadTextModel(
modelId: string,
timeoutMs: number = 120000,
- opts?: { override?: boolean },
+ opts?: { override?: boolean; textOnly?: boolean },
): Promise {
// Fast path — model already loaded (no lock; just sync the store).
if (this.isTextModelCurrent(modelId)) {
@@ -134,7 +138,7 @@ class ActiveModelService {
private async doLoadTextModelLocked(
modelId: string,
timeoutMs: number,
- opts?: { override?: boolean },
+ opts?: { override?: boolean; textOnly?: boolean },
): Promise {
// Re-check after acquiring — a queued call may have loaded it already.
if (this.isTextModelCurrent(modelId)) {
@@ -151,7 +155,11 @@ class ActiveModelService {
}
// Use estimated runtime RAM (file size + overhead), not just file size,
// so the residency budget reflects the model's real memory footprint.
- const textSizeMB = Math.round((hardwareService.estimateModelRam(model) || 0) / (1024 * 1024));
+ // Text-only loads (transcription/insights) skip the vision mmproj clip, so
+ // size the budget on the gguf weights alone - don't reserve for a clip we
+ // won't load.
+ const ramModel = opts?.textOnly ? { fileSize: model.fileSize, mmProjFileSize: 0 } : model;
+ const textSizeMB = Math.round((hardwareService.estimateModelRam(ramModel) || 0) / (1024 * 1024));
// LiteRT weights + KV are dirty/accelerator memory → gated on REAL free RAM (mmap GGUF
// stays clean/physical-cap). Derived once so makeRoomFor and register agree.
const textIsDirty = model.engine === 'litert';
@@ -175,6 +183,7 @@ class ActiveModelService {
store,
timeoutMs,
override: !!opts?.override || modelResidencyManager.hasSessionOverride(modelId),
+ textOnly: !!opts?.textOnly,
loadedTextModelId: this.loadedTextModelId,
onLoaded: id => {
this.loadedTextModelId = id;
diff --git a/src/services/activeModelService/loaders.ts b/src/services/activeModelService/loaders.ts
index 6516b0eeb..5e5f50d82 100644
--- a/src/services/activeModelService/loaders.ts
+++ b/src/services/activeModelService/loaders.ts
@@ -85,6 +85,9 @@ export interface TextLoadContext {
/** User forced this load ("Load Anyway"/continue) — skip the conservative native
* memory gate so the loader's own fallbacks try instead of a hard block. */
override?: boolean;
+ /** Text-only load (transcription/insights) — do NOT load the vision mmproj clip,
+ * saving its RAM. The model's stored mmproj link is preserved for later vision use. */
+ textOnly?: boolean;
onLoaded: (modelId: string) => void;
onError: () => void;
onFinally: () => void;
@@ -192,7 +195,8 @@ export async function doLoadTextModel(ctx: TextLoadContext): Promise {
ctx.onError(); // resets loadedTextModelId to null before reassignment
}
- const mmProjPath = await resolveMmProjPath(ctx.model, ctx.modelId);
+ // Text-only load (transcription/insights): skip the vision mmproj clip entirely.
+ const mmProjPath = ctx.textOnly ? undefined : await resolveMmProjPath(ctx.model, ctx.modelId);
let timeoutId: ReturnType | null = null;
const timeoutPromise = new Promise((_, reject) => {
@@ -222,7 +226,9 @@ export async function doLoadTextModel(ctx: TextLoadContext): Promise {
// (incompatible file), clear it so the eye icon reappears for repair.
// Only applies when the link was already persisted before this load attempt — not
// when resolveMmProjPath just discovered the file via directory scan.
- if (ctx.model.mmProjPath && !multimodalSupport?.vision) {
+ // (Skip when textOnly: we deliberately didn't load the clip, so its absence is
+ // expected and must NOT be mistaken for an incompatible file to clear.)
+ if (!ctx.textOnly && ctx.model.mmProjPath && !multimodalSupport?.vision) {
await modelManager.clearMmProjLink(ctx.modelId);
}
diff --git a/src/services/devInference.ts b/src/services/devInference.ts
new file mode 100644
index 000000000..4bd76cf72
--- /dev/null
+++ b/src/services/devInference.ts
@@ -0,0 +1,89 @@
+import { useDevInferenceStore } from '../stores/devInferenceStore';
+import logger from '../utils/logger';
+
+/**
+ * DEV-ONLY grammar test harness (see docs/plans/chat-grammar-test-harness-plan.md).
+ *
+ * When the dev inference override is enabled, mutate an in-flight llama.rn
+ * `completionParams` object to apply a pasted GBNF grammar, a fixed temperature,
+ * and/or an assistant prefill - and drop tools, since a custom grammar can't
+ * coexist with the tool-calling grammar.
+ *
+ * No-op unless explicitly enabled from the __DEV__ grammar modal, so it has zero
+ * effect on normal / production chat.
+ *
+ * @returns true if a custom grammar was applied, so the caller can fall back to
+ * an ungrammared retry if llama.rn rejects an invalid GBNF.
+ */
+/**
+ * Cheap sanity check so an obviously-malformed grammar never reaches native
+ * (a pathological GBNF can hard-crash llama.cpp below the JS layer, which a
+ * try/catch can't recover). A valid GBNF must define a `root` rule with `::=`.
+ * Returns an error string if the grammar looks invalid, else null.
+ */
+function grammarLooksInvalid(grammar: string): string | null {
+ const g = grammar.trim();
+ if (!g.includes('::=')) return 'no rule definition (missing "::=")';
+ if (!/(^|\n)\s*root\s*::=/.test(g)) return 'no "root" rule';
+ return null;
+}
+
+export function applyDevGrammarOverrides(params: Record): boolean {
+ const dev = useDevInferenceStore.getState();
+ if (!dev.enabled) return false;
+
+ const toolCount = Array.isArray(params.tools) ? params.tools.length : 0;
+ let grammarApplied = false;
+ const hasGrammar = !!(dev.grammar && dev.grammar.trim().length > 0);
+ if (hasGrammar) {
+ const invalid = grammarLooksInvalid(dev.grammar);
+ if (invalid) {
+ // Don't hand a broken grammar to native - record it and run this turn
+ // normally so chat never crashes.
+ useDevInferenceStore.getState().setLastError(`Invalid grammar: ${invalid}`);
+ logger.warn(`[DevGrammar] grammar rejected before native (${invalid}) - running turn normally`);
+ } else {
+ params.grammar = dev.grammar;
+ // A pasted grammar and the tool-calling grammar are mutually exclusive, so
+ // tools are off for any turn that carries a custom grammar.
+ delete params.tools;
+ delete params.tool_choice;
+ grammarApplied = true;
+ }
+ } else {
+ // Enabled but nothing pasted - the most common "why isn't it working" case.
+ logger.warn('[DevGrammar] override ENABLED but grammar is empty - this turn runs normally');
+ }
+ if (typeof dev.temperature === 'number' && !Number.isNaN(dev.temperature)) {
+ params.temperature = dev.temperature;
+ }
+ if (dev.assistantPrefix.length > 0 && Array.isArray(params.messages)) {
+ // Prefill: a trailing partial assistant turn the model continues from.
+ params.messages = [...params.messages, { role: 'assistant', content: dev.assistantPrefix }];
+ }
+ // Hard output cap (words -> tokens, ~1.5 tokens/word incl. formatting). Also
+ // the safety valve against a grammar that never lets the model stop.
+ if (typeof dev.maxWords === 'number' && dev.maxWords > 0) {
+ params.n_predict = Math.ceil(dev.maxWords * 1.5);
+ }
+ logger.log(
+ `[DevGrammar] APPLIED grammar=${grammarApplied} grammarLen=${grammarApplied ? dev.grammar.length : 0} ` +
+ `temp=${params.temperature} prefill=${dev.assistantPrefix ? JSON.stringify(dev.assistantPrefix) : 'none'} ` +
+ `maxWords=${dev.maxWords ?? 'none'} n_predict=${params.n_predict} toolsStripped=${grammarApplied ? toolCount : 0}`,
+ );
+ // A fresh run clears any stale error, unless we just set one above.
+ if (dev.lastError && grammarApplied) useDevInferenceStore.getState().setLastError(undefined);
+ return grammarApplied;
+}
+
+/**
+ * Record a completion failure that happened after a dev grammar was applied and
+ * strip the grammar from `params`, so the caller can retry ungrammared. A bad
+ * GBNF paste should surface in the modal, never brick chat.
+ */
+export function noteDevGrammarError(params: Record, error: unknown): void {
+ const msg = error instanceof Error ? error.message : String(error);
+ useDevInferenceStore.getState().setLastError(msg);
+ delete params.grammar;
+ logger.warn(`[DevGrammar] completion failed, retrying ungrammared: ${msg}`);
+}
diff --git a/src/services/index.ts b/src/services/index.ts
index 15b0305ff..7784c5644 100644
--- a/src/services/index.ts
+++ b/src/services/index.ts
@@ -7,7 +7,7 @@ export { intentClassifier, classifyToolsNeeded } from './intentClassifier';
export type { Intent } from './intentClassifier';
export { voiceService } from './voiceService';
export { authService } from './authService';
-export { whisperService, WHISPER_MODELS } from './whisperService';
+export { whisperService, WHISPER_MODELS, WhisperBusyError } from './whisperService';
// ttsService deprecated — logic absorbed into OuteTTSEngine (src/engine/tts/engines/outetts/).
export type { TranscriptionResult, TranscriptionCallback } from './whisperService';
export { backgroundDownloadService } from './backgroundDownloadService';
@@ -35,3 +35,9 @@ export type { LLMProvider, ProviderType, ProviderCapabilities, GenerationOptions
export { fetchWithTimeout, createStreamingRequest, imageToBase64DataUrl, testEndpoint, isPrivateNetworkEndpoint } from './httpClient';
// Remote Server Manager
export { remoteServerManager } from './remoteServerManager';
+// Text-model auto-load selection (memory-aware pick when none is resident)
+export { selectTextModelToLoad, fitsBudget } from './selectTextModel';
+// Residency manager - the single owner of the RAM budget + load gate. Callers
+// that pick a model to auto-load must budget against getBudgetMB() so the pick
+// and the load gate can never disagree (any memory-aware auto-load path).
+export { modelResidencyManager } from './modelResidency';
diff --git a/src/services/litert.ts b/src/services/litert.ts
index 0e3a94da1..006f61c06 100644
--- a/src/services/litert.ts
+++ b/src/services/litert.ts
@@ -13,6 +13,7 @@
import { NativeModules, NativeEventEmitter, EmitterSubscription } from 'react-native';
import logger from '../utils/logger';
import { summarizeSession, runCompaction } from './liteRTCompaction';
+import { useDevInferenceStore } from '../stores/devInferenceStore';
const TAG = '[LiteRTService]';
@@ -150,6 +151,20 @@ class LiteRTService {
const topP = samplerConfig?.topP ?? 0.95;
const toolsJson = tools && tools.length > 0 ? JSON.stringify(tools) : '';
const historyJson = history && history.length > 0 ? JSON.stringify(history) : '';
+ // DEV-only: arm/disarm an LLGuidance constraint before (re)creating the
+ // conversation, since it's a per-conversation flag natively. Guarded so a
+ // missing native method (older build) never blocks generation.
+ if (__DEV__ && typeof LiteRTModule?.setConstrainedDecoding === 'function') {
+ try {
+ const dev = useDevInferenceStore.getState();
+ const constraint = dev.litertConstraintString.trim();
+ const armed = dev.enabled && constraint.length > 0;
+ await LiteRTModule.setConstrainedDecoding(armed, dev.litertConstraintType, armed ? dev.litertConstraintString : '');
+ if (armed) logger.log(TAG, `[DevGrammar-LiteRT] armed constraint type=${dev.litertConstraintType} len=${constraint.length}`);
+ } catch (e) {
+ logger.warn(TAG, `[DevGrammar-LiteRT] setConstrainedDecoding failed: ${String(e)}`);
+ }
+ }
await LiteRTModule.resetConversation(systemPrompt, temperature, topK, topP, toolsJson, historyJson);
this.activeSystemPrompt = systemPrompt;
this.activeToolsJson = toolsJson;
@@ -501,6 +516,11 @@ class LiteRTService {
return this.loaded;
}
+ /** Configured context window (tokens) for the loaded LiteRT model. */
+ getContextTokens(): number {
+ return this.configuredMaxTokens;
+ }
+
isNPU(): boolean {
return this.activeBackend === 'npu';
}
diff --git a/src/services/llm.ts b/src/services/llm.ts
index 0d31e57ac..f79ee9633 100644
--- a/src/services/llm.ts
+++ b/src/services/llm.ts
@@ -1,3 +1,9 @@
+/* eslint-disable max-lines -- 541 lines. This is the core llama generation
+ service: prompt build, streaming, and tool-call orchestration in one cohesive
+ unit. The overflow is the union of two independently-valid changes landing in
+ the same merge (base's generation work + the streaming-accumulator reset for
+ the ungrammared grammar-retry); splitting it mid-reconcile would risk the
+ generation critical path for a cosmetic line count. Matches whisperService.ts. */
import { LlamaContext, RNLlamaOAICompatibleMessage } from 'llama.rn';
import { Platform } from 'react-native';
import RNFS from 'react-native-fs';
@@ -17,6 +23,7 @@ import { formatLlamaMessages, buildOAIMessages } from './llmMessages';
import { generateWithToolsImpl } from './llmToolGeneration';
import type { ToolCall } from './tools/types';
import type { MultimodalSupport, LLMPerformanceSettings, LLMPerformanceStats } from './llmTypes';
+import { applyDevGrammarOverrides, noteDevGrammarError } from './devInference';
import logger from '../utils/logger';
export type { MultimodalSupport, LLMPerformanceSettings, LLMPerformanceStats } from './llmTypes';
export type StreamToken = { content?: string; reasoningContent?: string };
@@ -301,8 +308,12 @@ class LLMService {
let firstTokenMs = 0, tokenCount = 0, firstReceived = false;
let fullContent = '', fullReasoningContent = '', streamedContentSoFar = '', streamedReasoningSoFar = '';
const completionParams = { messages: oaiMessages, ...buildCompletionParams(settings, { disableCtxShift: this.shouldDisableCtxShift() }), ...buildThinkingCompletionParams(this.isThinkingEnabled(), this.isGemma4Model()) };
+ // DEV-only: a pasted GBNF grammar / temp / prefill can override this turn.
+ // No-op unless enabled from the __DEV__ grammar modal.
+ const devGrammarApplied = applyDevGrammarOverrides(completionParams);
+ if (devGrammarApplied) logger.log(`[DevGrammar] reached native completion (no-tools path); grammar in params=${!!(completionParams as any).grammar}`);
logger.log(`[LLM][THINKING] thinkingSupported=${this.thinkingSupported}, thinkingEnabled=${useAppStore.getState().settings.thinkingEnabled}, isThinkingEnabled=${this.isThinkingEnabled()}, enable_thinking=${(completionParams as any).enable_thinking}, reasoning_format=${(completionParams as any).reasoning_format}`);
- const completionResult = await safeCompletion(ctx, () => ctx.completion(completionParams, (data: any) => {
+ const onCompletionData = (data: any) => {
if (!this.isGenerating || !data.token) return;
if (!firstReceived) { firstReceived = true; firstTokenMs = Date.now() - startTime; logger.log(`[LLM][THINKING] First token raw data — token: ${JSON.stringify(data.token)}, content: ${JSON.stringify(data.content)}, reasoning_content: ${JSON.stringify(data.reasoning_content)}`); }
tokenCount++;
@@ -314,7 +325,20 @@ class LLMService {
if (content) fullContent += content;
if (reasoningContent) fullReasoningContent += reasoningContent;
onStream?.({ reasoningContent, content });
- }), 'generateResponse');
+ };
+ let completionResult;
+ try {
+ completionResult = await safeCompletion(ctx, () => ctx.completion(completionParams, onCompletionData), 'generateResponse');
+ } catch (e) {
+ // A bad dev grammar must never brick chat: record it and retry ungrammared.
+ if (!devGrammarApplied) throw e;
+ noteDevGrammarError(completionParams, e);
+ // Reset streaming state so the ungrammared retry doesn't append to / re-emit
+ // the failed attempt's partial output (would duplicate/garble the result).
+ fullContent = ''; fullReasoningContent = '';
+ streamedContentSoFar = ''; streamedReasoningSoFar = '';
+ completionResult = await safeCompletion(ctx, () => ctx.completion(completionParams, onCompletionData), 'generateResponse-fallback');
+ }
const cr = completionResult as any;
this.performanceStats = recordGenerationStats(startTime, firstTokenMs, tokenCount);
if (completionResult?.context_full) { logger.log('[LLM] Context full detected — signalling for compaction'); throw new Error('Context is full'); }
@@ -395,10 +419,15 @@ class LLMService {
* user-facing). Pass onToken to stream the output as it is produced; the
* delta is the newly generated token text.
*/
- async generateWithMaxTokens(messages: Message[], maxTokens: number, onToken?: (delta: string) => void): Promise {
+ async generateWithMaxTokens(
+ messages: Message[],
+ maxTokens: number,
+ opts?: { onToken?: (delta: string) => void; grammar?: string; repeatPenalty?: number },
+ ): Promise {
if (!this.context) throw new Error('No model loaded');
if (this.isGenerating) throw new Error('Generation already in progress');
this.isGenerating = true;
+ const onToken = opts?.onToken;
const oaiMessages = this.convertToOAIMessages(messages);
const { settings } = useAppStore.getState();
let fullResponse = '';
@@ -407,11 +436,31 @@ class LLMService {
// model to "think" - reasoning wastes the token budget, is slow + hot, and
// leaks into the output. Force thinking OFF (for models that gate it via the
// thinking channel; prose chain-of-thought is additionally curbed by prompts).
- const params = { messages: oaiMessages, ...buildCompletionParams(settings, { disableCtxShift: this.shouldDisableCtxShift() }), ...buildThinkingCompletionParams(false, this.isGemma4Model()), n_predict: maxTokens };
- const completionWork = safeCompletion(ctx, () => ctx.completion(
- params,
+ const params: Record = { messages: oaiMessages, ...buildCompletionParams(settings, { disableCtxShift: this.shouldDisableCtxShift() }), ...buildThinkingCompletionParams(false, this.isGemma4Model()), n_predict: maxTokens };
+ // Optional GBNF grammar (llama.cpp constrained decoding) so callers like the
+ // insights pass can force a fixed output shape. A bad grammar must never
+ // brick generation, so retry once without it on failure.
+ if (opts?.grammar) params.grammar = opts.grammar;
+ // Stronger repetition penalty for callers (insights) prone to small-model
+ // loops; overrides the default penalty_repeat from buildCompletionParams.
+ if (opts?.repeatPenalty != null) params.penalty_repeat = opts.repeatPenalty;
+ const run = () => ctx.completion(
+ params as Parameters[0],
(data) => { if (this.isGenerating && data.token) { fullResponse += data.token; onToken?.(data.token); } },
- ), 'generateWithMaxTokens');
+ );
+ const completionWork = (async () => {
+ try {
+ return await safeCompletion(ctx, run, 'generateWithMaxTokens');
+ } catch (e) {
+ if (params.grammar) {
+ logger.warn(`[LLM] grammared generation failed, retrying without grammar: ${String(e)}`);
+ fullResponse = '';
+ delete params.grammar;
+ return await safeCompletion(ctx, run, 'generateWithMaxTokens-fallback');
+ }
+ throw e;
+ }
+ })();
this.activeCompletionPromise = completionWork.then(() => { }, () => { });
try { await completionWork; return fullResponse.trim(); } finally { this.isGenerating = false; this.activeCompletionPromise = null; }
}
diff --git a/src/services/llmToolGeneration.ts b/src/services/llmToolGeneration.ts
index a057f2013..b6f2c64cf 100644
--- a/src/services/llmToolGeneration.ts
+++ b/src/services/llmToolGeneration.ts
@@ -8,6 +8,7 @@ import type { Message } from '../types';
import type { ToolCall } from './tools/types';
import { recordGenerationStats, buildCompletionParams, buildThinkingCompletionParams, safeCompletion } from './llmHelpers';
import type { StreamToken } from './llm';
+import { applyDevGrammarOverrides, noteDevGrammarError } from './devInference';
import logger from '../utils/logger';
type ToolStreamCallback = (data: StreamToken) => void;
@@ -28,6 +29,12 @@ class ToolCallTokenFilter {
return this.flush();
}
+ /** Clear buffered state (used when a generation is retried from scratch). */
+ reset(): void {
+ this.inBlock = false;
+ this.buffer = '';
+ }
+
private flush(): string {
const openTag = '<|tool_call>';
const closeTag = '';
@@ -130,9 +137,13 @@ export async function generateWithToolsImpl(
tool_choice: 'auto',
...buildThinkingCompletionParams(deps.isThinkingEnabled, deps.isGemma4Model),
};
+ // DEV-only: a pasted GBNF grammar / temp / prefill can override this turn
+ // (and strips tools). No-op unless enabled from the __DEV__ grammar modal.
+ const devGrammarApplied = applyDevGrammarOverrides(completionParams);
+ if (devGrammarApplied) logger.log(`[DevGrammar] reached native completion (tools path); grammar in params=${!!(completionParams as any).grammar}`);
logger.log('[LLM-Tools] === INPUT ===');
logger.log(JSON.stringify(completionParams, null, 2));
- const completionResult: any = await safeCompletion(deps.context, () => deps.context.completion(completionParams as any, (data: any) => {
+ const onCompletionData = (data: any) => {
if (!generating) return;
if (data.tool_calls) {
for (const tc of data.tool_calls) {
@@ -145,7 +156,21 @@ export async function generateWithToolsImpl(
const visibleToken = toolCallFilter ? toolCallFilter.process(data.token) : data.token;
fullResponse += visibleToken;
if (visibleToken) options.onStream?.({ content: visibleToken });
- }), 'generateWithTools');
+ };
+ let completionResult: any;
+ try {
+ completionResult = await safeCompletion(deps.context, () => deps.context.completion(completionParams as any, onCompletionData), 'generateWithTools');
+ } catch (e) {
+ // A bad dev grammar must never brick chat: record it and retry ungrammared.
+ if (!devGrammarApplied) throw e;
+ noteDevGrammarError(completionParams, e);
+ // Reset streaming state so the ungrammared retry doesn't append to / re-emit
+ // the failed attempt's partial output or double-count its tool calls.
+ fullResponse = '';
+ collectedToolCalls.length = 0;
+ toolCallFilter?.reset();
+ completionResult = await safeCompletion(deps.context, () => deps.context.completion(completionParams as any, onCompletionData), 'generateWithTools-fallback');
+ }
logger.log('[LLM-Tools] === OUTPUT ===');
logger.log(JSON.stringify(completionResult, null, 2));
diff --git a/src/services/selectTextModel.ts b/src/services/selectTextModel.ts
new file mode 100644
index 000000000..5054a98de
--- /dev/null
+++ b/src/services/selectTextModel.ts
@@ -0,0 +1,44 @@
+import type { DownloadedModel } from '../types';
+
+/** Does an estimated footprint (MB) fit the budget (MB)? */
+export function fitsBudget(footprintMB: number, budgetMB: number): boolean {
+ return footprintMB <= budgetMB;
+}
+
+/**
+ * Pick which downloaded text model to AUTO-LOAD when none is resident.
+ *
+ * Only for the no-model auto-load path. If a model is already loaded (or a
+ * remote is active) the caller uses that as-is and never calls this.
+ *
+ * `footprintMB(model)` is the estimated resident RAM in MB. Pass the canonical
+ * estimator (`hardwareService.estimateModelRam`) so weights + the vision mmproj
+ * clip + runtime overhead are all counted the same way the loader budgets them
+ * - do not re-derive footprint here.
+ *
+ * Rule, in order:
+ * 1. the user's active model, IF it fits the budget (respect an explicit choice);
+ * 2. otherwise the LARGEST that fits (best quality the device can run);
+ * 3. otherwise the SMALLEST (run something rather than pick an OOM).
+ *
+ * Returns null only when there are no models to choose from.
+ */
+export function selectTextModelToLoad(
+ models: DownloadedModel[],
+ budgetMB: number,
+ opts: { activeId: string | null; footprintMB: (m: DownloadedModel) => number },
+): DownloadedModel | null {
+ const { activeId, footprintMB } = opts;
+ if (models.length === 0) return null;
+
+ const active = activeId ? models.find((m) => m.id === activeId) ?? null : null;
+ if (active && fitsBudget(footprintMB(active), budgetMB)) return active;
+
+ // Largest footprint first, so the first fitting one is the biggest that fits.
+ const bySizeDesc = [...models].sort((a, b) => footprintMB(b) - footprintMB(a));
+ const largestFit = bySizeDesc.find((m) => fitsBudget(footprintMB(m), budgetMB));
+ if (largestFit) return largestFit;
+
+ // Nothing fits — the smallest is the least-bad option (better than an OOM pick).
+ return bySizeDesc[bySizeDesc.length - 1];
+}
diff --git a/src/services/transcriptSummarizer.ts b/src/services/transcriptSummarizer.ts
index ad877d98a..ae3f111a3 100644
--- a/src/services/transcriptSummarizer.ts
+++ b/src/services/transcriptSummarizer.ts
@@ -44,24 +44,20 @@ const CHUNK_SUMMARY_TOKENS = 256;
/** Tokens reserved for the final combined summary output. */
const FINAL_SUMMARY_TOKENS = 512;
-/** Estimated overhead for the summarizer instruction + chat template. */
-const INSTRUCTION_OVERHEAD_TOKENS = 160;
-
-/** Safety margin so we never sit exactly at the context edge. */
-const SAFETY_MARGIN_TOKENS = 128;
-
/** Hard cap on reduce rounds, so a pathological input can't loop forever. */
const MAX_REDUCE_ROUNDS = 4;
-// Cap each MAP chunk well below the full context window. On CPU-only low-RAM
-// devices, prefill (reading the chunk in) dominates wall-clock and there is no
-// token callback during it, so a chunk that fills the whole 4096 context takes
-// ~2 min before the first token streams. ~1500 input tokens (~6000 chars, a
-// coherent few-minutes-of-speech slice) prefills in well under a minute, so each
-// part starts streaming quickly. Smaller = sooner first token but more chunks;
-// this is a deliberate balance, not the minimum. The reduce/combine passes still
-// use the full context budget.
-const MAP_INPUT_TOKEN_TARGET = 1500;
+// Fraction of the ACTIVE backend's context window we spend on input per chunk.
+// The rest is headroom for the summary output + the instruction/template +
+// safety, and keeps small models off the context edge (where they degrade).
+// Sized off the real context (see resolveContextTokens) so a big remote/flagship
+// window one-shots a long transcript while a 2k on-device model stays small.
+const INPUT_CONTEXT_FRACTION = 0.6;
+
+// Assumed context when a remote provider doesn't report its own (remote servers
+// are typically large; better to under-chunk a big window than over-chunk it).
+const REMOTE_DEFAULT_CONTEXT_TOKENS = 8192;
+const LITERT_DEFAULT_CONTEXT_TOKENS = 4096;
// The prompts forbid any reasoning/preamble up front: some on-device models
// (e.g. Gemma-style instruct models) otherwise spend the whole token budget
@@ -92,10 +88,37 @@ function isLiteRTActive(): boolean {
);
}
-/** Is a remote provider serving generation (no on-device native context)? */
+/**
+ * Is a remote provider available to serve summaries? Summaries PREFER remote
+ * whenever one is active, even if a local model is also loaded - offloading the
+ * generation off-device saves the phone's battery/RAM (chat generation keeps its
+ * own local-first policy; this only affects the summarizer). Deliberately does
+ * NOT check `llmService.isModelLoaded()`.
+ */
function isRemoteActive(): boolean {
const activeServerId = useRemoteServerStore.getState().activeServerId;
- return !!activeServerId && providerRegistry.hasProvider(activeServerId) && !llmService.isModelLoaded();
+ return !!activeServerId && providerRegistry.hasProvider(activeServerId);
+}
+
+/**
+ * The ACTIVE backend's real context window (tokens) + a label for logs. Chunk
+ * sizing is derived from this, so it adapts per backend instead of assuming a
+ * fixed on-device 2k. Remote uses the provider's reported context when known,
+ * else a large default; LiteRT uses its configured max; local uses the loaded
+ * model's setting.
+ */
+function resolveContextTokens(): { tokens: number; source: string } {
+ // Remote is preferred for summaries, so size chunks off its window first.
+ if (isRemoteActive()) {
+ const id = useRemoteServerStore.getState().activeServerId;
+ const provider = id ? providerRegistry.getProvider(id) : undefined;
+ const reported = provider?.capabilities?.maxContextLength;
+ return { tokens: reported && reported > 0 ? reported : REMOTE_DEFAULT_CONTEXT_TOKENS, source: 'remote' };
+ }
+ if (isLiteRTActive()) {
+ return { tokens: liteRTService.getContextTokens() || LITERT_DEFAULT_CONTEXT_TOKENS, source: 'litert' };
+ }
+ return { tokens: llmService.getPerformanceSettings().contextLength || 2048, source: 'local' };
}
/**
@@ -108,7 +131,7 @@ function isRemoteActive(): boolean {
async function generateSummaryText(
systemPrompt: string,
userText: string,
- opts: { maxTokens: number; onToken?: (delta: string) => void },
+ opts: { maxTokens: number; onToken?: (delta: string) => void; grammar?: string; repeatPenalty?: number },
): Promise {
const { maxTokens, onToken } = opts;
const messages: Message[] = [
@@ -116,19 +139,13 @@ async function generateSummaryText(
{ id: 'summarize-input', role: 'user', content: userText, timestamp: 0 },
];
- // LiteRT: run on a throwaway, tools-free conversation so it never pollutes a
- // real chat's KV/history (mirrors the LiteRT tool-selection pass).
- if (isLiteRTActive()) {
- await liteRTService.prepareConversation('__summarize__', systemPrompt, {
- tools: [],
- samplerConfig: { temperature: 0.3 },
- });
- return liteRTService.generateRaw(userText, undefined, { onToken });
- }
-
- // Remote provider: OpenAI-compatible streaming completion, tools off.
- const activeServerId = useRemoteServerStore.getState().activeServerId;
- if (activeServerId && providerRegistry.hasProvider(activeServerId) && !llmService.isModelLoaded()) {
+ // Remote provider (PREFERRED for summaries: offload off-device even when a
+ // local model is loaded). OpenAI-compatible streaming completion, tools off.
+ // If it fails BEFORE any token streams (e.g. the server left the LAN mid-use),
+ // fall through to on-device so a vanished server never turns into a hard error.
+ // A failure AFTER tokens have streamed is surfaced (we don't double-write).
+ if (isRemoteActive()) {
+ const activeServerId = useRemoteServerStore.getState().activeServerId as string;
const provider = providerRegistry.getProvider(activeServerId);
if (provider) {
const { settings } = useAppStore.getState();
@@ -139,22 +156,42 @@ async function generateSummaryText(
tools: [],
enableThinking: false,
};
- return new Promise((resolve, reject) => {
- let content = '';
- provider
- .generate(messages, options, {
- onToken: (t: string) => { content += t; onToken?.(t); },
- onReasoning: () => { /* summaries ignore reasoning output */ },
- onComplete: (result) => resolve(result.content || content),
- onError: (e: Error) => reject(e),
- })
- .catch(reject);
- });
+ let emittedAny = false;
+ try {
+ return await new Promise((resolve, reject) => {
+ let content = '';
+ provider
+ .generate(messages, options, {
+ onToken: (t: string) => { content += t; emittedAny = true; onToken?.(t); },
+ onReasoning: () => { /* summaries ignore reasoning output */ },
+ onComplete: (result) => resolve(result.content || content),
+ onError: (e: Error) => reject(e),
+ })
+ .catch(reject);
+ });
+ } catch (e) {
+ if (emittedAny) throw e;
+ logger.warn(
+ `[TranscriptSummarizer] remote summary failed before streaming, falling back to on-device: ${String(e)}`,
+ );
+ // fall through to LiteRT / local
+ }
}
}
- // Local llama.rn (default).
- return llmService.generateWithMaxTokens(messages, maxTokens, onToken);
+ // LiteRT: run on a throwaway, tools-free conversation so it never pollutes a
+ // real chat's KV/history (mirrors the LiteRT tool-selection pass).
+ if (isLiteRTActive()) {
+ await liteRTService.prepareConversation('__summarize__', systemPrompt, {
+ tools: [],
+ samplerConfig: { temperature: 0.3 },
+ });
+ return liteRTService.generateRaw(userText, undefined, { onToken });
+ }
+
+ // Local llama.rn (default). Grammar (GBNF) is applied here when the caller
+ // passes one; LiteRT/remote ignore it for now (constrained decoding TBD).
+ return llmService.generateWithMaxTokens(messages, maxTokens, { onToken, grammar: opts.grammar, repeatPenalty: opts.repeatPenalty });
}
class TranscriptSummarizerService {
@@ -165,6 +202,33 @@ class TranscriptSummarizerService {
return this._isSummarizing;
}
+ /**
+ * Abort the in-flight generation NOW (not just between chunks). A cooperative
+ * loop cancel only skips the next unit; the current native completion keeps
+ * running and holds the single-context lock, so callers that "Stop" still see
+ * "busy" until it finishes. This interrupts the current completion via
+ * llmService.stopGeneration, which lets the awaited summarize() unwind and
+ * clear _isSummarizing. Safe to call when idle (no-op).
+ */
+ async abort(): Promise {
+ logger.log(
+ `[TranscriptSummarizer] abort requested (isSummarizing=${this._isSummarizing}, ` +
+ `llmGenerating=${llmService.isCurrentlyGenerating()})`,
+ );
+ try {
+ await llmService.stopGeneration();
+ // Summaries PREFER a remote provider (and may run on one even with a local
+ // model loaded), so a local-only stop wouldn't interrupt a remote in-flight
+ // completion. Stop the active remote provider too (it aborts its stream).
+ if (isRemoteActive()) {
+ const activeServerId = useRemoteServerStore.getState().activeServerId as string;
+ await providerRegistry.getProvider(activeServerId)?.stopGeneration?.();
+ }
+ } finally {
+ this._isSummarizing = false;
+ }
+ }
+
/** True if any backend (local llama, LiteRT, or a remote provider) can summarize now. */
isBackendReady(): boolean {
return llmService.isModelLoaded() || isLiteRTActive() || isRemoteActive();
@@ -199,33 +263,40 @@ class TranscriptSummarizerService {
// (e.g. a bulleted, section-headed summary) pass their own here.
systemPrompt?: string;
combinePrompt?: string;
+ // Stronger repetition penalty (insights) to stop small-model loops.
+ repeatPenalty?: number;
+ // Optional GBNF grammar to force the final output shape (llama.rn only).
+ // Applied only on the final single-pass / combine pass so intermediate
+ // map/reduce partials stay free-form. Ignored by LiteRT/remote for now.
+ grammar?: string;
},
): Promise {
const onProgress = opts?.onProgress;
const onToken = opts?.onToken;
+ const grammar = opts?.grammar;
+ const repeatPenalty = opts?.repeatPenalty;
const mapPrompt = opts?.systemPrompt ?? SUMMARIZER_SYSTEM_PROMPT;
const combinePrompt = opts?.combinePrompt ?? COMBINE_SYSTEM_PROMPT;
this._isSummarizing = true;
try {
await llmService.clearKVCache(true);
- const ctxLength = llmService.getPerformanceSettings().contextLength || 2048;
- const inputBudgetTokens = Math.max(
- 256,
- ctxLength - CHUNK_SUMMARY_TOKENS - INSTRUCTION_OVERHEAD_TOKENS - SAFETY_MARGIN_TOKENS,
- );
+ // Size chunks dynamically off the ACTIVE backend's real context (local
+ // model setting / LiteRT / remote server), not a fixed number - so a big
+ // remote/flagship context one-shots a long transcript while a 2k on-device
+ // model stays conservative. Use a fraction of the window so there's always
+ // headroom for the output + instructions + safety (no fixed cap).
+ const ctx = resolveContextTokens();
+ const inputBudgetTokens = Math.max(512, Math.round(ctx.tokens * INPUT_CONTEXT_FRACTION));
const chunkCharBudget = inputBudgetTokens * CHARS_PER_TOKEN;
- // Map split is capped smaller than the full budget so each part prefills
- // fast and streams sooner; reduce/combine still use the full chunkCharBudget.
- const mapCharBudget = Math.min(chunkCharBudget, MAP_INPUT_TOKEN_TARGET * CHARS_PER_TOKEN);
- const chunks = splitIntoChunks(text.trim(), mapCharBudget);
- logger.log(`[TranscriptSummarizer] ${text.length} chars, ctx=${ctxLength}, mapBudget=${mapCharBudget} chars, chunks=${chunks.length}`);
+ const chunks = splitIntoChunks(text.trim(), chunkCharBudget);
+ logger.log(`[TranscriptSummarizer] ${text.length} chars, backend=${ctx.source} ctx=${ctx.tokens}, budget=${inputBudgetTokens}tok (${Math.round(INPUT_CONTEXT_FRACTION * 100)}%), chunks=${chunks.length}`);
// Small enough to summarize in one pass.
if (chunks.length <= 1) {
this.emit({ phase: 'mapping', current: 1, total: 1 }, onProgress);
- const summary = await this.summarizeOne(mapPrompt, chunks[0] ?? text, { maxTokens: FINAL_SUMMARY_TOKENS, onToken });
+ const summary = await this.summarizeOne(mapPrompt, chunks[0] ?? text, { maxTokens: FINAL_SUMMARY_TOKENS, onToken, grammar, repeatPenalty });
this.emit({ phase: 'done' }, onProgress);
return summary.trim();
}
@@ -260,7 +331,7 @@ class TranscriptSummarizerService {
// Final combine pass into one coherent summary. Streamed to the caller.
this.emit({ phase: 'combining' }, onProgress);
await llmService.clearKVCache(true);
- const finalSummary = await this.summarizeOne(combinePrompt, combined, { maxTokens: FINAL_SUMMARY_TOKENS, onToken });
+ const finalSummary = await this.summarizeOne(combinePrompt, combined, { maxTokens: FINAL_SUMMARY_TOKENS, onToken, grammar, repeatPenalty });
this.emit({ phase: 'done' }, onProgress);
return finalSummary.trim();
@@ -276,10 +347,10 @@ class TranscriptSummarizerService {
private async summarizeOne(
systemPrompt: string,
input: string,
- opts: { maxTokens: number; onToken?: (delta: string) => void },
+ opts: { maxTokens: number; onToken?: (delta: string) => void; grammar?: string; repeatPenalty?: number },
): Promise {
// Dispatches to the active backend (local llama.rn / LiteRT / remote).
- const out = await generateSummaryText(systemPrompt, input, { maxTokens: opts.maxTokens, onToken: opts.onToken });
+ const out = await generateSummaryText(systemPrompt, input, { maxTokens: opts.maxTokens, onToken: opts.onToken, grammar: opts.grammar, repeatPenalty: opts.repeatPenalty });
// Backstop for tag-based reasoning that slipped through (...).
return stripControlTokens(out);
}
diff --git a/src/services/whisperModels.ts b/src/services/whisperModels.ts
index e6aaf3ec0..830b49a5d 100644
--- a/src/services/whisperModels.ts
+++ b/src/services/whisperModels.ts
@@ -4,22 +4,44 @@
// whisperService.transcribeFile.
const GGML_BASE = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main';
-export const WHISPER_MODELS = [
+// CoreML encoder (iOS only): ggerganov ships a per-model `-encoder.mlmodelc.zip`
+// alongside each ggml model. Downloaded + unzipped next to the .bin, it lets
+// whisper.cpp run the encoder on the Apple Neural Engine (~2-3x faster encode,
+// frees the CPU). Path convention: `ggml-.bin` -> `ggml--encoder.mlmodelc`
+// (the zip's own top-level dir already matches). Not published for the akashmjn
+// tdrz checkpoint, so that entry has none.
+const coreML = (id: string) => `${GGML_BASE}/ggml-${id}-encoder.mlmodelc.zip`;
+
+export interface WhisperModel {
+ id: string;
+ name: string;
+ size: number; // MB, approximate
+ lang: string; // 'en' | 'multi'
+ url: string;
+ description: string;
+ coreMLUrl?: string; // iOS CoreML encoder zip, when published for this model
+}
+
+export const WHISPER_MODELS: WhisperModel[] = [
// ── English-only ──────────────────────────────────────────────────────────
- { id: 'tiny.en', name: 'Tiny', size: 75, lang: 'en', url: `${GGML_BASE}/ggml-tiny.en.bin`, description: 'Fastest, English only' },
- { id: 'base.en', name: 'Base', size: 142, lang: 'en', url: `${GGML_BASE}/ggml-base.en.bin`, description: 'Better accuracy, English only' },
- { id: 'small.en', name: 'Small', size: 466, lang: 'en', url: `${GGML_BASE}/ggml-small.en.bin`, description: 'High accuracy, English only' },
+ { id: 'tiny.en', name: 'Tiny', size: 75, lang: 'en', url: `${GGML_BASE}/ggml-tiny.en.bin`, coreMLUrl: coreML('tiny.en'), description: 'Fastest, English only' },
+ { id: 'base.en', name: 'Base', size: 142, lang: 'en', url: `${GGML_BASE}/ggml-base.en.bin`, coreMLUrl: coreML('base.en'), description: 'Better accuracy, English only' },
+ { id: 'small.en', name: 'Small', size: 466, lang: 'en', url: `${GGML_BASE}/ggml-small.en.bin`, coreMLUrl: coreML('small.en'), description: 'High accuracy, English only' },
// tinydiarize build of small.en: marks speaker-turn boundaries ([SPEAKER_TURN])
// when transcribed with diarization on. English only; required for the
// diarization toggle to produce anything (other models ignore tdrz).
// The only tdrz checkpoint that exists (akashmjn's repo, not ggerganov's). ~465 MB f16; no smaller/quantized variant is published.
- { id: 'small.en-tdrz', name: 'Small (speaker turns)', size: 465, lang: 'en', url: 'https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin', description: 'Marks who-spoke turn boundaries, English only (experimental)' },
- { id: 'medium.en', name: 'Medium', size: 1500, lang: 'en', url: `${GGML_BASE}/ggml-medium.en.bin`, description: 'Near human-level, English only, ~2 GB RAM' },
+ // CoreML: tinydiarize only fine-tunes the DECODER (adds the turn token), so the
+ // ENCODER is the standard small.en encoder - we reuse ggerganov's small.en
+ // CoreML encoder for the ANE. The download flow renames it to the tdrz path.
+ // whisper.cpp's ALLOW_FALLBACK drops to CPU + logs if it's ever incompatible.
+ { id: 'small.en-tdrz', name: 'Small (speaker turns)', size: 465, lang: 'en', url: 'https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin', coreMLUrl: coreML('small.en'), description: 'Marks who-spoke turn boundaries, English only (experimental)' },
+ { id: 'medium.en', name: 'Medium', size: 1500, lang: 'en', url: `${GGML_BASE}/ggml-medium.en.bin`, coreMLUrl: coreML('medium.en'), description: 'Near human-level, English only, ~2 GB RAM' },
// ── Multilingual ──────────────────────────────────────────────────────────
- { id: 'tiny', name: 'Tiny', size: 75, lang: 'multi', url: `${GGML_BASE}/ggml-tiny.bin`, description: 'Fastest, 99 languages' },
- { id: 'base', name: 'Base', size: 142, lang: 'multi', url: `${GGML_BASE}/ggml-base.bin`, description: 'Better accuracy, 99 languages' },
- { id: 'small', name: 'Small', size: 466, lang: 'multi', url: `${GGML_BASE}/ggml-small.bin`, description: 'High accuracy, 99 languages' },
- { id: 'medium', name: 'Medium', size: 1500, lang: 'multi', url: `${GGML_BASE}/ggml-medium.bin`, description: 'Near human-level, 99 languages, ~2 GB RAM' },
- { id: 'large-v3-turbo', name: 'Large v3 Turbo', size: 809, lang: 'multi', url: `${GGML_BASE}/ggml-large-v3-turbo.bin`, description: 'Fast + accurate, distilled large, 99 languages' },
- { id: 'large-v3', name: 'Large v3', size: 1550, lang: 'multi', url: `${GGML_BASE}/ggml-large-v3.bin`, description: 'Best quality, 99 languages, ~3 GB RAM' },
+ { id: 'tiny', name: 'Tiny', size: 75, lang: 'multi', url: `${GGML_BASE}/ggml-tiny.bin`, coreMLUrl: coreML('tiny'), description: 'Fastest, 99 languages' },
+ { id: 'base', name: 'Base', size: 142, lang: 'multi', url: `${GGML_BASE}/ggml-base.bin`, coreMLUrl: coreML('base'), description: 'Better accuracy, 99 languages' },
+ { id: 'small', name: 'Small', size: 466, lang: 'multi', url: `${GGML_BASE}/ggml-small.bin`, coreMLUrl: coreML('small'), description: 'High accuracy, 99 languages' },
+ { id: 'medium', name: 'Medium', size: 1500, lang: 'multi', url: `${GGML_BASE}/ggml-medium.bin`, coreMLUrl: coreML('medium'), description: 'Near human-level, 99 languages, ~2 GB RAM' },
+ { id: 'large-v3-turbo', name: 'Large v3 Turbo', size: 809, lang: 'multi', url: `${GGML_BASE}/ggml-large-v3-turbo.bin`, coreMLUrl: coreML('large-v3-turbo'), description: 'Fast + accurate, distilled large, 99 languages' },
+ { id: 'large-v3', name: 'Large v3', size: 1550, lang: 'multi', url: `${GGML_BASE}/ggml-large-v3.bin`, coreMLUrl: coreML('large-v3'), description: 'Best quality, 99 languages, ~3 GB RAM' },
];
diff --git a/src/services/whisperService.ts b/src/services/whisperService.ts
index 21e3fbd16..92ab06d39 100644
--- a/src/services/whisperService.ts
+++ b/src/services/whisperService.ts
@@ -7,6 +7,7 @@ import { initWhisper, WhisperContext, RealtimeTranscribeEvent } from 'whisper.rn
import * as WhisperRn from 'whisper.rn';
import { Platform, PermissionsAndroid } from 'react-native';
import RNFS from 'react-native-fs';
+import { unzip } from 'react-native-zip-archive';
import logger from '../utils/logger';
import { WHISPER_MODELS } from './whisperModels';
import { audioSessionManager } from './audioSessionManager';
@@ -70,6 +71,22 @@ interface TranscribeFileOptions {
// [SPEAKER_TURN] token. Requires a tdrz model (ggml-small.en-tdrz.bin);
// other models silently ignore it. English only.
diarize?: boolean;
+ // Optional vocabulary hint (whisper.cpp initial prompt): a short list of
+ // proper nouns / jargon (e.g. "Off Grid, Locket, Kokoro") that biases whisper
+ // toward spelling them correctly. Kept short - it competes with audio context.
+ prompt?: string;
+}
+
+/**
+ * Thrown when a file transcription is requested while one is already running on
+ * the single shared context. Lets callers distinguish "busy" from a real failure
+ * (and avoids the old behaviour of silently orphaning the first job's cancel handle).
+ */
+export class WhisperBusyError extends Error {
+ constructor(message = 'A transcription is already in progress') {
+ super(message);
+ this.name = 'WhisperBusyError';
+ }
}
class WhisperService {
@@ -82,6 +99,9 @@ class WhisperService {
private transcriptionFullyStopped: Promise = Promise.resolve();
private activeDownloadId: string | null = null;
private fileTranscribeStop: (() => void | Promise) | null = null;
+ // Models whose CoreML encoder we've already tried to backfill this session,
+ // so a missing/404 encoder isn't re-fetched on every load.
+ private coreMLBackfillTried = new Set();
getModelsDir(): string { return `${RNFS.DocumentDirectoryPath}/whisper-models`; }
async ensureModelsDirExists(): Promise {
@@ -91,6 +111,80 @@ class WhisperService {
getModelPath(modelId: string): string { return `${this.getModelsDir()}/ggml-${modelId}.bin`; }
async isModelDownloaded(modelId: string): Promise { return RNFS.exists(this.getModelPath(modelId)); }
+ // Path where whisper.cpp looks for a model's CoreML encoder: it derives it
+ // from the ggml filename, `.bin` -> `-encoder.mlmodelc`. Keep in lockstep with
+ // the load-time check below.
+ private coreMLPathFor(modelId: string): string {
+ return this.getModelPath(modelId).replace(/\.bin$/i, '-encoder.mlmodelc');
+ }
+
+ /** True when this model's CoreML encoder is present on disk (iOS only). */
+ async hasCoreMLEncoder(modelId: string): Promise {
+ if (Platform.OS !== 'ios') return false;
+ return RNFS.exists(this.coreMLPathFor(modelId));
+ }
+
+ /**
+ * iOS only: download + unzip a model's CoreML encoder next to its .bin so
+ * whisper.cpp can run the encoder on the Apple Neural Engine (~2-3x faster
+ * encode, frees the CPU). Non-fatal - on any failure the model still works on
+ * CPU. No-op on Android, when the model has no published encoder, or when it's
+ * already present.
+ */
+ async ensureCoreMLEncoder(modelId: string, onProgress?: (p: number) => void): Promise {
+ if (Platform.OS !== 'ios') return false;
+ const model = WHISPER_MODELS.find(m => m.id === modelId);
+ if (!model?.coreMLUrl) return false;
+ const targetDir = this.coreMLPathFor(modelId); // ggml--encoder.mlmodelc
+ if (await RNFS.exists(targetDir)) return true;
+ await this.ensureModelsDirExists();
+ const zipPath = `${this.getModelsDir()}/ggml-${modelId}-encoder.mlmodelc.zip`;
+ await RNFS.unlink(zipPath).catch(() => {}); // clear any partial from a prior run
+ try {
+ logger.log(`[Whisper][CoreML] START download ${modelId} from ${model.coreMLUrl}`);
+ const t0 = Date.now();
+ let lastPct = -1;
+ const { promise } = RNFS.downloadFile({
+ fromUrl: model.coreMLUrl,
+ toFile: zipPath,
+ progressInterval: 500,
+ progress: (r) => {
+ if (r.contentLength <= 0) return;
+ const frac = r.bytesWritten / r.contentLength;
+ onProgress?.(frac);
+ const pct = Math.floor(frac * 10) * 10; // log each 10%
+ if (pct !== lastPct) {
+ lastPct = pct;
+ logger.log(`[Whisper][CoreML] ${modelId} ${pct}% (${(r.bytesWritten / 1e6).toFixed(0)}/${(r.contentLength / 1e6).toFixed(0)} MB)`);
+ }
+ },
+ });
+ const res = await promise;
+ if (res.statusCode && res.statusCode >= 400) throw new Error(`HTTP ${res.statusCode}`);
+ const zipMB = (Number((await RNFS.stat(zipPath)).size) / 1e6).toFixed(0);
+ logger.log(`[Whisper][CoreML] downloaded ${modelId} (${zipMB} MB) in ${((Date.now() - t0) / 1000).toFixed(1)}s — unzipping`);
+ await unzip(zipPath, this.getModelsDir());
+ await RNFS.unlink(zipPath).catch(() => {});
+ // The zip's top-level dir is named after the SOURCE encoder in the URL. For
+ // most models that already equals targetDir; when a model reuses another's
+ // encoder (tdrz -> small.en) the names differ, so rename it into place.
+ const extractedName = model.coreMLUrl.split('/').pop()!.replace(/\.zip$/i, '');
+ const extractedDir = `${this.getModelsDir()}/${extractedName}`;
+ if (extractedDir !== targetDir && (await RNFS.exists(extractedDir))) {
+ await RNFS.unlink(targetDir).catch(() => {}); // clear any stale target
+ await RNFS.moveFile(extractedDir, targetDir);
+ logger.log(`[Whisper][CoreML] reused ${extractedName} → ${targetDir.split('/').pop()}`);
+ }
+ const ok = await RNFS.exists(targetDir);
+ logger.log(`[Whisper][CoreML] ${ok ? `READY for ${modelId} — next load will use the Neural Engine` : `FAILED for ${modelId}: no encoder dir after unzip`}`);
+ return ok;
+ } catch (e) {
+ logger.warn(`[Whisper][CoreML] fetch FAILED for ${modelId} (staying CPU-only): ${String(e)}`);
+ await RNFS.unlink(zipPath).catch(() => {});
+ return false;
+ }
+ }
+
async downloadModel(modelId: string, onProgress?: (progress: number) => void): Promise {
const model = WHISPER_MODELS.find(m => m.id === modelId);
if (!model) throw new Error(`Unknown model: ${modelId}`);
@@ -192,6 +286,11 @@ class WhisperService {
// refuses when an entry already exists for this modelKey).
useDownloadStore.getState().remove(modelKey);
}
+ // iOS: also fetch the CoreML encoder so the ANE can run the encoder. Non-fatal
+ // and no-op off-iOS / when unpublished - the model is already usable on CPU.
+ if (Platform.OS === 'ios') {
+ await this.ensureCoreMLEncoder(modelId).catch(() => {});
+ }
logger.log(`[Whisper] Downloaded to ${destPath}`);
return destPath;
}
@@ -295,17 +394,29 @@ class WhisperService {
// Native initWithModelPath calls abort() on invalid files, crashing the app.
await this.validateModelFile(modelPath);
- // CoreML only helps when the per-model encoder bundle (ggml--encoder.mlmodelc)
- // is present. Enabling it WITHOUT that asset makes whisper.rn fail to load CoreML,
- // fall back to CPU, AND then crash at transcribe (0%) on some iOS devices (e.g. the
- // A12 / iPhone XS). The encoder assets are not wired yet, so this guard keeps CoreML
- // off until they are - the toggle becomes a no-op rather than a crash.
- let useCoreML = options?.useCoreML ?? false;
- if (useCoreML) {
+ // CoreML runs the whisper encoder on the Apple Neural Engine (iOS only, ~2-3x
+ // faster encode, frees the CPU). Drive it purely off the per-model encoder
+ // asset: if ggml--encoder.mlmodelc sits next to the .bin we use CoreML,
+ // else CPU. Enabling CoreML WITHOUT that asset makes whisper.rn fail to load it
+ // and can crash at transcribe (0%) on some A12 devices (e.g. iPhone XS), so
+ // presence is the gate - not a user toggle. Non-iOS never uses CoreML.
+ let useCoreML = false;
+ if (Platform.OS === 'ios') {
const coreMLPath = modelPath.replace(/\.bin$/i, '-encoder.mlmodelc');
- if (!(await RNFS.exists(coreMLPath))) {
- logger.warn('[Whisper] CoreML requested but encoder asset missing; using CPU instead');
- useCoreML = false;
+ useCoreML = await RNFS.exists(coreMLPath);
+ if (useCoreML) {
+ logger.log(`[Whisper][CoreML] encoder PRESENT for this model (${coreMLPath.split('/').pop()}) — requesting Neural Engine. Watch for whisper.cpp native line "Core ML model loaded" to confirm actual use.`);
+ } else {
+ if (options?.useCoreML) logger.warn('[Whisper] CoreML requested but encoder asset missing; using CPU instead');
+ // Backfill: model was downloaded before CoreML shipping. Fetch its encoder
+ // in the background (non-blocking, once per session) so the NEXT load uses
+ // the ANE. This load stays CPU.
+ const model = WHISPER_MODELS.find(m => this.getModelPath(m.id) === modelPath);
+ if (model?.coreMLUrl && !this.coreMLBackfillTried.has(model.id)) {
+ this.coreMLBackfillTried.add(model.id);
+ logger.log(`[Whisper] CoreML encoder missing for ${model.id}; fetching in background for next load`);
+ this.ensureCoreMLEncoder(model.id).catch(() => {});
+ }
}
}
@@ -322,9 +433,15 @@ class WhisperService {
useFlashAttn: options?.useFlashAttn ?? false,
useCoreMLIos: useCoreML,
};
+ // Time initWhisper: this covers reading the .bin into memory AND, when
+ // useCoreML is true, the one-time ANE compile of the .mlmodelc (whisper.cpp
+ // logs "first run on a device may take a while"). If startup is slow, this
+ // number vs the first "transcribe progress" elapsed tells us whether it's
+ // load/compile or the first encode.
+ const tInit = Date.now();
this.context = await initWhisper(initOpts as unknown as Parameters[0]);
this.currentModelPath = modelPath;
- logger.log('[Whisper] Model loaded successfully');
+ logger.log(`[Whisper] Model loaded successfully — initWhisper took ${((Date.now() - tInit) / 1000).toFixed(1)}s (useCoreML=${useCoreML})`);
} catch (error) {
logger.error('[Whisper] Failed to load model:', error);
this.context = null;
@@ -335,12 +452,20 @@ class WhisperService {
async unloadModel(): Promise {
if (!this.context) return;
- // Stop active transcription to prevent SIGSEGV on freed context
+ // Stop active transcription to prevent SIGSEGV on a freed context.
+ // Realtime path (isTranscribing/stopFn):
if (this.isTranscribing || this.stopFn) {
- logger.log('[WhisperService] Stopping active transcription before unloading model');
+ logger.log('[WhisperService] Stopping active realtime transcription before unloading model');
await this.stopTranscription();
await this.transcriptionFullyStopped;
}
+ // File path (fileTranscribeStop): a resumable/whole-file transcribe can be
+ // in flight on this same context (it survives navigation by design). Releasing
+ // underneath it is a use-after-free, so cancel and await it first.
+ if (this.fileTranscribeStop) {
+ logger.log('[WhisperService] Stopping in-flight file transcription before unloading model');
+ await this.stopFileTranscription();
+ }
if (this.isReleasingContext) { logger.log('[WhisperService] Context release already in progress, skipping'); return; }
this.isReleasingContext = true;
this.contextReleasePromise = (async () => {
@@ -553,6 +678,15 @@ class WhisperService {
},
};
if (language !== 'auto') transcribeOpts.language = language;
+ // Do NOT condition on previously-decoded text (whisper.cpp -mc 0). On noisy /
+ // ambient clips whisper otherwise falls into a repetition death-spiral,
+ // looping the same token or phrase; clearing the text context is the standard
+ // fix and the biggest lever against hallucinated repeats.
+ transcribeOpts.maxContext = 0;
+ // Vocabulary hint: whisper.cpp seeds decoding with this text so proper nouns
+ // and jargon are spelled the user's way. Trimmed; empty is omitted entirely.
+ const promptHint = options?.prompt?.trim();
+ if (promptHint) transcribeOpts.prompt = promptHint;
if (maxThreads > 0) transcribeOpts.maxThreads = maxThreads;
if (nProcessors > 1) transcribeOpts.nProcessors = nProcessors;
if (options?.offset && options.offset > 0) transcribeOpts.offset = Math.floor(options.offset);
@@ -587,6 +721,12 @@ class WhisperService {
if (!this.context) {
throw new Error('No Whisper model loaded');
}
+ // Single shared context: refuse a second overlapping file transcription
+ // instead of overwriting the in-flight job's cancel handle (which would leave
+ // the first job un-cancellable and both racing the one native context).
+ if (this.fileTranscribeStop) {
+ throw new WhisperBusyError();
+ }
const requestedLanguage = options?.language || 'auto';
// English-only models (ggml-*.en) have ONLY English tokens. Asking them for
@@ -622,6 +762,7 @@ class WhisperService {
tStart,
});
+ logger.log(`[Whisper] dispatching native transcribe (lang=${language} diarize=${options?.diarize ?? false} threads=${maxThreads} nProc=${nProcessors}) — awaiting first progress...`);
const { stop, promise } = this.context.transcribe(
filePath,
transcribeOpts as Parameters[1],
@@ -692,8 +833,10 @@ export function cleanTranscription(raw: string): string {
.replace(/\([^)]*\)/g, ' ') // (silence), (speaking foreign language)
.replace(/\s+/g, ' ')
.trim();
- // Only markers / punctuation left → no real speech.
- if (!/[a-z0-9]/i.test(stripped)) return '';
+ // Only markers / punctuation left → no real speech. Match letters/digits in ANY
+ // script (\p{L}\p{N}), not just ASCII - else a Hindi / Arabic / CJK transcript
+ // has no a-z and gets wiped to '' (silent data loss for non-English users).
+ if (!/[\p{L}\p{N}]/u.test(stripped)) return '';
return stripped;
}
diff --git a/src/stores/devInferenceStore.ts b/src/stores/devInferenceStore.ts
new file mode 100644
index 000000000..bc12515c4
--- /dev/null
+++ b/src/stores/devInferenceStore.ts
@@ -0,0 +1,80 @@
+import { create } from 'zustand';
+import { persist, createJSONStorage } from 'zustand/middleware';
+import AsyncStorage from '@react-native-async-storage/async-storage';
+
+/**
+ * DEV-ONLY store for the chat grammar test harness.
+ *
+ * Lets a developer paste a GBNF grammar (plus optional temperature / assistant
+ * prefill / word cap) and route it into the next chat completion, to see how
+ * the real on-device model behaves under grammar + prefill before wiring GBNF
+ * into a shipped feature. The only UI that can flip `enabled` is `__DEV__`-gated,
+ * so it has no effect in production.
+ *
+ * Persisted (except the transient `lastError`) so a pasted grammar survives an
+ * app kill / reload - you don't have to paste it again each session.
+ */
+interface DevInferenceState {
+ enabled: boolean; // master toggle
+ grammar: string; // raw GBNF pasted by the user
+ temperature?: number; // e.g. 0 for deterministic; undefined = leave default
+ assistantPrefix: string; // prefill, e.g. "TITLE: "
+ maxWords?: number; // hard output cap; converted to n_predict. Guards runaway grammars.
+ // LiteRT backend uses a different engine (LLGuidance) that takes JSON schema /
+ // Lark grammar / regex, NOT GBNF. Kept separate from `grammar` since the two
+ // backends can't share a format. Only used when the active model is LiteRT.
+ litertConstraintType: 'json_schema' | 'lark' | 'regex';
+ litertConstraintString: string;
+ lastError?: string; // GBNF parse / apply error from the last run, shown in the modal
+ setEnabled: (v: boolean) => void;
+ setGrammar: (g: string) => void;
+ setTemperature: (t?: number) => void;
+ setAssistantPrefix: (p: string) => void;
+ setMaxWords: (n?: number) => void;
+ setLitertConstraintType: (t: 'json_schema' | 'lark' | 'regex') => void;
+ setLitertConstraintString: (s: string) => void;
+ setLastError: (e?: string) => void;
+ clear: () => void;
+}
+
+const EMPTY = {
+ enabled: false,
+ grammar: '',
+ temperature: undefined,
+ assistantPrefix: '',
+ maxWords: undefined,
+ litertConstraintType: 'json_schema' as const,
+ litertConstraintString: '',
+ lastError: undefined,
+} as const;
+
+export const useDevInferenceStore = create()(
+ persist(
+ (set) => ({
+ ...EMPTY,
+ setEnabled: (v) => set({ enabled: v }),
+ setGrammar: (g) => set({ grammar: g }),
+ setTemperature: (t) => set({ temperature: t }),
+ setAssistantPrefix: (p) => set({ assistantPrefix: p }),
+ setMaxWords: (n) => set({ maxWords: n }),
+ setLitertConstraintType: (t) => set({ litertConstraintType: t }),
+ setLitertConstraintString: (s) => set({ litertConstraintString: s }),
+ setLastError: (e) => set({ lastError: e }),
+ clear: () => set({ ...EMPTY }),
+ }),
+ {
+ name: 'dev-inference-storage',
+ storage: createJSONStorage(() => AsyncStorage),
+ // lastError is per-run state; don't carry it across restarts.
+ partialize: (s) => ({
+ enabled: s.enabled,
+ grammar: s.grammar,
+ temperature: s.temperature,
+ assistantPrefix: s.assistantPrefix,
+ maxWords: s.maxWords,
+ litertConstraintType: s.litertConstraintType,
+ litertConstraintString: s.litertConstraintString,
+ }),
+ },
+ ),
+);
diff --git a/src/stores/whisperStore.ts b/src/stores/whisperStore.ts
index 11ab4265b..6fbcf677d 100644
--- a/src/stores/whisperStore.ts
+++ b/src/stores/whisperStore.ts
@@ -217,8 +217,16 @@ export const useWhisperStore = create()(
await whisperService.unloadModel();
// Then delete
await whisperService.deleteModel(downloadedModelId);
+ // Fall back to another downloaded model on disk if there is one, and
+ // drop the just-deleted model from presentModelIds (recompute from disk
+ // so the models list doesn't keep showing a model whose file is gone).
+ const onDisk = await whisperService.listDownloadedModels();
+ const remaining = onDisk.map((m) => m.modelId).filter((id) => id !== downloadedModelId);
+ const fallback = remaining[0] ?? null;
+ logger.log(`[WhisperStore] deleted active ${downloadedModelId}; present [${remaining.join(', ') || 'none'}]; active -> ${fallback ?? 'none'}`);
set({
- downloadedModelId: null,
+ presentModelIds: remaining,
+ downloadedModelId: fallback,
isModelLoaded: false,
});
} catch (error) {
@@ -236,13 +244,22 @@ export const useWhisperStore = create()(
deleteModelById: async (modelId: string) => {
try {
- if (get().downloadedModelId === modelId) await whisperService.unloadModel();
+ const wasActive = get().downloadedModelId === modelId;
+ if (wasActive) await whisperService.unloadModel();
await whisperService.deleteModel(modelId);
- set((s) => ({
- presentModelIds: s.presentModelIds.filter((id) => id !== modelId),
- ...(s.downloadedModelId === modelId ? { downloadedModelId: null, isModelLoaded: false } : {}),
- }));
+ // Fall back to another model still on disk (e.g. delete small -> use
+ // base) instead of leaving no active model. Scans the real dir so it
+ // catches any downloaded model, not just the catalogue.
+ const onDisk = await whisperService.listDownloadedModels();
+ const remaining = onDisk.map((m) => m.modelId).filter((id) => id !== modelId);
+ const fallback = wasActive ? (remaining[0] ?? null) : get().downloadedModelId;
+ logger.log(`[WhisperStore] deleted ${modelId} (wasActive=${wasActive}); on-disk now [${remaining.join(', ') || 'none'}]; active -> ${fallback ?? 'none'}`);
+ set({
+ presentModelIds: remaining,
+ ...(wasActive ? { downloadedModelId: fallback, isModelLoaded: false } : {}),
+ });
} catch (error) {
+ logger.warn(`[WhisperStore] deleteModelById(${modelId}) failed: ${String(error)}`);
set({ error: error instanceof Error ? error.message : 'Failed to delete model' });
}
},
@@ -258,11 +275,19 @@ export const useWhisperStore = create()(
// which left the Home banner showing a deleted model. Check the active
// model's own file (works for custom HF ids, not just the catalogue).
const activeId = get().downloadedModelId;
- const activeOnDisk = activeId ? await whisperService.isModelDownloaded(activeId) : true;
- set({
- presentModelIds: present,
- ...(activeId && !activeOnDisk ? { downloadedModelId: null, isModelLoaded: false } : {}),
- });
+ const activeOnDisk = activeId ? await whisperService.isModelDownloaded(activeId) : false;
+ // Active model is fine (set and on disk): only refresh the present list.
+ if (activeId && activeOnDisk) {
+ set({ presentModelIds: present });
+ return;
+ }
+ // Otherwise there's no valid active model - either it was never set, or
+ // its file is gone (e.g. small deleted). Adopt a model that IS on disk
+ // (base) so transcription keeps working instead of silently doing
+ // nothing with "no model".
+ const fallback = present[0] ?? null;
+ logger.log(`[WhisperStore] no valid active whisper model (was ${activeId ?? 'none'}); present [${present.join(', ') || 'none'}]; active -> ${fallback ?? 'none'}`);
+ set({ presentModelIds: present, downloadedModelId: fallback, isModelLoaded: false });
},
clearError: () => {