diff --git a/.gitignore b/.gitignore index c44bbc869..e21fd5477 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,11 @@ fastlane/*.p8 !.yarn/sdks !.yarn/versions docs/TRACTION_KNOWLEDGE_BASE.md +# Dev-only insights sim harness + private real recordings (never commit) +/sim/ +# Debug screenshots (scratch) +/image.png +/image-*.png # Local marketing drafts (not part of the app) marketing/ diff --git a/App.tsx b/App.tsx index 4a21237ef..17e2c0d95 100644 --- a/App.tsx +++ b/App.tsx @@ -31,6 +31,7 @@ import { LockScreen } from './src/screens'; import { useAppState } from './src/hooks/useAppState'; import { useDownloadStore } from './src/stores/downloadStore'; import { ErrorBoundary } from './src/components/ErrorBoundary'; +import { Toast } from './src/components'; LogBox.ignoreAllLogs(); // Suppress all logs @@ -392,6 +393,7 @@ function App() { > + ); diff --git a/__tests__/rntl/navigation/AppNavigator.test.tsx b/__tests__/rntl/navigation/AppNavigator.test.tsx index 1c043e114..0de77cde5 100644 --- a/__tests__/rntl/navigation/AppNavigator.test.tsx +++ b/__tests__/rntl/navigation/AppNavigator.test.tsx @@ -179,7 +179,7 @@ describe('AppNavigator', () => { }); describe('Tab bar rendering', () => { - it('renders all five tab labels', () => { + it('renders all five tab labels (Recorder replaced by Settings)', () => { const { getAllByText } = renderAppNavigator(); expect(getAllByText('Home').length).toBeGreaterThanOrEqual(1); @@ -198,6 +198,11 @@ describe('AppNavigator', () => { expect(getByTestId('models-tab')).toBeTruthy(); expect(getByTestId('settings-tab')).toBeTruthy(); }); + + it('no longer renders a Recorder tab (moved to a Home card)', () => { + const { queryByTestId } = renderAppNavigator(); + expect(queryByTestId('recorder-tab')).toBeNull(); + }); }); describe('Tab bar safe area insets', () => { @@ -279,14 +284,12 @@ describe('AppNavigator', () => { expect(getAllByText('Chats').length).toBeGreaterThanOrEqual(1); expect(getAllByText('Projects').length).toBeGreaterThanOrEqual(1); expect(getAllByText('Models').length).toBeGreaterThanOrEqual(1); - expect(getAllByText('Settings').length).toBeGreaterThanOrEqual(1); // All tab buttons should be pressable expect(getByTestId('home-tab')).toBeTruthy(); expect(getByTestId('chats-tab')).toBeTruthy(); expect(getByTestId('projects-tab')).toBeTruthy(); expect(getByTestId('models-tab')).toBeTruthy(); - expect(getByTestId('settings-tab')).toBeTruthy(); }); }); }); diff --git a/__tests__/rntl/onboarding/HomeScreenSpotlight.test.tsx b/__tests__/rntl/onboarding/HomeScreenSpotlight.test.tsx index 22b6dc339..41f907ded 100644 --- a/__tests__/rntl/onboarding/HomeScreenSpotlight.test.tsx +++ b/__tests__/rntl/onboarding/HomeScreenSpotlight.test.tsx @@ -299,7 +299,7 @@ describe('HomeScreen Spotlight Integration', () => { // Flow 5: Explore Settings // ======================================================================== describe('Flow 5: exploredSettings', () => { - it('queues pending spotlight 6, navigates to SettingsTab, fires goTo(5)', () => { + it('queues pending spotlight 6, navigates to Settings, fires goTo(5)', () => { const { getByTestId } = renderHomeScreen(); act(() => { @@ -307,7 +307,7 @@ describe('HomeScreen Spotlight Integration', () => { }); expect(peekPendingSpotlight()).toBe(6); - expect(mockNavigate).toHaveBeenCalledWith('SettingsTab'); + expect(mockNavigate).toHaveBeenCalledWith('Settings'); act(() => { jest.advanceTimersByTime(800); }); expect(mockGoTo).toHaveBeenCalledWith(5); diff --git a/__tests__/unit/onboarding/handleStepPress.test.ts b/__tests__/unit/onboarding/handleStepPress.test.ts index 42670b3ef..f44cfe57f 100644 --- a/__tests__/unit/onboarding/handleStepPress.test.ts +++ b/__tests__/unit/onboarding/handleStepPress.test.ts @@ -307,9 +307,9 @@ describe('handleStepPress', () => { expect(peekPendingSpotlight()).toBe(6); }); - it('navigates to SettingsTab', () => { + it('navigates to Settings', () => { simulateHandleStepPress('exploredSettings', callbacks()); - expect(navigate).toHaveBeenCalledWith('SettingsTab'); + expect(navigate).toHaveBeenCalledWith('Settings'); }); it('fires goTo(5) after delay', () => { diff --git a/__tests__/unit/onboarding/onboardingFlows.test.ts b/__tests__/unit/onboarding/onboardingFlows.test.ts index 0d9389e75..3b3c7d348 100644 --- a/__tests__/unit/onboarding/onboardingFlows.test.ts +++ b/__tests__/unit/onboarding/onboardingFlows.test.ts @@ -80,7 +80,7 @@ describe('Onboarding Flows', () => { downloadedModel: 'ModelsTab', loadedModel: 'HomeTab', sentMessage: 'ChatsTab', - exploredSettings: 'SettingsTab', + exploredSettings: 'Settings', createdProject: 'ProjectsTab', triedImageGen: 'ModelsTab', }); diff --git a/__tests__/unit/services/rag/database.test.ts b/__tests__/unit/services/rag/database.test.ts index 206f587fd..5f433d1e3 100644 --- a/__tests__/unit/services/rag/database.test.ts +++ b/__tests__/unit/services/rag/database.test.ts @@ -42,11 +42,13 @@ describe('RagDatabase', () => { it('opens the database and creates tables', async () => { await ragDatabase.ensureReady(); expect(open).toHaveBeenCalledWith({ name: 'rag.db' }); - // rag_documents, rag_chunks, rag_embeddings = 3 tables - expect(mockExecuteSync).toHaveBeenCalledTimes(3); + // rag_documents, rag_chunks, ALTER rag_chunks (metadata migration), rag_embeddings + expect(mockExecuteSync).toHaveBeenCalledTimes(4); expect(mockExecuteSync.mock.calls[0][0]).toContain('rag_documents'); expect(mockExecuteSync.mock.calls[1][0]).toContain('rag_chunks'); - expect(mockExecuteSync.mock.calls[2][0]).toContain('rag_embeddings'); + expect(mockExecuteSync.mock.calls[2][0]).toContain('ALTER TABLE rag_chunks'); + expect(mockExecuteSync.mock.calls[2][0]).toContain('metadata'); + expect(mockExecuteSync.mock.calls[3][0]).toContain('rag_embeddings'); }); it('does not re-initialize on second call', async () => { @@ -86,8 +88,22 @@ describe('RagDatabase', () => { (c: any[]) => typeof c[0] === 'string' && c[0].includes('INSERT INTO rag_chunks') ); expect(chunkInserts).toHaveLength(2); - expect(chunkInserts[0][1]).toEqual(['chunk one', 42, 0]); - expect(chunkInserts[1][1]).toEqual(['chunk two', 42, 1]); + // 4th bind is metadata (null when the chunk carries none). + expect(chunkInserts[0][1]).toEqual(['chunk one', 42, 0, null]); + expect(chunkInserts[1][1]).toEqual(['chunk two', 42, 1, null]); + }); + + it('serializes chunk metadata to a JSON string on the way into the DB', async () => { + await ragDatabase.ensureReady(); + mockExecuteSync.mockReturnValue({ insertId: 7, rowsAffected: 1, rows: [] }); + + const metadata = { recordingId: 'rec-1', startMs: 100, eventTitle: 'Standup' }; + ragDatabase.insertChunks(42, [{ content: 'has meta', position: 0, metadata } as any]); + + const chunkInsert = mockExecuteSync.mock.calls.find( + (c: any[]) => typeof c[0] === 'string' && c[0].includes('INSERT INTO rag_chunks'), + ); + expect(chunkInsert![1]).toEqual(['has meta', 42, 0, JSON.stringify(metadata)]); }); }); diff --git a/__tests__/unit/services/selectTextModel.test.ts b/__tests__/unit/services/selectTextModel.test.ts new file mode 100644 index 000000000..190183f19 --- /dev/null +++ b/__tests__/unit/services/selectTextModel.test.ts @@ -0,0 +1,63 @@ +import { selectTextModelToLoad, fitsBudget } from '../../../src/services/selectTextModel'; +import type { DownloadedModel } from '../../../src/types'; + +const MB = 1024 * 1024; + +function model(id: string, fileSizeMB: number): DownloadedModel { + return { + id, + name: id, + author: 'test', + filePath: `/models/${id}`, + fileName: `${id}.gguf`, + fileSize: fileSizeMB * MB, + quantization: 'Q4', + downloadedAt: '2026-07-13', + engine: 'llama', + }; +} + +// Footprint = fileSize in MB (1x) — the selection logic is independent of the +// multiplier; the real caller passes hardwareService.estimateModelRam. +const footprint = (m: DownloadedModel) => (m.fileSize || 0) / MB; + +const small = model('small', 500); +const medium = model('medium', 1000); +const large = model('large', 3000); + +describe('fitsBudget', () => { + it('fits when footprint <= budget, not otherwise', () => { + expect(fitsBudget(1000, 1000)).toBe(true); // exactly fits + expect(fitsBudget(1001, 1000)).toBe(false); + }); +}); + +describe('selectTextModelToLoad', () => { + it('returns null when nothing is downloaded', () => { + expect(selectTextModelToLoad([], 4000, { activeId: null, footprintMB: footprint })).toBeNull(); + expect(selectTextModelToLoad([], 4000, { activeId: 'medium', footprintMB: footprint })).toBeNull(); + }); + + it('uses the active model when it fits the budget', () => { + expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: 'small', footprintMB: footprint })?.id).toBe('small'); + }); + + it('ignores the active model when it does NOT fit, and picks the largest that fits', () => { + // budget 2000: large(3000) does not fit -> largest fitting is medium(1000) + expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: 'large', footprintMB: footprint })?.id).toBe('medium'); + }); + + it('with no active id, picks the largest model that fits (best quality within RAM)', () => { + expect(selectTextModelToLoad([small, medium, large], 2000, { activeId: null, footprintMB: footprint })?.id).toBe('medium'); + expect(selectTextModelToLoad([small, medium, large], 4000, { activeId: null, footprintMB: footprint })?.id).toBe('large'); + }); + + it('falls back to the SMALLEST when nothing fits (run something, not an OOM)', () => { + // budget 400: smallest is small(500) > 400, nothing fits -> smallest + expect(selectTextModelToLoad([small, medium, large], 400, { activeId: 'large', footprintMB: footprint })?.id).toBe('small'); + }); + + it('ignores an active id that is not among the downloaded models', () => { + expect(selectTextModelToLoad([small, medium], 2000, { activeId: 'ghost', footprintMB: footprint })?.id).toBe('medium'); + }); +}); diff --git a/__tests__/unit/services/whisperService.test.ts b/__tests__/unit/services/whisperService.test.ts index 9e7197968..abdc6350f 100644 --- a/__tests__/unit/services/whisperService.test.ts +++ b/__tests__/unit/services/whisperService.test.ts @@ -311,11 +311,49 @@ describe('WhisperService', () => { await whisperService.loadModel('/path/to/model.bin'); - expect(initWhisper).toHaveBeenCalledWith({ filePath: '/path/to/model.bin' }); + expect(initWhisper).toHaveBeenCalledWith({ + filePath: '/path/to/model.bin', + useGpu: false, + useFlashAttn: false, + // The test's RNFS.exists mock reports the CoreML encoder present, so + // loadModel auto-enables ANE CoreML on iOS. + useCoreMLIos: true, + }); expect(whisperService.isModelLoaded()).toBe(true); expect(whisperService.getLoadedModelPath()).toBe('/path/to/model.bin'); }); + it('falls back to CPU when CoreML requested but the encoder asset is missing', async () => { + // Valid model file, but the ggml--encoder.mlmodelc bundle is absent. + // Enabling CoreML without it makes whisper.rn crash at 0% on some iOS devices, + // so the guard must silently downgrade to CPU (useCoreMLIos: false). + mockedRNFS.stat.mockResolvedValue({ size: 75 * 1024 * 1024, isFile: () => true } as any); + mockedRNFS.exists.mockImplementation(async (p: string) => + !p.endsWith('-encoder.mlmodelc'), + ); + const mockContext = { id: 'ctx', release: jest.fn(), transcribeRealtime: jest.fn(), transcribe: jest.fn() }; + mockedInitWhisper.mockResolvedValue(mockContext as any); + + await whisperService.loadModel('/path/to/model.bin', { useCoreML: true }); + + expect(initWhisper).toHaveBeenCalledWith( + expect.objectContaining({ filePath: '/path/to/model.bin', useCoreMLIos: false }), + ); + }); + + it('enables CoreML when the encoder asset is present', async () => { + mockedRNFS.stat.mockResolvedValue({ size: 75 * 1024 * 1024, isFile: () => true } as any); + mockedRNFS.exists.mockResolvedValue(true); // both the .bin and the .mlmodelc exist + const mockContext = { id: 'ctx', release: jest.fn(), transcribeRealtime: jest.fn(), transcribe: jest.fn() }; + mockedInitWhisper.mockResolvedValue(mockContext as any); + + await whisperService.loadModel('/path/to/model.bin', { useCoreML: true, useGpu: true, useFlashAttn: true }); + + expect(initWhisper).toHaveBeenCalledWith( + expect.objectContaining({ useCoreMLIos: true, useGpu: true, useFlashAttn: true }), + ); + }); + it('unloads different model before loading new one', async () => { mockValidModelFile(); const mockContext1 = { @@ -788,7 +826,8 @@ describe('WhisperService', () => { })), }; mockedInitWhisper.mockResolvedValueOnce(mockContext as any); - await whisperService.loadModel('/path/model.bin'); + // English-only model (.en.bin) so transcribeFile forces language 'en'. + await whisperService.loadModel('/path/model.en.bin'); const result = await whisperService.transcribeFile('/audio.wav'); diff --git a/__tests__/unit/utils/memorySnapshot.test.ts b/__tests__/unit/utils/memorySnapshot.test.ts new file mode 100644 index 000000000..24fb138de --- /dev/null +++ b/__tests__/unit/utils/memorySnapshot.test.ts @@ -0,0 +1,68 @@ +/** + * memorySnapshot unit tests + * + * logMemory() is a diagnostics probe used around whisper model load and each + * transcribe chunk to capture the app's footprint. On iOS it surfaces whether + * an apparent transcription "crash" was actually a jetsam low-memory kill. + * + * Guarantees under test: + * - formats used/total in MB and a percentage + * - never throws (a failing probe must not break the path it observes) + * - no divide-by-zero when total memory is reported as 0 + */ + +import DeviceInfo from 'react-native-device-info'; +import logger from '../../../src/utils/logger'; +import { logMemory } from '../../../src/utils/memorySnapshot'; + +const mockedDeviceInfo = DeviceInfo as jest.Mocked; + +describe('logMemory', () => { + let logSpy: jest.SpyInstance; + let warnSpy: jest.SpyInstance; + + beforeEach(() => { + jest.clearAllMocks(); + logSpy = jest.spyOn(logger, 'log').mockImplementation(() => {}); + warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + warnSpy.mockRestore(); + }); + + it('logs used/total in MB with a percentage, tagged with the call site', async () => { + mockedDeviceInfo.getUsedMemory.mockResolvedValue(1.4 * 1024 * 1024 * 1024); + mockedDeviceInfo.getTotalMemory.mockResolvedValue(4 * 1024 * 1024 * 1024); + + await logMemory('whisper:beforeLoad'); + + expect(logSpy).toHaveBeenCalledTimes(1); + const msg = logSpy.mock.calls[0][0] as string; + expect(msg).toContain('[mem] whisper:beforeLoad'); + expect(msg).toContain('used=1434MB'); + expect(msg).toContain('total=4096MB'); + expect(msg).toContain('(35%)'); + }); + + it('never throws and warns when the probe fails', async () => { + mockedDeviceInfo.getUsedMemory.mockRejectedValue(new Error('boom')); + mockedDeviceInfo.getTotalMemory.mockResolvedValue(4 * 1024 * 1024 * 1024); + + await expect(logMemory('transcribe:chunk@0s')).resolves.toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('snapshot failed')); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it('does not divide by zero when total memory is unavailable', async () => { + mockedDeviceInfo.getUsedMemory.mockResolvedValue(100 * 1024 * 1024); + mockedDeviceInfo.getTotalMemory.mockResolvedValue(0); + + await logMemory('zero'); + + const msg = logSpy.mock.calls[0][0] as string; + expect(msg).toContain('total=0MB'); + expect(msg).toContain('(0%)'); + }); +}); diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index 64d5ebf85..66a00e017 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -13,6 +13,7 @@ import com.google.ai.edge.litertlm.BenchmarkInfo import com.google.ai.edge.litertlm.ConversationConfig import com.google.ai.edge.litertlm.Engine import com.google.ai.edge.litertlm.EngineConfig +import com.google.ai.edge.litertlm.ExperimentalFlags import com.google.ai.edge.litertlm.Content import com.google.ai.edge.litertlm.Contents import com.google.ai.edge.litertlm.ExperimentalApi @@ -107,8 +108,30 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : private val pendingToolCalls = ConcurrentHashMap>() private var configuredMaxTokens: Int = 4096 + // DEV-only constrained decoding (LLGuidance: json_schema / lark / regex). + // Set from JS via setConstrainedDecoding() before resetConversation. The map + // contract below is UNVERIFIED - every use is wrapped so a wrong shape logs + // and falls back to unconstrained generation, never crashing the chat path. + @Volatile private var constrainedEnabled = false + @Volatile private var constraintType = "" + @Volatile private var constraintString = "" + override fun getName(): String = "LiteRTModule" + // ------------------------------------------------------------------------- + // setConstrainedDecoding (DEV) — arm/disarm an LLGuidance constraint that + // resetConversation + sendMessage will apply. type = json_schema|lark|regex. + // ------------------------------------------------------------------------- + + @ReactMethod + fun setConstrainedDecoding(enabled: Boolean, type: String, constraint: String, promise: Promise) { + constrainedEnabled = enabled && constraint.isNotEmpty() + constraintType = type + constraintString = constraint + Log.i(TAG, "[DevGrammar-LiteRT] setConstrainedDecoding enabled=$constrainedEnabled type=$type len=${constraint.length}") + promise.resolve(null) + } + // ------------------------------------------------------------------------- // loadModel // ------------------------------------------------------------------------- @@ -238,6 +261,7 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : // resetConversation — closes and recreates Conversation only, Engine stays // ------------------------------------------------------------------------- + @OptIn(ExperimentalApi::class) @ReactMethod fun resetConversation(systemPrompt: String, temperature: Double, topK: Int, topP: Double, toolsJson: String, historyJson: String, promise: Promise) { val safe = SafePromise(promise, TAG) @@ -268,6 +292,16 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : ) } + // DEV: constrained decoding is a per-conversation experimental flag, + // so it must be set before createConversation. Guarded - a missing/renamed + // API in a future SDK must not break conversation setup. + try { + ExperimentalFlags.enableConversationConstrainedDecoding = constrainedEnabled + if (constrainedEnabled) debugLog("[DevGrammar-LiteRT] enableConversationConstrainedDecoding=true (type=$constraintType len=${constraintString.length})") + } catch (e: Throwable) { + Log.w(TAG, "[DevGrammar-LiteRT] could not set constrained-decoding flag: ${e.message}") + } + val toolProviders = buildToolProviders(toolsJson) val initialMessages = parseHistoryMessages(historyJson) debugLog("ConversationConfig — historyTurns=${initialMessages.size} tools=${toolProviders.size} maxTokenBudget=$configuredMaxTokens autoToolCalling=${toolProviders.isNotEmpty()}") @@ -409,16 +443,37 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : safe.reject("LITERT_NO_CONV", "No conversation. Call resetConversation first.", null) return@launch } - currentJob = launch { try { val contents = buildSendContents(imageUris, audioUris, text, safe) ?: return@launch + // DEV: attach an LLGuidance constraint via OptionalArgs. The exact + // map shape is UNVERIFIED (C++ docs only), so if building/starting the + // constrained flow throws we log and fall back to an unconstrained send + // - a probe that can reveal the contract from logs without breaking chat. + val flow = if (constrainedEnabled && constraintString.isNotEmpty()) { + try { + val optionalArgs = mapOf( + "decoding_constraint" to mapOf( + "constraint_type" to constraintType, + "constraint_string" to constraintString, + ), + ) + Log.i(TAG, "[DevGrammar-LiteRT] sending WITH decoding_constraint type=$constraintType len=${constraintString.length}") + conv.sendMessageAsync(contents, optionalArgs) + } catch (e: Throwable) { + Log.w(TAG, "[DevGrammar-LiteRT] constrained send failed to start (${e.message}); falling back unconstrained") + conv.sendMessageAsync(contents) + } + } else { + conv.sendMessageAsync(contents) + } + var tokenCount = 0 - conv.sendMessageAsync(contents) + flow .collect { message -> tokenCount++ - if (tokenCount == 1) Log.i(TAG, "sendMessage — first message from model (audio=${audioUris.size} image=${imageUris.size})") + if (tokenCount == 1) Log.i(TAG, "sendMessage — first message from model (audio=${audioUris.size} image=${imageUris.size} constrained=${constrainedEnabled && constraintString.isNotEmpty()})") dispatchStreamToken(message) } @@ -543,10 +598,34 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : } try { conv.close() - Log.d(TAG, "closeConversationSafely — closed") + Log.d(TAG, "closeConversationSafely — closed id=${System.identityHashCode(conv)}") } catch (e: Exception) { Log.w(TAG, "closeConversationSafely — error: ${e.message}") } + + // litert-uaf mitigation (crash observed on the fbjni "HybridData Dest" + // GC thread during insight generation). Insight runs churn conversations + // hard: reset -> close -> recreate, back to back. Each finished generation + // leaves fbjni HybridData peers (event/bridge objects) waiting to be + // reclaimed on the GC finalizer thread. Under that churn the reclaim can + // fire LATER, in the middle of the NEXT conversation's active decode, and + // dereference memory that is already gone -> SIGSEGV (fault 0x0101..). + // We are at a quiescent point here: the previous generation is cancelled + // and joined (above) and the next one has not started, so drain the + // reference queue NOW, off the decode path, so those reclaims do not + // overlap live native work. This is a mitigation aimed at the observed + // timing, not a proven root-cause fix - verify on-device before relying on it. + try { + System.gc() + System.runFinalization() + // gc() only ENQUEUES fbjni's phantom-ref reclaims; the "HybridData + // Dest" thread drains them asynchronously. Yield briefly (non-blocking, + // we are in a suspend fun) so that thread runs the reclaims before the + // caller creates the next conversation and starts a fresh decode. + delay(16) + } catch (e: Throwable) { + Log.w(TAG, "closeConversationSafely — gc drain skipped: ${e.message}") + } } private suspend fun cleanupEngine() { diff --git a/docs/plans/audio-mode-progress-captions.md b/docs/plans/audio-mode-progress-captions.md deleted file mode 100644 index cf64fee15..000000000 --- a/docs/plans/audio-mode-progress-captions.md +++ /dev/null @@ -1,131 +0,0 @@ -# Plan: Phase-aware progress captions for audio mode (no "is it broken?" gaps) - -## Goal -When a user sends a voice message, several silent gaps occur (prefill, thinking, -TTS synthesis). Today all of them show the same pulsing dots, so the user can't -tell working-but-slow from stuck. Add a short status caption that names the phase, -designed so it can never flicker, reverse, or get stranded. - -## Non-goal -Do NOT build a new independent state machine or timers. The caption must be a pure -function of state that already exists, gated by the same condition that mounts the -in-progress bubble, so it cannot outlive or contradict that bubble. - ---- - -## The gaps (current behavior) - -| Phase | Trigger | Shown now | -|---|---|---| -| A. Prefill | sent → before first token (~3-5s, audio TTFT) | pulsing dots, silent | -| B. Thinking | reasoning tokens stream (thinking enabled); never spoken | same dots, silent | -| C. Synthesis | answer streaming; Kokoro synthesizing first sentence | play button + tiny spinner, silent | -| D. Playing | audio plays | waveform animates (good) | - -Streaming TTS (`pro/audio/index.ts` `audio.onStreamingToken` → `feedStreamingText`) -speaks sentence-by-sentence, so C and the tail of answer-generation OVERLAP. The -design must tolerate overlap rather than force a linear phase. - ---- - -## Design: monotonic phase, pure-derived, playback wins - -### Signal sources (read-only; do not add new state) -- Message: `msg.isThinking`, `msg.reasoningContent`, `msg.content`. -- TTS store (`pro/audio/ttsStore.ts`): `playbackStatus`, `currentMessageId`, derived `isSpeaking`/`isLoading`. -- The bubble already mounts on `isStreamingThis || isThinkingItem` - (`pro/audio/ui/MessageAudioMode.tsx` `renderAudioInProgress`). - -### Phase ladder (advance-only) -``` -0 waiting → "Processing your message…" (in-progress, no reasoning, no answer yet) -1 thinking → "Thinking…" (reasoningContent growing AND content empty) -2 answering → "Preparing audio…" (content non-empty, TTS not yet playing THIS msg) -3 playing → (no caption; waveform owns UI)(currentMessageId===msg.id && playing) -``` -Rules that kill the failure modes: -1. **Monotonic.** Keep a ref of the highest phase reached for this messageId; never - render a lower phase even if a signal momentarily regresses. (Prevents flicker / - backward "Preparing→Thinking".) -2. **Playback wins.** If `ttsStore.currentMessageId === msg.id` and status is - playing/processing → phase 3, caption empty, waveform shows progress. Never draw a - caption over real audio. -3. **Cross-message gate.** Only read `playbackStatus` when - `currentMessageId === msg.id`; otherwise treat as not-playing. (Prevents a prior - message's TTS state bleeding onto this bubble.) -4. **Lifetime = bubble.** Caption is rendered only inside the in-progress bubble - (gated on `isStreamingThis || isThinkingItem`). On error/abort/finalize the bubble - unmounts → caption gone. It can never strand on "Thinking…". -5. **Graceful "thinking" detection.** Only show "Thinking…" when reasoning is actively - growing AND answer still empty. If the model's reasoning format is unparseable - (see `buildAudioBubbleProps` note), fall back to "Responding…" rather than assert a - phase. Never block the answer on a wrong thinking guess. - -### Why this avoids the documented risks -- Flicker/reverse → blocked by monotonic ref (rule 1). -- Overlap of generate+speak → playback-wins (rule 2) yields to the waveform. -- Singleton bleed → message-id gate (rule 3). -- Stuck terminal state → lifetime tied to bubble (rule 4); no independent state to leak. -- Parser fragility → graceful fallback (rule 5). - ---- - -## Implementation - -### 1. New hook: `useAudioProgressPhase(msg)` — `pro/audio/ui/AudioMessageBubble/useAudioProgressPhase.ts` -- Subscribe narrowly to `ttsStore`: `currentMessageId`, `playbackStatus` (select only - these two to limit re-renders). -- Compute raw phase from the ladder above using `msg` + gated TTS read. -- Hold `useRef(maxPhase)` keyed by `msg.id`; clamp raw phase up to it (reset the ref - when `msg.id` changes). -- Return `{ phase, caption }` where caption is '' for phase 3. -- Copy (brand voice — plain, no exclamation, no em dash): - - 0 → `Processing your message…` - - 1 → `Thinking…` - - 2 → `Preparing audio…` - - fallback → `Responding…` - -### 2. `AudioMessageBubble/index.tsx` -- Accept optional `statusCaption?: string` (or call the hook directly when `isLoading`). -- In the `isLoading && !isUser` branch, render the caption as a META-styled `Text` - beside/under `ThinkingDots`. Keep dots animating (moving indicator + label). -- Use `TYPOGRAPHY.meta` + `colors.textMuted`. No new colors, no hardcoded sizes. - -### 3. `MessageAudioMode.tsx` -- In `renderAudioInProgress`, pass the computed caption into `AudioMessageBubble` - (or let the bubble call the hook; either is fine since it's pure). - -### 4. Instant bubble on send (close Gap A's "nothing on screen") -- Verify the assistant in-progress placeholder (`msg.isThinking` item or streaming - message) is added to the store IMMEDIATELY on send, before prefill completes. If - there's a delay, the user sees only their own bubble + silence during prefill. -- Trace: core generation path that creates the thinking/streaming placeholder - (`src/services/generationService*`, `useChatGenerationActions`). If the placeholder - is created only on first token, add it at generation start for audio mode. Confirm - on-device that the dots+caption appear within ~100ms of sending. - ---- - -## Tests (both unit + integration, per project rules) - -Unit (`__tests__/unit/...`): -- `useAudioProgressPhase`: returns correct caption per signal combo. -- Monotonic clamp: feed phase 2 then a regressing signal → still phase 2. -- Cross-message gate: `currentMessageId` = other id → playback ignored. -- Playback wins: this-id + playing → phase 3, empty caption. -- Unparseable thinking → "Responding…", never blocks. -- Reset on `msg.id` change → maxPhase ref resets. - -Integration (`__tests__/integration/...`): -- Simulated send → prefill → thinking → answer → TTS preparing → playing: caption - advances 0→1→2→'' and never regresses. -- Error mid-thinking: in-progress bubble unmounts, no stranded caption. - ---- - -## Risk register (carry into review) -- Re-render cost: select only 2 TTS fields; memoize caption. Verify no per-token - re-render storm of all bubbles. -- Streaming vs fallback TTS: "Preparing audio…" may flash briefly (streaming) or - linger (fallback) — acceptable; both read as "working". -- Copy review: run the brand-voice checklist on the four strings before commit. diff --git a/docs/plans/desktop-parity-roadmap.md b/docs/plans/desktop-parity-roadmap.md deleted file mode 100644 index 500c54b17..000000000 --- a/docs/plans/desktop-parity-roadmap.md +++ /dev/null @@ -1,103 +0,0 @@ -# Mobile <- Desktop Parity Roadmap - -Status: Planning -Goal: Bring the mobile app to feature parity with the desktop app, plus mobile's own -native bet (offline recordings). This doc is the full picture across epics; sprint -assignment and estimates are decided separately in Jira. - -## Two tracks - -1. Mobile-native bet: Offline Recordings & Transcriptions - (see `offline-recordings.md`). Mobile's analog of the desktop meeting recorder. -2. Desktop parity: close the capability gaps where mobile lags desktop. - -## Parity status (full map) - -### Already at parity - no work -Chat + vision, image generation, voice STT (Whisper), tools/MCP, projects + RAG, -model catalog/downloads, Keygen licensing. - -TTS / Audio Mode - BUILT and shipped in the `mobile-pro` submodule -(`pro/audio/`): Kokoro + OuteTTS + Qwen3 engines behind an EngineRegistry, full -`ttsService` lifecycle (download/load/generate/save/speak/stop/cache), streaming -playback, waveforms, and complete audio-mode UI. At polish stage (recent iOS playback -fix). NOTE: the public `TTS_IMPLEMENTATION_PLAN.md` is stale and says "NOT STARTED" - -trust the pro code. Only follow-up: the audio module has no tests yet (quality task, -not a feature gap). - -### Gaps - candidate epics -| Desktop capability | Mobile today | Epic | -|--------------------|--------------|------| -| Meeting recorder | none | A: Offline Recordings (planned) | -| Personas (assistants w/ memory + integrations) | none in pro yet | C: Personas | -| Artifacts / canvas (HTML/React/SVG/Mermaid render) | none | D: Artifacts | -| Local OpenAI-compatible gateway (serve over LAN) | none | E: On-phone server (legacy SCRUM-150, SCRUM-157) | -| Clipboard manager | basic copy only | F: Clipboard (low priority) | - -### OS-constrained desktop features - kept in-scope for now -Rely on capabilities macOS exposes but iOS/Android restrict. Treated as doable for -planning; the OS constraint is a risk to resolve (research spike) before build, not a -hard exclusion yet. -| Desktop capability | Constraint | Epic | -|--------------------|-----------|------| -| Screen capture -> OCR -> entities | No continuous background screenshot on iOS/Android | G: Capture loop (spike first) | -| Day / Replay / Reflect | Fed by capture; data model ports, input source does not | H: Memory/Reflect (depends on G) | -| System / call-audio capture | OS-restricted; mobile recordings are mic-only | folded into Recordings risk notes | - -## Build sequence (active order; not sprint-bound) - -A -> G -> H -> C -> D -> E -> F - -(B / TTS is already shipped, so it is not in the build sequence - it sits under -"already at parity".) - -1. A - Offline Recordings & Transcriptions (mobile-native; in flight / next) -2. G - Capture loop (research spike first to define mobile feasibility) -3. H - Memory / Day / Replay / Reflect (depends on G's outcome) -4. C - Personas -5. D - Artifacts / Canvas -6. E - On-phone OpenAI-compatible server -7. F - Clipboard manager (low priority) - -## Epics overview - -### Epic A - Offline Recordings & Transcriptions (mobile-native) -Detailed in `offline-recordings.md`. Record (fg + bg) -> chunked Whisper transcription --> LLM summary + action items. Pro-gated. Reuse note: `pro/audio/recordBridge.ts` -already bridges mic input for audio mode. - -### Epic G - Capture loop (research spike first) -Screen/context capture -> OCR -> observations + entities. Desktop-style "sees your -work" loop. Mobile feasibility uncertain (no background screenshot API); start with a -spike to define what is achievable (share-sheet capture, manual screenshots, -accessibility APIs) before committing build stories. - -### Epic H - Memory / Day / Replay / Reflect -Journal (Day), timeline (Replay), analytics (Reflect). Data structures port directly -from desktop; the input source depends on Epic G. Sequenced after G. - -### Epic C - Personas -Named assistants with system prompt, memory (cross-conversation RAG), capabilities -(text/voice/vision/image/RAG), and skills/integrations. Plan exists -(`PERSONAS_IMPLEMENTATION_PLAN.md`). No personas module in pro yet - genuine gap. - -### Epic D - Artifacts / Canvas -Render model output as HTML / React-JSX / SVG / Mermaid / Markdown in a sandboxed -webview. Pure RN, highly portable from desktop. - -### Epic E - On-phone OpenAI-compatible server -Expose local models as an OpenAI-compatible API over the home network so other -devices/apps can use the phone's models. Parity with desktop gateway. Maps to legacy -tickets SCRUM-150 (Android server) and SCRUM-157 (OpenAI-compatible API). - -### Epic F - Clipboard manager (low priority) -Searchable on-device clipboard history. Desktop has a `@offgrid/clipboard` engine to -reuse. Lower user value on mobile; parked low. - -## Notes for the team -- Pro-gating reuses `proLicenseService` (shared Keygen account with desktop; one - license already spans platforms). -- Reuse-first: desktop packages (`@offgrid/rag`, `@offgrid/models`, `@offgrid/clipboard`), - the `mobile-pro` audio module, and desktop plan docs are the reference; do not fork. -- Epics G/H carry real OS-feasibility risk - resolve via spike before sizing build work. -- TTS audio module needs test coverage added (repo mandates unit + integration tests). diff --git a/docs/plans/offline-recordings.md b/docs/plans/offline-recordings.md deleted file mode 100644 index 4b283c0f2..000000000 --- a/docs/plans/offline-recordings.md +++ /dev/null @@ -1,116 +0,0 @@ -# Offline Recordings & Transcriptions - -Status: Planning -Epic: Offline Recordings & Transcriptions (single epic, all stories inside) -Tier: Pro-gated - -## What this is - -A Pro surface in the mobile app to record audio on-device, transcribe it locally -with Whisper, and generate an LLM title, summary, and action items. Recording works -in the foreground and in the background (lock screen). Nothing leaves the phone. - -This is mobile's native analog of the desktop meeting recorder. The desktop recorder -relies on macOS ScreenCaptureKit + system-audio loopback, which iOS and Android do not -expose. Mobile therefore captures microphone audio rather than call/system audio. - -## Why this is mostly orchestration, not new capability - -The core primitives already exist in the codebase: - -- Recording: `src/services/audioRecorderService.ts` records 16 kHz mono WAV to disk - (foreground today). -- Transcription: `src/services/whisperService.ts` `transcribeFile(path)` transcribes - any audio file with progress callbacks. -- Summarization: `src/services/generationService.ts` runs the active LLM; reuse it with - a summary prompt. -- Persistence/metadata: `chatStore` already models audio fields (audioPath, - waveformData, audioDurationSeconds); Whisper model download/management is solved. - -The genuinely new work: the Recordings product surface, the record -> transcribe -> -summarize orchestration and persistence, background capture (the heavy native piece), -Pro-gating, and storage management. - -## Decisions locked - -- Transcription trigger: automatic after recording stops, with a setting to disable - (auto-with-toggle). -- Long audio: chunked transcription with a progress indicator (handles long meetings). - Confirm whisper.rn practical segment limits during story 4. -- Summary depth: structured summary + extracted action items (title, TL;DR, bullets, - action items). -- Background recording: in scope (iOS background-audio + AVAudioSession; Android - foreground service + mic notification). - -## Stories (single epic - no sprint assignment yet) - -1. Recordings data layer - `recordingStore` (Zustand) + SQLite table: - `id, title, audioPath, durationSeconds, createdAt, transcript, summary, - actionItems, status`. Foundation for all other stories. - -2. Record screen (foreground) - Start/stop, elapsed timer, live amplitude meter (react-native-audio-api), - save WAV to `Documents/recordings/`. - -3. Recordings list + playback - New tab. List by date, play/pause/seek, delete a recording. - -4. Transcription orchestration - On stop (when auto enabled) -> `transcribeFile` with progress UI. Chunk long audio - into sequential segments. Persist transcript. Handle model-not-loaded. - -5. LLM summary + action items - Transcript -> summary prompt -> structured title / TL;DR / bullets / action items. - Re-run summary on demand. Reuses `generationService`. - -6. Recording detail screen - Tabbed Transcript / Summary view, copy, export as text. - -7. Background recording (iOS + Android) - iOS background-audio mode + AVAudioSession; Android foreground service with a - persistent mic notification (FOREGROUND_SERVICE_MICROPHONE on Android 14+). - Interruption handling (incoming call, Siri, app kill/restart). - Heaviest, highest-risk story. Native work on both platforms. - -8. Pro-gating - Gate the whole surface behind the Pro license via `proLicenseService`; upsell entry - point for non-Pro users. Land before story 7 so native work isn't redone. - -9. Storage management - Size display, bulk delete, quota warning. Mirrors desktop retention behavior. - -10. Settings + auto-transcribe toggle - Setting to enable/disable auto transcription; transcription language; clear-cache. - -11. QA, edge cases, polish - Interrupted-recording recovery, permissions flows, no-model prompt, brand-voice - copy pass, design-token compliance, unit + integration tests per repo conventions - (eslint + tsc + tests; Gemini/Codecov/Sonar gates green). - -## Dependencies and sequencing notes - -- Story 1 (data layer) blocks everything. -- Stories 2 -> 3 -> 4 -> 5 -> 6 form the core foreground happy path. -- Story 8 (Pro-gating) should land before story 7 (background recording) so the - expensive native work is not restructured for gating later. -- Story 7 is ~the largest effort and carries App Store / Android-policy review risk. - Even inside one epic, treat it as separable: stories 1-6 + 8-11 deliver a complete, - shippable foreground feature without it. - -## Out of scope (flag for stakeholders) - -- System / call-audio capture (OS-restricted on iOS and Android). -- Speaker diarization (who-said-what). -- Cross-device sync of recordings (would route through Off Grid Sync later). - -## Reuse map (build on, do not fork) - -| Need | Existing | -|------|----------| -| Record WAV | `audioRecorderService.ts` | -| Transcribe file | `whisperService.transcribeFile()` | -| Summarize | `generationService.ts` | -| Audio metadata shape | `chatStore` audio fields | -| Whisper model mgmt | existing model manager + whisper store | -| Pro license check | `proLicenseService.ts` | diff --git a/ios/OffgridMobile/Info.plist b/ios/OffgridMobile/Info.plist index a5fff7a7b..259ef6eed 100644 --- a/ios/OffgridMobile/Info.plist +++ b/ios/OffgridMobile/Info.plist @@ -18,6 +18,10 @@ $(PRODUCT_NAME) CFBundlePackageType APPL + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleSignature + ???? CFBundleURLTypes @@ -29,19 +33,15 @@ - CFBundleShortVersionString - $(MARKETING_VERSION) - CFBundleSignature - ???? CFBundleVersion $(CURRENT_PROJECT_VERSION) - LSRequiresIPhoneOS - LSApplicationQueriesSchemes twitter x + LSRequiresIPhoneOS + NSAppTransportSecurity NSAllowsLocalNetworking @@ -53,6 +53,10 @@ _ollama._tcp _lmstudio._tcp + NSCalendarsFullAccessUsageDescription + Used to read and create calendar events on your request. + NSCalendarsUsageDescription + Used to read and create calendar events on your request. NSCameraUsageDescription This app needs access to your camera to take photos and attach them to conversations. NSFaceIDUsageDescription @@ -65,10 +69,6 @@ This app needs permission to save generated images to your photo library. NSPhotoLibraryUsageDescription This app needs access to your photo library to attach images to conversations. - NSCalendarsUsageDescription - Used to read and create calendar events on your request. - NSCalendarsFullAccessUsageDescription - Used to read and create calendar events on your request. NSSpeechRecognitionUsageDescription This app uses on-device speech recognition to transcribe voice input. RCTNewArchEnabled @@ -95,6 +95,10 @@ SimpleLineIcons.ttf FontAwesome6_Brands.ttf + UIBackgroundModes + + audio + UILaunchStoryboardName LaunchScreen UIRequiredDeviceCapabilities diff --git a/ios/Podfile b/ios/Podfile index bf037c631..496cffee9 100644 --- a/ios/Podfile +++ b/ios/Podfile @@ -23,6 +23,11 @@ target 'OffgridMobile' do :app_path => "#{Pod::Config.instance.installation_root}/.." ) + # onnxruntime-objc ships no module map; OffgridPro (a Swift pod) imports it for + # the auto-detect VAD, so it must be built with modular headers when pods are + # statically linked. Without this, pod install fails to integrate the Swift pod. + pod 'onnxruntime-objc', :modular_headers => true + target 'OffgridMobileTests' do inherit! :search_paths end diff --git a/jest.setup.ts b/jest.setup.ts index 88aa8b53b..2dca797d9 100644 --- a/jest.setup.ts +++ b/jest.setup.ts @@ -341,6 +341,10 @@ jest.mock('react-native-device-info', () => ({ isEmulator: jest.fn(() => Promise.resolve(false)), getDeviceId: jest.fn(() => 'test-device-id'), getHardware: jest.fn(() => Promise.resolve('unknown')), + // Power/battery — the pro recorder gates capture on power state. usePowerState is a + // hook (must return synchronously); getPowerState is the imperative form. + getPowerState: jest.fn(() => Promise.resolve({ batteryLevel: 0.8, batteryState: 'unplugged', lowPowerMode: false })), + usePowerState: jest.fn(() => ({ batteryLevel: 0.8, batteryState: 'unplugged', lowPowerMode: false })), })); // react-native-image-picker mock @@ -405,6 +409,33 @@ jest.mock('@react-native-documents/picker', () => ({ }, })); +// @notifee/react-native mock — the pro locket meeting-reminder code (permissions.ts + +// meetingReminders.ts) imports notifee at module load, so WITHOUT this mock +// require('@offgrid/pro') throws "Notifee native module not found" in jsdom/node, pro +// activation aborts, and NO pro slots register (breaking every voice-mode/TTS test that +// mounts the app.root EngineBridge). Enum values mirror notifee's real ones so any value +// comparison in the code holds. +jest.mock('@notifee/react-native', () => ({ + __esModule: true, + default: { + getNotificationSettings: jest.fn(async () => ({ authorizationStatus: 1 })), + requestPermission: jest.fn(async () => ({ authorizationStatus: 1 })), + createChannel: jest.fn(async () => 'channel-id'), + createTriggerNotification: jest.fn(async () => 'notif-id'), + displayNotification: jest.fn(async () => 'notif-id'), + cancelTriggerNotification: jest.fn(async () => {}), + getTriggerNotificationIds: jest.fn(async () => []), + getTriggerNotifications: jest.fn(async () => []), + getInitialNotification: jest.fn(async () => null), + onForegroundEvent: jest.fn(() => () => {}), + onBackgroundEvent: jest.fn(() => {}), + }, + AndroidImportance: { DEFAULT: 3, HIGH: 4 }, + AuthorizationStatus: { NOT_DETERMINED: -1, DENIED: 0, AUTHORIZED: 1, PROVISIONAL: 2 }, + TriggerType: { TIMESTAMP: 0, INTERVAL: 1 }, + EventType: { DISMISSED: 0, PRESS: 1, ACTION_PRESS: 2, DELIVERED: 3 }, +})); + // @react-native-documents/viewer mock jest.mock('@react-native-documents/viewer', () => ({ viewDocument: jest.fn(() => Promise.resolve(null)), diff --git a/package-lock.json b/package-lock.json index 2665cac7a..7f48f2c86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@dr.pogodin/react-native-fs": "^2.38.1", "@kesha-antonov/react-native-background-downloader": "^4.5.6", "@modelcontextprotocol/sdk": "^1.29.0", + "@notifee/react-native": "^9.1.8", "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "^2.2.0", "@react-native-community/slider": "^5.1.2", @@ -4335,6 +4336,15 @@ "node": ">= 8" } }, + "node_modules/@notifee/react-native": { + "version": "9.1.8", + "resolved": "https://registry.npmjs.org/@notifee/react-native/-/react-native-9.1.8.tgz", + "integrity": "sha512-Az/dueoPerJsbbjRxu8a558wKY+gONUrfoy3Hs++5OqbeMsR0dYe6P+4oN6twrLFyzAhEA1tEoZRvQTFDRmvQg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native": "*" + } + }, "node_modules/@op-engineering/op-sqlite": { "version": "15.2.5", "resolved": "https://registry.npmjs.org/@op-engineering/op-sqlite/-/op-sqlite-15.2.5.tgz", diff --git a/package.json b/package.json index c1995200b..10a553db2 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@dr.pogodin/react-native-fs": "^2.38.1", "@kesha-antonov/react-native-background-downloader": "^4.5.6", "@modelcontextprotocol/sdk": "^1.29.0", + "@notifee/react-native": "^9.1.8", "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "^2.2.0", "@react-native-community/slider": "^5.1.2", diff --git a/patches/whisper.rn+0.5.5.patch b/patches/whisper.rn+0.5.5.patch index ebb100cc7..5a7ae3b05 100644 --- a/patches/whisper.rn+0.5.5.patch +++ b/patches/whisper.rn+0.5.5.patch @@ -79,3 +79,67 @@ index b1fa548..5f0b7f1 100644 if (job->audio_output_path != nullptr) { RNWHISPER_LOG_INFO("job->params.language: %s\n", job->params.language); std::vector slice_n_samples_vec; +diff --git a/node_modules/whisper.rn/ios/RNWhisper.mm b/node_modules/whisper.rn/ios/RNWhisper.mm +index 27908d4..3292b68 100644 +--- a/node_modules/whisper.rn/ios/RNWhisper.mm ++++ b/node_modules/whisper.rn/ios/RNWhisper.mm +@@ -505,6 +505,7 @@ - (void)transcribeData:(RNWhisperContext *)context + float *data = [RNWhisperAudioUtils decodeWaveData:pcmData count:&count cutHeader:NO]; + + NSArray *segments = [vadContext detectSpeech:data samplesCount:count options:options]; ++ if (data != nil) free(data); // decodeWaveFile/Data malloc this PCM buffer; detectSpeech consumes it synchronously. Without this it leaks ~one slice of float PCM per call and OOMs over a long file. + resolve(segments); + } + +@@ -541,6 +542,7 @@ - (void)transcribeData:(RNWhisperContext *)context + } + + NSArray *segments = [vadContext detectSpeech:data samplesCount:count options:options]; ++ if (data != nil) free(data); // decodeWaveFile/Data malloc this PCM buffer; detectSpeech consumes it synchronously. Without this it leaks ~one slice of float PCM per call and OOMs over a long file. + resolve(segments); + } + +diff --git a/node_modules/whisper.rn/ios/RNWhisperContext.mm b/node_modules/whisper.rn/ios/RNWhisperContext.mm +index 13a880f..4ccf878 100644 +--- a/node_modules/whisper.rn/ios/RNWhisperContext.mm ++++ b/node_modules/whisper.rn/ios/RNWhisperContext.mm +@@ -419,6 +419,24 @@ - (void)transcribeData:(int)jobId + + whisper_full_params params = [self createParams:options jobId:jobId]; + ++ // Hoisted to the dispatch-block scope so it outlives the `if` below and ++ // stays valid through fullTranscribe. It was declared inside the ++ // `if (onNewSegments)` block but its address is handed to whisper as the ++ // segment-callback context; once that `if` closed, the stack struct was ++ // dead, and the first segment callback (~first 30s window) dereferenced a ++ // dangling pointer -> deterministic native crash on iOS. (onProgress was ++ // unaffected: it passes the block directly, no struct.) ++ struct rnwhisper_segments_callback_data user_data = { ++ .onNewSegments = onNewSegments, ++ .tdrzEnable = options[@"tdrzEnable"] && [options[@"tdrzEnable"] boolValue], ++ .total_n_new = 0, ++ }; ++ // Marker proving this binary carries the whisper.rn+0.5.5 segment-callback ++ // lifetime patch. If you DON'T see this line in the device log when iOS ++ // streaming is enabled, the running app was built without the patch (likely ++ // a Metro reload over a stale binary) and the live callback will crash. ++ NSLog(@"[RNWhisper][PATCH-LIVE] segment-callback user_data hoisted (whisper.rn+0.5.5 lifetime patch compiled in)"); ++ + if (options[@"onProgress"] && [options[@"onProgress"] boolValue]) { + params.progress_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) { + void (^onProgress)(int) = (__bridge void (^)(int))user_data; +@@ -463,11 +481,9 @@ - (void)transcribeData:(int)jobId + void (^onNewSegments)(NSDictionary *) = (void (^)(NSDictionary *))data->onNewSegments; + onNewSegments(result); + }; +- struct rnwhisper_segments_callback_data user_data = { +- .onNewSegments = onNewSegments, +- .tdrzEnable = options[@"tdrzEnable"] && [options[@"tdrzEnable"] boolValue], +- .total_n_new = 0, +- }; ++ // user_data is declared at the dispatch-block scope above so it stays ++ // alive through fullTranscribe (declaring it here, inside the if, left a ++ // dangling pointer once this block closed). + params.new_segment_callback_user_data = &user_data; + } + diff --git a/pro b/pro index ff0d87423..c031a33a3 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit ff0d874234c23d3dd2a781b77baafe8102c3fad7 +Subproject commit c031a33a3f05b0e119b513ac2fc0e89d1b6d62e5 diff --git a/react-native.config.js b/react-native.config.js new file mode 100644 index 000000000..75d366f64 --- /dev/null +++ b/react-native.config.js @@ -0,0 +1,36 @@ +const fs = require('fs'); +const path = require('path'); + +// Autolink the pro submodule's native library ONLY when it is actually on +// disk. Mirrors the fs.existsSync(pro) guard metro.config.js uses for the pro +// JS: a public clone without the private submodule sees an empty/absent pro/ +// dir, this entry is omitted, and the open build compiles with no pro native. +// +// IMPORTANT: check a real file inside pro/, never just the pro/ directory - an +// uninitialised submodule leaves an empty pro/ folder behind. +const proRoot = path.resolve(__dirname, 'pro'); +const proAndroidGradle = path.join(proRoot, 'android', 'build.gradle'); +const proPodspec = path.join(proRoot, 'ios', 'OffgridPro.podspec'); +const proHasNative = fs.existsSync(proAndroidGradle); + +module.exports = { + dependencies: { + ...(proHasNative + ? { + '@offgrid/pro': { + root: proRoot, + platforms: { + android: { + sourceDir: path.join(proRoot, 'android'), + packageImportPath: 'import ai.offgridmobile.alwayson.AlwaysOnTranscriptionPackage;', + packageInstance: 'new AlwaysOnTranscriptionPackage()', + }, + ios: { + podspecPath: proPodspec, + }, + }, + }, + } + : {}), + }, +}; diff --git a/src/bootstrap/slotRegistry.ts b/src/bootstrap/slotRegistry.ts index 5866a3b17..756c873cc 100644 --- a/src/bootstrap/slotRegistry.ts +++ b/src/bootstrap/slotRegistry.ts @@ -78,4 +78,8 @@ export const SLOTS = { * download/management). The tab itself only appears when this is * registered, so free builds show just Text/Image. */ modelsScreenVoiceTab: 'modelsScreen.voiceTab', + /** Compact recorder entry on the Home screen (Pro): a tap-to-record card that + * starts/stops the recorder and links to the recordings list. Replaces the + * old dedicated Recorder tab. Absent in free builds. */ + homeRecorder: 'home.recorder', } as const; diff --git a/src/components/ChatInput/Attachments.tsx b/src/components/ChatInput/Attachments.tsx index e8b73e24a..4acd1db37 100644 --- a/src/components/ChatInput/Attachments.tsx +++ b/src/components/ChatInput/Attachments.tsx @@ -2,13 +2,14 @@ import React, { useState, useRef } from 'react'; let _attachmentIdSeq = 0; const nextAttachmentId = () => `${Date.now()}-${(++_attachmentIdSeq).toString(36)}`; -import { View, Text, Image, ScrollView, TouchableOpacity, Platform, ActionSheetIOS } from 'react-native'; +import { View, Text, Image, ScrollView, TouchableOpacity, Platform, ActionSheetIOS, ActivityIndicator } from 'react-native'; import { launchImageLibrary, launchCamera, Asset } from 'react-native-image-picker'; import { pick, types, isErrorWithCode, errorCodes } from '@react-native-documents/picker'; import Icon from 'react-native-vector-icons/Feather'; import { useTheme, useThemedStyles } from '../../theme'; import { MediaAttachment } from '../../types'; import { documentService } from '../../services/documentService'; +import { takePendingChatAttachments } from '../../services/chatAttachmentInbox'; import { audioSessionManager } from '../../services/audioSessionManager'; import { AlertState, showAlert, hideAlert } from '../CustomAlert'; import { createStyles } from './styles'; @@ -17,7 +18,9 @@ import { isPickerStuck } from '../../utils/pickerErrorUtils'; // ─── useAttachments hook ────────────────────────────────────────────────────── export function useAttachments(setAlertState: (state: AlertState) => void) { - const [attachments, setAttachments] = useState([]); + // Seed from the inbox (e.g. a transcript handed off by the Pro recorder's + // "Attach to chat"), consumed once on mount. + const [attachments, setAttachments] = useState(() => takePendingChatAttachments()); const isPickingRef = useRef(false); const addAttachments = (assets: Asset[]) => { @@ -157,13 +160,17 @@ export function useAttachments(setAlertState: (state: AlertState) => void) { interface AttachmentPreviewProps { attachments: MediaAttachment[]; onRemove: (id: string) => void; + // Summarize a document/transcript attachment that may be too large for the + // context window. Optional so other ChatInput consumers can omit it. + onSummarize?: (attachment: MediaAttachment) => void; + summarizingId?: string | null; /** Tapping an image thumbnail opens the shared fullscreen image viewer (same * handler the in-message generated/attached images use). Optional so the * component still renders without a viewer wired up. */ onImagePress?: (uri: string) => void; } -export const AttachmentPreview: React.FC = ({ attachments, onRemove, onImagePress }) => { +export const AttachmentPreview: React.FC = ({ attachments, onRemove, onSummarize, summarizingId, onImagePress }) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); @@ -177,42 +184,73 @@ export const AttachmentPreview: React.FC = ({ attachment contentContainerStyle={styles.attachmentsContent} showsHorizontalScrollIndicator={false} > - {attachments.map(attachment => ( - - {attachment.type === 'image' ? ( + {attachments.map(attachment => { + const canSummarize = !!onSummarize && !!attachment.textContent && attachment.type !== 'image'; + const isBusy = summarizingId === attachment.id; + return ( + + {attachment.type === 'image' ? ( + onImagePress?.(attachment.uri)} + > + + + ) : attachment.type === 'audio' ? ( + + + Voice + + ) : ( + + + + + {attachment.fileName || 'Document'} + + + {canSummarize ? ( + isBusy ? ( + + + Summarizing + + ) : ( + onSummarize!(attachment)} + activeOpacity={0.8} + > + + Summarize + + ) + ) : null} + + )} onImagePress?.(attachment.uri)} + testID={`remove-attachment-${attachment.id}`} + style={styles.removeAttachment} + onPress={() => onRemove(attachment.id)} > - + × - ) : attachment.type === 'audio' ? ( - - - Voice - - ) : ( - - - - {attachment.fileName || 'Document'} - - - )} - onRemove(attachment.id)} - > - × - - - ))} + + ); + })} ); }; diff --git a/src/components/ChatInput/index.tsx b/src/components/ChatInput/index.tsx index 42e8ac2d0..6858411ee 100644 --- a/src/components/ChatInput/index.tsx +++ b/src/components/ChatInput/index.tsx @@ -1,3 +1,6 @@ +/* eslint-disable max-lines -- 520 lines. Combines two independent attachment + features that landed on separate branches (document Summarize + image tap-to-view). + Extracting the attachment toolbar into its own component is deferred. */ import React, { useState, useRef, useEffect } from 'react'; import { View, TextInput, TouchableOpacity, Animated, StyleSheet, Platform, ActionSheetIOS } from 'react-native'; import Icon from 'react-native-vector-icons/Feather'; @@ -13,6 +16,7 @@ import { CustomAlert, showAlert, hideAlert, AlertState, initialAlertState } from import { createStyles, PILL_ICON_SIZE, ANIM_DURATION_IN, ANIM_DURATION_OUT } from './styles'; import { QueueRow } from './Toolbar'; import { AttachmentPreview, useAttachments } from './Attachments'; +import { useSummarizeAttachment } from './useSummarizeAttachment'; import { useVoiceInput } from './Voice'; import { buildVoiceNoteHandlers } from './voiceNoteSend'; import { QuickSettingsPopover, AttachPickerPopover } from './Popovers'; @@ -67,6 +71,67 @@ const IMAGE_MODE_CYCLE: ImageModeState[] = ['auto', 'force', 'disabled']; // (collapsing) row — it's rendered persistently above the input instead. const computePillIconsWidth = (): number => PILL_ICON_SIZE * 2; +// ─── Send / Stop / Voice button ───────────────────────────────────────────── +// The trailing circle button: Send when there's something to send, Stop while +// generating, otherwise the voice-record button. Extracted so the main +// component stays within the max-lines-per-function budget; behaviour is +// identical to the previous inline ternary. +interface ActionButtonProps { + canSend: boolean; + isGenerating?: boolean; + disabled?: boolean; + onStop?: () => void; + onSendPress: () => void; + onStopPress: () => void; + isRecording: boolean; + voiceAvailable: boolean; + isModelLoading: boolean; + isTranscribing: boolean; + partialResult: string; + error: string | null; + onStartRecording: () => void; + onStopRecording: () => void; + onCancelRecording: () => void; +} + +const ActionButton: React.FC = (props) => { + const { colors } = useTheme(); + const styles = useThemedStyles(createStyles); + if (props.canSend) { + return ( + + + + ); + } + if (props.isGenerating && props.onStop) { + return ( + + + + ); + } + return ( + + ); +}; + /** * Alert shown when the user attaches an image to a model without vision support. * Remote (server) models have no local vision-projector file to repair, so the @@ -159,6 +224,11 @@ export const ChatInput: React.FC = ({ const { attachments, removeAttachment, clearAttachments, handlePickImage, handlePickDocument, addAudioAttachment } = useAttachments(setAlertState); attachmentsRef.current = attachments; + const { summarizingId, handleSummarize } = useSummarizeAttachment(); + const onSummarizeAttachment = async (attachment: MediaAttachment) => { + await handleSummarize(attachment); + removeAttachment(attachment.id); + }; const interfaceMode = useUiModeStore((s) => s.interfaceMode); const isAudioMode = interfaceMode === 'audio'; @@ -323,32 +393,20 @@ export const ChatInput: React.FC = ({ // Pro-only inline Chat↔Audio toggle (empty slot in free builds → null). const pillIconsExpandedWidth = computePillIconsWidth(); - const actionButton = canSend ? ( - - - - ) : isGenerating && onStop ? ( - - - - ) : ( - = ({ return ( - + ({ borderRadius: 8, overflow: 'hidden' as const, }, + // Wider, taller chip for document/transcript attachments so the file name and + // the Summarize action are both fully visible (the square image size clipped + // the button). + attachmentPreviewDoc: { + width: 168, + height: 76, + }, attachmentImage: { width: '100%' as const, height: '100%' as const, @@ -42,6 +49,17 @@ export const createStyles = (colors: ThemeColors, _shadows: ThemeShadows) => ({ alignItems: 'center' as const, padding: 4, }, + documentPreviewDoc: { + justifyContent: 'space-between' as const, + alignItems: 'stretch' as const, + padding: 8, + paddingRight: 22, + }, + documentNameRow: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + gap: 6, + }, documentName: { fontSize: 10, fontFamily: FONTS.mono, @@ -49,6 +67,33 @@ export const createStyles = (colors: ThemeColors, _shadows: ThemeShadows) => ({ textAlign: 'center' as const, marginTop: 4, }, + summarizeButton: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + justifyContent: 'center' as const, + gap: 4, + paddingHorizontal: SPACING.sm, + paddingVertical: 5, + borderRadius: 8, + backgroundColor: colors.primary, + }, + summarizeButtonText: { + fontSize: 11, + fontFamily: FONTS.mono, + color: colors.background, + }, + summarizeBusy: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + justifyContent: 'center' as const, + gap: 6, + paddingVertical: 4, + }, + summarizeBusyText: { + fontSize: 11, + fontFamily: FONTS.mono, + color: colors.primary, + }, removeAttachment: { position: 'absolute' as const, top: 2, diff --git a/src/components/ChatInput/useSummarizeAttachment.ts b/src/components/ChatInput/useSummarizeAttachment.ts new file mode 100644 index 000000000..d3e474f61 --- /dev/null +++ b/src/components/ChatInput/useSummarizeAttachment.ts @@ -0,0 +1,124 @@ +import { useState } from 'react'; +import { MediaAttachment } from '../../types'; +import { transcriptSummarizer } from '../../services'; +import { useChatStore, useAppStore } from '../../stores'; +import logger from '../../utils/logger'; + +/** Throttle for streaming the summary into the message (~20 paints/sec). */ +const STREAM_FLUSH_MS = 50; + +/** mm:ss for a millisecond offset, used to label an attached transcript range. */ +function fmtClock(ms: number): string { + const total = Math.floor(ms / 1000); + const m = Math.floor(total / 60); + const s = total % 60; + return `${m}:${s.toString().padStart(2, '0')}`; +} + +/** + * Summarize an attached document/transcript that is too large to fit the model's + * context window. Posts a user message ("Summarize ") and an assistant + * message, then streams progress into that assistant message (part i of N, + * combining) before replacing it with the final summary. Self-contained: reads + * the active conversation + model from the global stores, so it does not need + * props threaded down from the chat screen. + */ +export function useSummarizeAttachment() { + const [summarizingId, setSummarizingId] = useState(null); + + const handleSummarize = async (attachment: MediaAttachment): Promise => { + if (summarizingId) return; + const text = attachment.textContent?.trim(); + if (!text) return; + + const chat = useChatStore.getState(); + let conversationId = chat.activeConversationId; + if (!conversationId) { + const modelId = useAppStore.getState().activeModelId; + if (!modelId) return; // no model loaded - nothing to summarize with + conversationId = chat.createConversation(modelId); + chat.setActiveConversation(conversationId); + } + + const label = attachment.fileName || 'transcript'; + const range = + attachment.transcriptStartMs != null && attachment.transcriptEndMs != null + ? ` (${fmtClock(attachment.transcriptStartMs)} to ${fmtClock(attachment.transcriptEndMs)})` + : ''; + chat.addMessage(conversationId, { role: 'user', content: `Summarize ${label}${range}` }); + const placeholder = chat.addMessage(conversationId, { role: 'assistant', content: 'Starting...' }); + + setSummarizingId(attachment.id); + // Stream the work in place. The map phase streams each part as it is written + // (so a multi-chunk run shows text from part 1, not a static counter for + // minutes), then the final combine pass restreams the answer over the top. + // updateMessageContent rebuilds the conversations tree on every call, so we + // flush on a ~50ms timer (matching the main generation loop) rather than per + // token, otherwise the JS thread saturates and the UI only paints at the end. + let uiPhase: 'map' | 'final' = 'map'; + let total = 0; + let current = 0; + const doneParts: string[] = []; + let curPart = ''; + let finalText = ''; + let flushTimer: ReturnType | null = null; + + const compose = (): string => { + if (uiPhase === 'final') return finalText || 'Combining the parts...'; + const parts = [...doneParts, curPart].filter((s) => s.trim()); + const header = total > 1 ? `Summarizing part ${current} of ${total}\n\n` : 'Summarizing...\n\n'; + return parts.length ? header + parts.join('\n\n') : header.trim(); + }; + const flush = () => { + flushTimer = null; + useChatStore.getState().updateMessageContent(conversationId!, placeholder.id, compose()); + }; + const scheduleFlush = () => { if (!flushTimer) flushTimer = setTimeout(flush, STREAM_FLUSH_MS); }; + + try { + const summary = await transcriptSummarizer.summarize(text, { + onProgress: (p) => { + if (p.phase === 'chunking') { + total = p.total; + } else if (p.phase === 'mapping') { + if (p.total <= 1) { + uiPhase = 'final'; // single pass: the streamed text is the answer + } else { + if (curPart.trim()) doneParts.push(curPart.trim()); + curPart = ''; + total = p.total; + current = p.current; + } + } else if (p.phase === 'combining') { + if (curPart.trim()) doneParts.push(curPart.trim()); + curPart = ''; + uiPhase = 'final'; + finalText = ''; + } + scheduleFlush(); + }, + onToken: (delta) => { + if (uiPhase === 'final') finalText += delta; + else curPart += delta; + scheduleFlush(); + }, + }); + if (flushTimer) clearTimeout(flushTimer); + // Final trimmed summary (streamed text may have leading/trailing space). + useChatStore.getState().updateMessageContent(conversationId, placeholder.id, summary); + } catch (e) { + if (flushTimer) clearTimeout(flushTimer); + const msg = e instanceof Error ? e.message : 'Summarization failed'; + useChatStore.getState().updateMessageContent( + conversationId, + placeholder.id, + `Could not summarize this transcript.\n\n${msg}`, + ); + logger.warn('[useSummarizeAttachment] failed:', e); + } finally { + setSummarizingId(null); + } + }; + + return { summarizingId, handleSummarize }; +} diff --git a/src/components/DevGrammarModal.tsx b/src/components/DevGrammarModal.tsx new file mode 100644 index 000000000..97e8ea14d --- /dev/null +++ b/src/components/DevGrammarModal.tsx @@ -0,0 +1,271 @@ +import React, { useEffect, useState } from 'react'; +import { + Modal, + View, + Text, + TextInput, + TouchableOpacity, + Switch, + ScrollView, + Pressable, +} from 'react-native'; +import Icon from 'react-native-vector-icons/Feather'; +import { useTheme, useThemedStyles } from '../theme'; +import type { ThemeColors, ThemeShadows } from '../theme'; +import { SPACING, TYPOGRAPHY, FONTS } from '../constants'; +import { useDevInferenceStore } from '../stores/devInferenceStore'; +import logger from '../utils/logger'; + +const STARTER_GRAMMAR = `root ::= "TITLE: " line "\\nSUMMARY: " line "\\nACTIONS:\\n" acts +acts ::= "none\\n" | item+ +item ::= "- " line "\\n" +line ::= [^\\n]+`; + +interface DevGrammarModalProps { + visible: boolean; + onClose: () => void; +} + +/** + * DEV-ONLY test harness: paste a GBNF grammar (plus optional temperature / + * assistant prefill) and apply it to the next chat completion(s). Lets us test + * grammar-constrained / prefill / temp=0 output on the real on-device model + * without leaving the app. Only mounted behind `__DEV__`. + */ +export const DevGrammarModal: React.FC = ({ visible, onClose }) => { + const styles = useThemedStyles(createStyles); + const { colors } = useTheme(); + const store = useDevInferenceStore(); + + // Local drafts so edits aren't live until Apply. Seed from the store on open. + const [grammar, setGrammar] = useState(''); + const [temperature, setTemperature] = useState(''); + const [prefix, setPrefix] = useState(''); + const [maxWords, setMaxWords] = useState(''); + const [litertType, setLitertType] = useState<'json_schema' | 'lark' | 'regex'>('json_schema'); + const [litertConstraint, setLitertConstraint] = useState(''); + const [enabled, setEnabled] = useState(false); + + useEffect(() => { + if (!visible) return; + setGrammar(store.grammar); + setTemperature(store.temperature != null ? String(store.temperature) : ''); + setPrefix(store.assistantPrefix); + setMaxWords(store.maxWords != null ? String(store.maxWords) : ''); + setLitertType(store.litertConstraintType); + setLitertConstraint(store.litertConstraintString); + setEnabled(store.enabled); + // Seed once per open; store fields are intentionally not deps. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [visible]); + + const apply = () => { + const t = temperature.trim(); + const parsedTemp = t.length > 0 ? Number(t) : NaN; + const w = maxWords.trim(); + const parsedWords = w.length > 0 ? Math.round(Number(w)) : NaN; + store.setGrammar(grammar); + store.setTemperature(Number.isFinite(parsedTemp) ? parsedTemp : undefined); + store.setAssistantPrefix(prefix); + store.setMaxWords(Number.isFinite(parsedWords) && parsedWords > 0 ? parsedWords : undefined); + store.setLitertConstraintType(litertType); + store.setLitertConstraintString(litertConstraint); + store.setLastError(undefined); + store.setEnabled(enabled); + logger.log( + `[DevGrammar] ARMED enabled=${enabled} grammarLen=${grammar.trim().length} ` + + `temp=${Number.isFinite(parsedTemp) ? parsedTemp : 'default'} prefill=${prefix ? JSON.stringify(prefix) : 'none'} ` + + `maxWords=${Number.isFinite(parsedWords) && parsedWords > 0 ? parsedWords : 'none'}`, + ); + onClose(); + }; + + const clearAll = () => { + store.clear(); + setGrammar(''); + setTemperature(''); + setPrefix(''); + setMaxWords(''); + setLitertType('json_schema'); + setLitertConstraint(''); + setEnabled(false); + }; + + return ( + + + + + + + Grammar test harness + DEV + + + + + + + + GBNF grammar + + setGrammar(STARTER_GRAMMAR)}> + Insert starter grammar + + + + + Temperature + + + + Max words + + + + + Assistant prefill + + + + LiteRT constraint (LLGuidance) + Used when a LiteRT model is active. Not GBNF - pick a format below. + + {(['json_schema', 'lark', 'regex'] as const).map((t) => ( + setLitertType(t)} + > + {t} + + ))} + + + + + + Enable override + GBNF applies on llama.cpp; the LiteRT constraint applies on LiteRT. Tools off while a grammar is active. + + { logger.log(`[DevGrammar] enable toggle -> ${v}`); setEnabled(v); }} + /> + + + {store.lastError ? ( + Grammar error: {store.lastError} + ) : null} + + + + + Clear + + + Apply + + + + + + ); +}; + +DevGrammarModal.displayName = 'DevGrammarModal'; + +const createStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({ + backdrop: { ...StyleSheetAbsolute, backgroundColor: 'rgba(0,0,0,0.5)' }, + centerWrap: { flex: 1, alignItems: 'center' as const, justifyContent: 'center' as const, padding: SPACING.lg }, + card: { + width: '100%' as const, + maxWidth: 460, + maxHeight: '85%' as const, + backgroundColor: colors.surface, + borderRadius: 14, + padding: SPACING.lg, + ...shadows.medium, + }, + headerRow: { flexDirection: 'row' as const, alignItems: 'center' as const, gap: SPACING.sm, marginBottom: SPACING.md }, + title: { ...TYPOGRAPHY.h3, color: colors.text }, + devBadge: { backgroundColor: `${colors.primary}22`, borderRadius: 5, paddingHorizontal: 5, paddingVertical: 1 }, + devBadgeText: { ...TYPOGRAPHY.labelSmall, color: colors.primary }, + flex: { flex: 1 }, + body: { flexGrow: 0 }, + label: { ...TYPOGRAPHY.label, color: colors.textSecondary, marginBottom: SPACING.xs, marginTop: SPACING.sm }, + input: { + borderWidth: 1, + borderColor: colors.border, + borderRadius: 8, + paddingHorizontal: SPACING.sm, + paddingVertical: SPACING.sm, + color: colors.text, + backgroundColor: colors.background, + ...TYPOGRAPHY.bodySmall, + }, + grammarInput: { minHeight: 120, maxHeight: 220, fontFamily: FONTS.mono, textAlignVertical: 'top' as const }, + starterLink: { ...TYPOGRAPHY.meta, color: colors.primary, marginTop: SPACING.xs }, + twoCol: { flexDirection: 'row' as const, gap: SPACING.md }, + col: { flex: 1 }, + divider: { height: 1, backgroundColor: colors.border, marginTop: SPACING.lg, marginBottom: SPACING.xs }, + sectionLabel: { ...TYPOGRAPHY.label, color: colors.textSecondary, marginTop: SPACING.sm }, + typeRow: { flexDirection: 'row' as const, gap: SPACING.xs, marginTop: SPACING.sm, marginBottom: SPACING.xs }, + typeChip: { paddingHorizontal: SPACING.sm, paddingVertical: 5, borderRadius: 7, borderWidth: 1, borderColor: colors.border }, + typeChipOn: { backgroundColor: `${colors.primary}22`, borderColor: colors.primary }, + typeChipText: { ...TYPOGRAPHY.meta, color: colors.textMuted }, + typeChipTextOn: { color: colors.primary }, + enableRow: { flexDirection: 'row' as const, alignItems: 'center' as const, gap: SPACING.md, marginTop: SPACING.md }, + enableLabel: { ...TYPOGRAPHY.body, color: colors.text }, + hint: { ...TYPOGRAPHY.meta, color: colors.textMuted, marginTop: 2 }, + error: { ...TYPOGRAPHY.bodySmall, color: colors.error, marginTop: SPACING.md }, + actions: { flexDirection: 'row' as const, justifyContent: 'flex-end' as const, gap: SPACING.sm, marginTop: SPACING.lg }, + secondaryBtn: { paddingHorizontal: SPACING.lg, paddingVertical: SPACING.sm, borderRadius: 8, borderWidth: 1, borderColor: colors.border }, + secondaryText: { ...TYPOGRAPHY.body, color: colors.textSecondary }, + primaryBtn: { paddingHorizontal: SPACING.lg, paddingVertical: SPACING.sm, borderRadius: 8, backgroundColor: colors.primary }, + primaryText: { ...TYPOGRAPHY.body, color: colors.background }, +}); + +const StyleSheetAbsolute = { position: 'absolute' as const, top: 0, left: 0, right: 0, bottom: 0 }; diff --git a/src/components/Toast.tsx b/src/components/Toast.tsx new file mode 100644 index 000000000..020c9a486 --- /dev/null +++ b/src/components/Toast.tsx @@ -0,0 +1,144 @@ +import React, { useEffect, useRef } from 'react'; +import { Animated, Text, TouchableOpacity, StyleSheet } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import Icon from 'react-native-vector-icons/Feather'; +import { create } from 'zustand'; +import { useTheme, useThemedStyles } from '../theme'; +import type { ThemeColors, ThemeShadows } from '../theme'; +import { SPACING, TYPOGRAPHY } from '../constants'; + +/** + * One cross-platform toast (not ToastAndroid, which is Android-only): a brief, + * non-blocking message that slides up from the bottom and auto-dismisses. A + * single host is mounted once at the app root; anywhere in the app (screens or + * services) calls `showToast(message)` imperatively - no per-screen wiring. + */ +export interface ToastOptions { + /** Optional leading Feather icon name. */ + icon?: string; + /** Auto-dismiss delay in ms (default 2600). */ + durationMs?: number; +} + +interface ToastState { + visible: boolean; + message: string; + icon?: string; + durationMs: number; + /** Bumped on every show so the host restarts its timer even for the same text. */ + nonce: number; + show: (message: string, opts?: ToastOptions) => void; + hide: () => void; +} + +const DEFAULT_DURATION_MS = 2600; + +const useToastStore = create((set) => ({ + visible: false, + message: '', + icon: undefined, + durationMs: DEFAULT_DURATION_MS, + nonce: 0, + show: (message, opts) => + set((s) => ({ + visible: true, + message, + icon: opts?.icon, + durationMs: opts?.durationMs ?? DEFAULT_DURATION_MS, + nonce: s.nonce + 1, + })), + hide: () => set({ visible: false }), +})); + +/** Show a toast from anywhere (screens or services). */ +export const showToast = (message: string, opts?: ToastOptions): void => + useToastStore.getState().show(message, opts); + +/** Hide the current toast early. */ +export const hideToast = (): void => useToastStore.getState().hide(); + +/** + * The toast host. Mount exactly once near the app root (inside SafeAreaProvider). + * Renders nothing until a toast is shown. + */ +export const Toast: React.FC = () => { + const { colors } = useTheme(); + const styles = useThemedStyles(createStyles); + const insets = useSafeAreaInsets(); + + const visible = useToastStore((s) => s.visible); + const message = useToastStore((s) => s.message); + const icon = useToastStore((s) => s.icon); + const durationMs = useToastStore((s) => s.durationMs); + const nonce = useToastStore((s) => s.nonce); + const hide = useToastStore((s) => s.hide); + + const opacity = useRef(new Animated.Value(0)).current; + const translateY = useRef(new Animated.Value(20)).current; + const timer = useRef | null>(null); + // Keep the last message on screen through the fade-out so it doesn't blank mid-animation. + const [shown, setShown] = React.useState(false); + + useEffect(() => { + if (timer.current) { clearTimeout(timer.current); timer.current = null; } + if (visible) { + setShown(true); + Animated.parallel([ + Animated.timing(opacity, { toValue: 1, duration: 180, useNativeDriver: true }), + Animated.timing(translateY, { toValue: 0, duration: 180, useNativeDriver: true }), + ]).start(); + timer.current = setTimeout(hide, durationMs); + } else { + Animated.parallel([ + Animated.timing(opacity, { toValue: 0, duration: 160, useNativeDriver: true }), + Animated.timing(translateY, { toValue: 20, duration: 160, useNativeDriver: true }), + ]).start(({ finished }) => { if (finished) setShown(false); }); + } + return () => { if (timer.current) { clearTimeout(timer.current); timer.current = null; } }; + // nonce forces re-run (and timer reset) even when message text is unchanged. + }, [visible, nonce, durationMs, hide, opacity, translateY]); + + if (!shown) return null; + + return ( + + + {icon ? : null} + {message} + + + ); +}; + +Toast.displayName = 'Toast'; + +const createStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({ + wrap: { + position: 'absolute' as const, + left: SPACING.lg, + right: SPACING.lg, + alignItems: 'center' as const, + }, + toast: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + maxWidth: '100%' as const, + paddingHorizontal: SPACING.lg, + paddingVertical: SPACING.md, + borderRadius: 12, + backgroundColor: colors.surface, + borderWidth: StyleSheet.hairlineWidth, + borderColor: colors.border, + ...shadows.small, + }, + icon: { marginRight: SPACING.sm }, + text: { ...TYPOGRAPHY.bodySmall, color: colors.text, flexShrink: 1 }, +}); diff --git a/src/components/index.ts b/src/components/index.ts index 67ef174d5..b67228d0e 100644 --- a/src/components/index.ts +++ b/src/components/index.ts @@ -8,8 +8,10 @@ export { ChatInput } from './ChatInput'; ; export { ModelSelectorModal } from './ModelSelectorModal'; export { GenerationSettingsModal } from './GenerationSettingsModal'; +export { Toast, showToast, hideToast } from './Toast'; +export type { ToastOptions } from './Toast'; export { CustomAlert, showAlert, hideAlert, initialAlertState } from './CustomAlert'; -export type { AlertState } from './CustomAlert'; +export type { AlertState, AlertButton } from './CustomAlert'; export { CenteredAlert } from './CenteredAlert'; ; export { ModelFailureCard } from './ModelFailureCard'; diff --git a/src/components/onboarding/spotlightConfig.tsx b/src/components/onboarding/spotlightConfig.tsx index 5b9b69d0c..15f196669 100644 --- a/src/components/onboarding/spotlightConfig.tsx +++ b/src/components/onboarding/spotlightConfig.tsx @@ -75,7 +75,7 @@ export const STEP_TAB_MAP: Record = { downloadedModel: 'ModelsTab', loadedModel: 'HomeTab', sentMessage: 'ChatsTab', - exploredSettings: 'SettingsTab', + exploredSettings: 'Settings', createdProject: 'ProjectsTab', triedImageGen: 'ModelsTab', }; diff --git a/src/constants/index.ts b/src/constants/index.ts index 0c0a53dc3..95fbfd36f 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -150,6 +150,9 @@ export const ONBOARDING_SLIDES = [ // Fonts export const FONTS = { + // iOS ships Menlo; Android has no Menlo (it would silently fall back to the + // sans-serif default), so use Android's built-in monospace so both platforms + // render a similar fixed-width face. mono: 'Menlo', }; diff --git a/src/navigation/AppNavigator.tsx b/src/navigation/AppNavigator.tsx index fac8bb11c..721847f9c 100644 --- a/src/navigation/AppNavigator.tsx +++ b/src/navigation/AppNavigator.tsx @@ -232,6 +232,7 @@ export const AppNavigator: React.FC = () => { /> + diff --git a/src/navigation/types.ts b/src/navigation/types.ts index 04c7b8dfa..51c0688b1 100644 --- a/src/navigation/types.ts +++ b/src/navigation/types.ts @@ -13,6 +13,7 @@ export type RootStackParamList = { KnowledgeBase: { projectId: string }; DocumentPreview: { filePath: string; fileName: string; fileSize: number }; // Former SettingsStack + Settings: undefined; ModelSettings: undefined; RemoteServers: undefined; DeviceInfo: undefined; diff --git a/src/screens/HomeScreen/index.tsx b/src/screens/HomeScreen/index.tsx index 160f88483..e3fe8662f 100644 --- a/src/screens/HomeScreen/index.tsx +++ b/src/screens/HomeScreen/index.tsx @@ -26,6 +26,7 @@ import { VoiceModelsSheet } from '../../components/models/VoiceModelsSheet'; import { useWhisperStore } from '../../stores/whisperStore'; import { WHISPER_MODELS } from '../../services'; import { useUiModeStore } from '../../stores/uiModeStore'; +import { useSlot, SLOTS } from '../../bootstrap/slotRegistry'; type HomeScreenProps = { navigation: HomeScreenNavigationProp; @@ -96,6 +97,9 @@ export const HomeScreen: React.FC = ({ navigation }) => { const whisperModelId = useWhisperStore((s) => s.downloadedModelId); const whisperPresentCount = useWhisperStore((s) => s.presentModelIds?.length ?? 0); const voiceSummary = useUiModeStore((s) => s.voiceSummary); + // Pro recorder entry (tap-to-record card). Empty in free builds. Replaces the + // old dedicated Recorder tab - the recorder now lives here on Home. + const HomeRecorder = useSlot(SLOTS.homeRecorder); const modelLabels: Record = { text: activeTextModel?.name ?? '—', @@ -144,9 +148,11 @@ export const HomeScreen: React.FC = ({ navigation }) => { Off Grid AI {showIcon && } - navigation.navigate('ProDetail')} hitSlop={8} style={styles.crownButton}> - - + + navigation.navigate('ProDetail')} hitSlop={8} style={styles.crownButton}> + + + {/* Collapsed Models summary — tap to open the manager sheet. Both the @@ -164,6 +170,14 @@ export const HomeScreen: React.FC = ({ navigation }) => { + {/* Pro recorder card (tap to record + link to recordings). Renders + only when Pro registered it; free builds show nothing here. */} + {HomeRecorder ? ( + + + + ) : null} + {/* New Chat Button */} { (activeTextModel || activeImageModelId) ? ( diff --git a/src/screens/HomeScreen/styles.ts b/src/screens/HomeScreen/styles.ts index ffbf4da48..08eaf80a8 100644 --- a/src/screens/HomeScreen/styles.ts +++ b/src/screens/HomeScreen/styles.ts @@ -23,6 +23,18 @@ const createLayoutStyles = (colors: ThemeColors) => ({ alignItems: 'center' as const, gap: 8, }, + headerRight: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + gap: 8, + }, + iconButton: { + width: 32, + height: 32, + borderRadius: 16, + alignItems: 'center' as const, + justifyContent: 'center' as const, + }, crownButton: { width: 32, height: 32, diff --git a/src/screens/MemoryTabScreen.tsx b/src/screens/MemoryTabScreen.tsx new file mode 100644 index 000000000..7281ac87d --- /dev/null +++ b/src/screens/MemoryTabScreen.tsx @@ -0,0 +1,194 @@ +import React from 'react'; +import { View, Text, ScrollView, Platform } from 'react-native'; +import { SafeAreaView } from 'react-native-safe-area-context'; +import { useNavigation } from '@react-navigation/native'; +import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; +import Icon from 'react-native-vector-icons/Feather'; +import { useThemedStyles, useTheme } from '../theme'; +import type { ThemeColors } from '../theme'; +import { TYPOGRAPHY, SPACING } from '../constants'; +import { Button } from '../components'; +import { useRegisteredScreens } from '../navigation/screenRegistry'; +import type { RootStackParamList } from '../navigation/types'; + +// Name the recorder registers itself under in pro.activate (screenRegistry). +// Present only when Pro is active; absent in the free build. This is the main +// recorder screen (start/stop controls + dashboard); it pushes to the full +// recordings archive (LocketRecordings) itself. +const RECORDER_SCREEN = 'AlwaysOnTranscription'; + +/** + * The Memory bottom tab. Renders the Pro recorder when it has been registered + * (pro.activate runs only behind the entitlement gate), otherwise a paywall. + * The lookup is reactive (useRegisteredScreens), so unlocking Pro at runtime + * swaps the paywall for the recorder with no app restart. The recorder screen + * uses useNavigation internally, so it works rendered as a tab root. + */ +export const MemoryTabScreen: React.FC = () => { + const recorder = useRegisteredScreens().find((s) => s.name === RECORDER_SCREEN); + if (recorder) { + const Recorder = recorder.component; + return ; + } + return ; +}; + +interface Feature { + icon: string; + title: string; + desc: string; +} + +const FEATURES: Feature[] = [ + { + icon: 'mic', + title: `Always-on recording${Platform.OS === 'android' ? '' : ' (Android)'}`, + desc: 'Capture meetings and conversations in the background, all day.', + }, + { + icon: 'file-text', + title: 'On-device transcription', + desc: 'Whisper turns recordings into readable text right on your phone.', + }, + { + icon: 'align-left', + title: 'Summaries', + desc: 'Condense a long recording into the key points and action items.', + }, + { + icon: 'calendar', + title: 'Calendar context', + desc: 'Each recording is labelled with the meeting and the people in it.', + }, +]; + +const FeatureRow: React.FC<{ feature: Feature; styles: ReturnType; colors: ThemeColors }> = ({ feature, styles, colors }) => ( + + + + + + {feature.title} + {feature.desc} + + +); + +const MemoryPaywall: React.FC = () => { + const navigation = useNavigation>(); + const { colors } = useTheme(); + const styles = useThemedStyles(createStyles); + return ( + + + + + + Recorder + + Capture your meetings and conversations, then transcribe, summarise, and + search them - entirely on your phone. + + + + {FEATURES.map((f) => ( + + ))} + + + + + + The audio and transcript run in your phone and never leave the device. + + + + +