From f83cbac08513326780ab764caf5ee189e98b2c31 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:56 +0530 Subject: [PATCH 01/96] feat(pro): autolink the pro native library via react-native.config.js Co-Authored-By: Dishit Karia --- react-native.config.js | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 react-native.config.js diff --git a/react-native.config.js b/react-native.config.js new file mode 100644 index 000000000..75d366f64 --- /dev/null +++ b/react-native.config.js @@ -0,0 +1,36 @@ +const fs = require('fs'); +const path = require('path'); + +// Autolink the pro submodule's native library ONLY when it is actually on +// disk. Mirrors the fs.existsSync(pro) guard metro.config.js uses for the pro +// JS: a public clone without the private submodule sees an empty/absent pro/ +// dir, this entry is omitted, and the open build compiles with no pro native. +// +// IMPORTANT: check a real file inside pro/, never just the pro/ directory - an +// uninitialised submodule leaves an empty pro/ folder behind. +const proRoot = path.resolve(__dirname, 'pro'); +const proAndroidGradle = path.join(proRoot, 'android', 'build.gradle'); +const proPodspec = path.join(proRoot, 'ios', 'OffgridPro.podspec'); +const proHasNative = fs.existsSync(proAndroidGradle); + +module.exports = { + dependencies: { + ...(proHasNative + ? { + '@offgrid/pro': { + root: proRoot, + platforms: { + android: { + sourceDir: path.join(proRoot, 'android'), + packageImportPath: 'import ai.offgridmobile.alwayson.AlwaysOnTranscriptionPackage;', + packageInstance: 'new AlwaysOnTranscriptionPackage()', + }, + ios: { + podspecPath: proPodspec, + }, + }, + }, + } + : {}), + }, +}; From e4443b8fb909df7f693be329139a29b2a6f67910 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:56 +0530 Subject: [PATCH 02/96] feat(locket): core whisper file-transcription support (chunkable opts, stop, native log) Co-Authored-By: Dishit Karia --- src/services/whisperService.ts | 137 ++++++++++++++++++++++++++++++--- src/stores/whisperStore.ts | 6 +- 2 files changed, 131 insertions(+), 12 deletions(-) diff --git a/src/services/whisperService.ts b/src/services/whisperService.ts index 949d87048..e3b2b0b94 100644 --- a/src/services/whisperService.ts +++ b/src/services/whisperService.ts @@ -1,7 +1,33 @@ import { initWhisper, WhisperContext, RealtimeTranscribeEvent, AudioSessionIos } from 'whisper.rn'; +import * as WhisperRn from 'whisper.rn'; import { Platform, PermissionsAndroid } from 'react-native'; import RNFS from 'react-native-fs'; import logger from '../utils/logger'; + +// Pipe whisper.cpp's native logs (system_info with the real n_threads, model +// load info, encode/decode timings) into our logger so they show in both the +// JS debug-log screen and logcat. Wired once, lazily. Accessed via a cast +// because the local whisper.rn type shim doesn't declare these (they exist at +// runtime in whisper.rn >= 0.5). +let nativeWhisperLogWired = false; +function wireNativeWhisperLog(): void { + if (nativeWhisperLogWired) return; + nativeWhisperLogWired = true; + const w = WhisperRn as unknown as { + toggleNativeLog?: (enabled: boolean) => void; + addNativeLogListener?: (l: (level: string, text: string) => void) => void; + }; + try { + w.toggleNativeLog?.(true); + w.addNativeLogListener?.((level: string, text: string) => { + const msg = text.trim(); + if (msg) logger.log(`[whisper.cpp:${level}] ${msg}`); + }); + logger.log('[Whisper] native logging enabled'); + } catch (e) { + logger.warn(`[Whisper] could not enable native logging: ${String(e)}`); + } +} import { backgroundDownloadService } from './backgroundDownloadService'; import { useDownloadStore } from '../stores/downloadStore'; import { makeModelKey } from '../utils/modelKey'; @@ -40,6 +66,7 @@ class WhisperService { private contextReleasePromise: Promise = Promise.resolve(); private transcriptionFullyStopped: Promise = Promise.resolve(); private activeDownloadId: string | null = null; + private fileTranscribeStop: (() => void | Promise) | null = null; getModelsDir(): string { return `${RNFS.DocumentDirectoryPath}/whisper-models`; } async ensureModelsDirExists(): Promise { @@ -224,7 +251,8 @@ class WhisperService { logger.log(`[Whisper] Model file validated: ${modelPath} (${Math.round(fileSize / (1024 * 1024))} MB)`); } - async loadModel(modelPath: string): Promise { + async loadModel(modelPath: string, options?: { useGpu?: boolean; useFlashAttn?: boolean }): Promise { + wireNativeWhisperLog(); if (this.context && this.currentModelPath !== modelPath) await this.unloadModel(); if (this.context && this.currentModelPath === modelPath) return; if (this.isReleasingContext) { @@ -236,9 +264,16 @@ class WhisperService { // Native initWithModelPath calls abort() on invalid files, crashing the app. await this.validateModelFile(modelPath); - logger.log(`[Whisper] Loading model: ${modelPath}`); + logger.log(`[Whisper] Loading model: ${modelPath} useGpu=${options?.useGpu ?? false} useFlashAttn=${options?.useFlashAttn ?? false}`); try { - this.context = await initWhisper({ filePath: modelPath }); + // useGpu/useFlashAttn are real whisper.rn runtime options but are absent + // from this version's WhisperContextOptions type, so pass via a cast. + const initOpts: Record = { + filePath: modelPath, + useGpu: options?.useGpu ?? false, + useFlashAttn: options?.useFlashAttn ?? false, + }; + this.context = await initWhisper(initOpts as unknown as Parameters[0]); this.currentModelPath = modelPath; logger.log('[Whisper] Model loaded successfully'); } catch (error) { @@ -446,19 +481,103 @@ class WhisperService { options?: { language?: string; onProgress?: (progress: number) => void; + // Fires every time Whisper finishes decoding a chunk (~30s of audio). + // `text` is the cumulative transcript so far, ready to drop straight + // into the UI. Use this for progressive display on long files. + onPartial?: (text: string) => void; + maxThreads?: number; + nProcessors?: number; } ): Promise { + wireNativeWhisperLog(); if (!this.context) { throw new Error('No Whisper model loaded'); } - const { promise } = this.context.transcribe(filePath, { - language: options?.language || 'en', - onProgress: options?.onProgress, - }); + const language = options?.language || 'auto'; + const maxThreads = options?.maxThreads ?? 0; + const nProcessors = options?.nProcessors ?? 1; + const loadedPath = this.currentModelPath ?? '(unknown)'; + const gpu = (this.context as unknown as { gpu?: boolean }).gpu; + + logger.log( + `[Whisper] transcribeFile START path=${filePath} lang=${language} ` + + `maxThreads=${maxThreads} nProcessors=${nProcessors} ` + + `model=${loadedPath} gpu=${gpu}`, + ); + const tStart = Date.now(); + let lastProgressLog = 0; + + // 'auto' means "let Whisper sniff the first ~30s of audio and pick". + // whisper.rn does this when the language field is omitted entirely; + // passing the string 'auto' would be interpreted as a literal code. + const transcribeOpts: Record = { + onProgress: (progress: number) => { + if (progress - lastProgressLog >= 10 || progress >= 100) { + lastProgressLog = progress; + logger.log( + `[Whisper] transcribe progress ${progress.toFixed(0)}% ` + + `elapsed=${((Date.now() - tStart) / 1000).toFixed(1)}s`, + ); + } + options?.onProgress?.(progress); + }, + }; + if (language !== 'auto') transcribeOpts.language = language; + if (maxThreads > 0) transcribeOpts.maxThreads = maxThreads; + if (nProcessors > 1) transcribeOpts.nProcessors = nProcessors; + + // whisper.rn fires onNewSegments after every decoded chunk; `result` is + // the cumulative text. nProcessors > 1 disables this in whisper.cpp + // (parallel chunks can't stream in order), so it only fires when nProcessors == 1. + if (options?.onPartial) { + transcribeOpts.onNewSegments = (eventData: { result: string }) => { + try { + options.onPartial?.(eventData.result); + } catch (err) { + logger.warn(`[Whisper] onPartial callback threw: ${String(err)}`); + } + }; + } + + const { stop, promise } = this.context.transcribe( + filePath, + transcribeOpts as Parameters[1], + ); + this.fileTranscribeStop = stop; - const { result } = await promise; - return result; + try { + const { result } = await promise; + const totalMs = Date.now() - tStart; + logger.log( + `[Whisper] transcribeFile DONE elapsed=${(totalMs / 1000).toFixed(1)}s ` + + `outputLen=${result.length} preview="${result.slice(0, 100)}"`, + ); + return result; + } catch (e) { + const totalMs = Date.now() - tStart; + logger.error(`[Whisper] transcribeFile FAILED after ${(totalMs / 1000).toFixed(1)}s`, e); + throw e; + } finally { + this.fileTranscribeStop = null; + } + } + + /** + * Cancels an in-flight file transcription. The Stop button calls this so + * whisper.cpp actually stops, otherwise the next Transcribe tap throws + * "Context is already transcribing". + */ + async stopFileTranscription(): Promise { + const fn = this.fileTranscribeStop; + this.fileTranscribeStop = null; + if (!fn) { + logger.log('[Whisper] stopFileTranscription: no active file transcription'); + return; + } + logger.log('[Whisper] stopFileTranscription: cancelling native job'); + try { await fn(); } + catch (e) { logger.warn(`[Whisper] stopFileTranscription threw: ${String(e)}`); } } } diff --git a/src/stores/whisperStore.ts b/src/stores/whisperStore.ts index 63b34a810..60e63d57a 100644 --- a/src/stores/whisperStore.ts +++ b/src/stores/whisperStore.ts @@ -27,7 +27,7 @@ interface WhisperState { downloadFromUrl: (url: string, modelId: string) => Promise; /** Activate an already-downloaded model without re-downloading. */ selectModel: (modelId: string) => Promise; - loadModel: () => Promise; + loadModel: (options?: { useGpu?: boolean; useFlashAttn?: boolean }) => Promise; unloadModel: () => Promise; deleteModel: () => Promise; /** Delete a specific on-disk model (active or not). */ @@ -115,7 +115,7 @@ export const useWhisperStore = create()( } }, - loadModel: async () => { + loadModel: async (options?: { useGpu?: boolean; useFlashAttn?: boolean }) => { const { downloadedModelId, isModelLoading } = get(); if (!downloadedModelId) { set({ error: 'No model downloaded' }); @@ -137,7 +137,7 @@ export const useWhisperStore = create()( // then register so future loads can evict it. await modelResidencyManager.runExclusive('load:whisper', async () => { await modelResidencyManager.makeRoomFor({ key: 'whisper', type: 'whisper', sizeMB }); - await whisperService.loadModel(modelPath); + await whisperService.loadModel(modelPath, options); modelResidencyManager.register( { key: 'whisper', type: 'whisper', sizeMB }, () => get().unloadModel(), From dd7f01b08fe364df0aed05e4181804b7a90fb134 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:56 +0530 Subject: [PATCH 03/96] feat(locket): add 'whisper' remote-server provider type Co-Authored-By: Dishit Karia --- src/services/remoteServerManagerUtils.ts | 6 ++++++ src/types/remoteServer.ts | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/services/remoteServerManagerUtils.ts b/src/services/remoteServerManagerUtils.ts index 4e5fa440f..aa75ffb4f 100644 --- a/src/services/remoteServerManagerUtils.ts +++ b/src/services/remoteServerManagerUtils.ts @@ -84,6 +84,12 @@ export function detectToolCallingCapability(modelId: string): boolean { // --------------------------------------------------------------------------- export async function createProviderForServerImpl(server: RemoteServer): Promise { + // Whisper servers don't expose an LLM API - they're used only for + // speech-to-text via the always-on recorder. Skip provider registration. + if (server.providerType === 'whisper') { + logger.log('[RemoteServerManager] skipping LLM provider for whisper server:', server.name); + return; + } const apiKey = await getApiKeyImpl(server.id); logger.log('[RemoteServerManager] createProvider:', server.name, '| endpoint:', server.endpoint, '| hasApiKey:', !!apiKey); const provider = createOpenAIProvider(server.id, server.endpoint, { apiKey: apiKey || undefined }); diff --git a/src/types/remoteServer.ts b/src/types/remoteServer.ts index 7bfbe201d..60abb6150 100644 --- a/src/types/remoteServer.ts +++ b/src/types/remoteServer.ts @@ -6,7 +6,7 @@ */ /** Provider types supported by the system */ -export type RemoteProviderType = 'openai-compatible' | 'anthropic'; +export type RemoteProviderType = 'openai-compatible' | 'anthropic' | 'whisper'; /** Remote server configuration */ export interface RemoteServer { From 5748d1cb5288c687f956c2909745cc521cd59982 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:57 +0530 Subject: [PATCH 04/96] feat(core): persistent + in-memory debug logging for export Co-Authored-By: Dishit Karia --- src/utils/logger.ts | 65 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 998763c8b..d0f6af5cf 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -1,13 +1,78 @@ +import RNFS from 'react-native-fs'; +import { useDebugLogsStore } from '../stores/debugLogsStore'; + +// Persistent on-disk log for export while testing. Rotated so a long session +// doesn't grow unbounded (~250 bytes/line -> 20 MB buys ~80k lines). +const LOG_FILE_NAME = 'download-debug.log'; +const MAX_LOG_FILE_BYTES = 20 * 1024 * 1024; +const RETAINED_LOG_LINES = 50000; + +let writeQueue: Promise = Promise.resolve(); + +function getLogFilePath(): string { + return `${RNFS.DocumentDirectoryPath}/${LOG_FILE_NAME}`; +} + +function formatArg(arg: unknown): string { + if (arg instanceof Error) { + return `${arg.name}: ${arg.message}${arg.stack ? `\n${arg.stack}` : ''}`; + } + if (typeof arg === 'string') return arg; + if (typeof arg === 'number' || typeof arg === 'boolean' || arg == null) return String(arg); + try { + return JSON.stringify(arg); + } catch { + return String(arg); + } +} + +function appendPersistentLog(level: 'log' | 'warn' | 'error', message: string): void { + const line = `[${new Date().toISOString()}] ${level.toUpperCase()}: ${message}\n`; + writeQueue = writeQueue.then(async () => { + try { + const path = getLogFilePath(); + if (await RNFS.exists(path)) { + await RNFS.appendFile(path, line, 'utf8'); + } else { + await RNFS.writeFile(path, line, 'utf8'); + } + const stat = await RNFS.stat(path); + const size = typeof stat.size === 'string' ? Number.parseInt(stat.size, 10) : stat.size; + if (size > MAX_LOG_FILE_BYTES) { + const content = await RNFS.readFile(path, 'utf8'); + const trimmed = content.split('\n').filter(Boolean).slice(-RETAINED_LOG_LINES).join('\n'); + await RNFS.writeFile(path, trimmed ? `${trimmed}\n` : '', 'utf8'); + } + } catch { + // Logging must never break app execution. + } + }); +} + +function capture(level: 'log' | 'warn' | 'error', args: unknown[]): void { + const message = args.map(formatArg).join(' '); + try { + useDebugLogsStore.getState().addLog({ timestamp: Date.now(), level, message }); + } catch { + // Ignore store failures during logger bootstrap. + } + appendPersistentLog(level, message); +} + const logger = { log: (...args: unknown[]): void => { + capture('log', args); if (__DEV__) console.log(...args); // NOSONAR }, warn: (...args: unknown[]): void => { + capture('warn', args); if (__DEV__) console.warn(...args); // NOSONAR }, error: (...args: unknown[]): void => { + capture('error', args); if (__DEV__) console.error(...args); // NOSONAR }, + getLogFilePath, }; export default logger; From 0b28261e96bda150bc969bb5f81d0c33d3930144 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:57 +0530 Subject: [PATCH 05/96] feat(ios): background-audio mode for locket recorder Co-Authored-By: Dishit Karia --- ios/OffgridMobile/Info.plist | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ios/OffgridMobile/Info.plist b/ios/OffgridMobile/Info.plist index 3887c9675..3ab0d6712 100644 --- a/ios/OffgridMobile/Info.plist +++ b/ios/OffgridMobile/Info.plist @@ -18,6 +18,10 @@ $(PRODUCT_NAME) CFBundlePackageType APPL + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleSignature + ???? CFBundleURLTypes @@ -29,19 +33,15 @@ - CFBundleShortVersionString - $(MARKETING_VERSION) - CFBundleSignature - ???? CFBundleVersion $(CURRENT_PROJECT_VERSION) - LSRequiresIPhoneOS - LSApplicationQueriesSchemes twitter x + LSRequiresIPhoneOS + NSAppTransportSecurity NSAllowsLocalNetworking @@ -53,6 +53,10 @@ _ollama._tcp _lmstudio._tcp + NSCalendarsFullAccessUsageDescription + Used to read and create calendar events on your request. + NSCalendarsUsageDescription + Used to read and create calendar events on your request. NSCameraUsageDescription This app needs access to your camera to take photos and attach them to conversations. NSFaceIDUsageDescription @@ -65,10 +69,6 @@ This app needs permission to save generated images to your photo library. NSPhotoLibraryUsageDescription This app needs access to your photo library to attach images to conversations. - NSCalendarsUsageDescription - Used to read and create calendar events on your request. - NSCalendarsFullAccessUsageDescription - Used to read and create calendar events on your request. NSSpeechRecognitionUsageDescription This app uses on-device speech recognition to transcribe voice input. RCTNewArchEnabled @@ -95,6 +95,10 @@ SimpleLineIcons.ttf FontAwesome6_Brands.ttf + UIBackgroundModes + + audio + UILaunchStoryboardName LaunchScreen UIRequiredDeviceCapabilities From 59e4439aeced6e82c48ad50392f01fd5ce2fec87 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:57 +0530 Subject: [PATCH 06/96] chore(ios): pod install for OffgridPro Co-Authored-By: Dishit Karia --- ios/Podfile.lock | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 7ab823027..0e4ad37c0 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -73,6 +73,34 @@ PODS: - MMKV (2.4.0): - MMKVCore (~> 2.4.0) - MMKVCore (2.4.0) + - OffgridPro (0.0.1): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga - op-sqlite (15.2.5): - boost - DoubleConversion @@ -3570,6 +3598,7 @@ DEPENDENCIES: - hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`) - llama-rn (from `../node_modules/llama.rn`) - lottie-react-native (from `../node_modules/lottie-react-native`) + - OffgridPro (from `../pro/ios`) - "op-sqlite (from `../node_modules/@op-engineering/op-sqlite`)" - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTDeprecation (from `../node_modules/react-native/ReactApple/Libraries/RCTFoundation/RCTDeprecation`) @@ -3708,6 +3737,8 @@ EXTERNAL SOURCES: :path: "../node_modules/llama.rn" lottie-react-native: :path: "../node_modules/lottie-react-native" + OffgridPro: + :path: "../pro/ios" op-sqlite: :path: "../node_modules/@op-engineering/op-sqlite" RCT-Folly: @@ -3917,12 +3948,13 @@ SPEC CHECKSUMS: FBLazyVector: 309703e71d3f2f1ed7dc7889d58309c9d77a95a4 fmt: a40bb5bd0294ea969aaaba240a927bd33d878cdd glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 - hermes-engine: 8a072b98dfc36920dbf6eb10468ab5520f2c4b37 + hermes-engine: 8c6be38f94b3bf8b864981980e64e55f08e467ec llama-rn: 7c9b9610cc259118508bd1264bbd9be47e17ffd0 lottie-ios: a881093fab623c467d3bce374367755c272bdd59 lottie-react-native: 691b8363e8c591fb78a78254ff2517258891456b MMKV: 86859fdfa2b0b21db1fd6e48788474a6416a2c77 MMKVCore: 3d16ce9f7d411e135020915fde98a056859a1efa + OffgridPro: bd17e59d526e0cd255c29d92dddbfa3a5796b142 op-sqlite: bafff369cecaee4fe65c89eec47deaba26f2db95 opencv-rne: 2305807573b6e29c8c87e3416ab096d09047a7a0 PurchasesHybridCommon: eed735a411c1aee8c05d62933fa7c4a40ede4009 @@ -3967,7 +3999,7 @@ SPEC CHECKSUMS: react-native-blur: 6af83e7e3c4c1446a188d9b2c493600fc4beb173 react-native-document-picker: dc2d83366e47e89e7c51e8a41eab99c1d54e941c react-native-document-viewer: 8c6ed07e7e27352743fa98e8dd6d288ad925b884 - react-native-executorch: 992a2b4ce98cbf47e4b7904d6680c0be379f7ec1 + react-native-executorch: 65df20362342afff0040d227d270a1b9a59f0c54 react-native-get-random-values: d16467cf726c618e9c7a8c3c39c31faa2244bbba react-native-image-picker: 0314366753615115fa55c3cc937ac44cb7e75702 react-native-keyboard-controller: 7534b5a39d1e8b2b79f86e8e998ed71c7154f69f From eb9aea11a66e2c04114d4e41e224c1ded487cd82 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 03:00:57 +0530 Subject: [PATCH 07/96] chore: bump pro submodule to feat/locket (locket feature) Co-Authored-By: Dishit Karia --- pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pro b/pro index d32d94f0c..e2e4297e6 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit d32d94f0c7aedaaaf70870d5cd9665ff5ff06cb5 +Subproject commit e2e4297e6ad3f2be9491db4eb7b1bcccb6218d4c From e72ac6823e27286dd3767e1b48d83f7d63aaac31 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 14:45:08 +0530 Subject: [PATCH 08/96] feat(transcription): offset/duration chunking, live segments, CoreML option (core) whisperService.transcribeFile gains offset/duration so a recording can be transcribed in chunks, and streams segments live via onNewSegments (cumulative result + per-segment t0/t1). loadModel takes useGpu/useFlashAttn/useCoreML and passes them to initWhisper (useCoreMLIos on iOS). Bumps the pro submodule to the chunked/resumable transcription experience. Co-Authored-By: Dishit Karia --- pro | 2 +- src/services/whisperService.ts | 53 +++++++++++++++++++++++++++++----- src/stores/whisperStore.ts | 4 +-- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/pro b/pro index e2e4297e6..6a4aa1e7d 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit e2e4297e6ad3f2be9491db4eb7b1bcccb6218d4c +Subproject commit 6a4aa1e7d3c78a178f26e6bd5feb93b8206fc489 diff --git a/src/services/whisperService.ts b/src/services/whisperService.ts index e3b2b0b94..81fa6d68f 100644 --- a/src/services/whisperService.ts +++ b/src/services/whisperService.ts @@ -251,7 +251,10 @@ class WhisperService { logger.log(`[Whisper] Model file validated: ${modelPath} (${Math.round(fileSize / (1024 * 1024))} MB)`); } - async loadModel(modelPath: string, options?: { useGpu?: boolean; useFlashAttn?: boolean }): Promise { + async loadModel( + modelPath: string, + options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }, + ): Promise { wireNativeWhisperLog(); if (this.context && this.currentModelPath !== modelPath) await this.unloadModel(); if (this.context && this.currentModelPath === modelPath) return; @@ -264,14 +267,20 @@ class WhisperService { // Native initWithModelPath calls abort() on invalid files, crashing the app. await this.validateModelFile(modelPath); - logger.log(`[Whisper] Loading model: ${modelPath} useGpu=${options?.useGpu ?? false} useFlashAttn=${options?.useFlashAttn ?? false}`); + logger.log( + `[Whisper] Loading model: ${modelPath} useGpu=${options?.useGpu ?? false} ` + + `useFlashAttn=${options?.useFlashAttn ?? false} useCoreML=${options?.useCoreML ?? false}`, + ); try { - // useGpu/useFlashAttn are real whisper.rn runtime options but are absent - // from this version's WhisperContextOptions type, so pass via a cast. + // useGpu/useFlashAttn/useCoreMLIos are real whisper.rn runtime options but + // absent from this version's WhisperContextOptions type, so pass via a cast. + // useCoreMLIos accelerates on the Apple Neural Engine; whisper.rn falls + // back to CPU when the CoreML encoder assets aren't present (iOS only). const initOpts: Record = { filePath: modelPath, useGpu: options?.useGpu ?? false, useFlashAttn: options?.useFlashAttn ?? false, + useCoreMLIos: options?.useCoreML ?? false, }; this.context = await initWhisper(initOpts as unknown as Parameters[0]); this.currentModelPath = modelPath; @@ -487,6 +496,13 @@ class WhisperService { onPartial?: (text: string) => void; maxThreads?: number; nProcessors?: number; + // Transcribe only a window of the file (ms). Used for chunked / + // resumable transcription of long recordings. + offset?: number; + duration?: number; + // Receives the final segments with whisper.cpp timestamps. t0/t1 are in + // centiseconds (10ms units) relative to the processed window. + onSegments?: (segments: { text: string; t0: number; t1: number }[]) => void; } ): Promise { wireNativeWhisperLog(); @@ -526,14 +542,24 @@ class WhisperService { if (language !== 'auto') transcribeOpts.language = language; if (maxThreads > 0) transcribeOpts.maxThreads = maxThreads; if (nProcessors > 1) transcribeOpts.nProcessors = nProcessors; + if (options?.offset && options.offset > 0) transcribeOpts.offset = Math.floor(options.offset); + if (options?.duration && options.duration > 0) transcribeOpts.duration = Math.floor(options.duration); // whisper.rn fires onNewSegments after every decoded chunk; `result` is // the cumulative text. nProcessors > 1 disables this in whisper.cpp // (parallel chunks can't stream in order), so it only fires when nProcessors == 1. - if (options?.onPartial) { - transcribeOpts.onNewSegments = (eventData: { result: string }) => { + if (options?.onPartial || options?.onSegments) { + transcribeOpts.onNewSegments = (eventData: { + result: string; + segments?: { text: string; t0: number; t1: number }[]; + }) => { try { options.onPartial?.(eventData.result); + // Stream cumulative segments (with t0/t1) live, so timestamps appear + // as transcription progresses, not only at the end. + if (options.onSegments && Array.isArray(eventData.segments)) { + options.onSegments(eventData.segments); + } } catch (err) { logger.warn(`[Whisper] onPartial callback threw: ${String(err)}`); } @@ -547,7 +573,20 @@ class WhisperService { this.fileTranscribeStop = stop; try { - const { result } = await promise; + const res = await promise; + const result = res.result; + // The local whisper.rn type shim only declares `result`; segments exist + // at runtime (whisper.cpp t0/t1 in centiseconds). + const segments = (res as unknown as { + segments?: { text: string; t0: number; t1: number }[]; + }).segments; + if (options?.onSegments && Array.isArray(segments)) { + try { + options.onSegments(segments); + } catch (err) { + logger.warn(`[Whisper] onSegments callback threw: ${String(err)}`); + } + } const totalMs = Date.now() - tStart; logger.log( `[Whisper] transcribeFile DONE elapsed=${(totalMs / 1000).toFixed(1)}s ` + diff --git a/src/stores/whisperStore.ts b/src/stores/whisperStore.ts index 60e63d57a..231bdd812 100644 --- a/src/stores/whisperStore.ts +++ b/src/stores/whisperStore.ts @@ -27,7 +27,7 @@ interface WhisperState { downloadFromUrl: (url: string, modelId: string) => Promise; /** Activate an already-downloaded model without re-downloading. */ selectModel: (modelId: string) => Promise; - loadModel: (options?: { useGpu?: boolean; useFlashAttn?: boolean }) => Promise; + loadModel: (options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }) => Promise; unloadModel: () => Promise; deleteModel: () => Promise; /** Delete a specific on-disk model (active or not). */ @@ -115,7 +115,7 @@ export const useWhisperStore = create()( } }, - loadModel: async (options?: { useGpu?: boolean; useFlashAttn?: boolean }) => { + loadModel: async (options?: { useGpu?: boolean; useFlashAttn?: boolean; useCoreML?: boolean }) => { const { downloadedModelId, isModelLoading } = get(); if (!downloadedModelId) { set({ error: 'No model downloaded' }); From 52cdfdfb90f3aada3cf2050ffd9865d5f2f8e7a7 Mon Sep 17 00:00:00 2001 From: Dishit Date: Sun, 28 Jun 2026 21:33:01 +0530 Subject: [PATCH 09/96] feat(nav): Memory tab (Pro recorder/paywall) + move Settings to Home header Replaces the Settings bottom tab with a Memory tab. MemoryTabScreen renders the Pro recorder when it is registered (pro.activate) and a paywall otherwise, read reactively so unlocking Pro swaps it in without a restart - the tab shell stays in core, the recorder injects from the submodule. Settings becomes a pushed RootStack screen reached from a gear in the Home header. Updates nav types and the onboarding step that pointed at the old SettingsTab, and the SettingsScreen nav-prop type (no longer a tab). Co-Authored-By: Dishit Karia --- src/components/onboarding/spotlightConfig.tsx | 2 +- src/navigation/AppNavigator.tsx | 10 +- src/navigation/types.ts | 3 +- src/screens/HomeScreen/index.tsx | 11 +- src/screens/HomeScreen/styles.ts | 12 +++ src/screens/MemoryTabScreen.tsx | 100 ++++++++++++++++++ src/screens/SettingsScreen.tsx | 2 +- 7 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 src/screens/MemoryTabScreen.tsx diff --git a/src/components/onboarding/spotlightConfig.tsx b/src/components/onboarding/spotlightConfig.tsx index 5b9b69d0c..15f196669 100644 --- a/src/components/onboarding/spotlightConfig.tsx +++ b/src/components/onboarding/spotlightConfig.tsx @@ -75,7 +75,7 @@ export const STEP_TAB_MAP: Record = { downloadedModel: 'ModelsTab', loadedModel: 'HomeTab', sentMessage: 'ChatsTab', - exploredSettings: 'SettingsTab', + exploredSettings: 'Settings', createdProject: 'ProjectsTab', triedImageGen: 'ModelsTab', }; diff --git a/src/navigation/AppNavigator.tsx b/src/navigation/AppNavigator.tsx index fac8bb11c..1a08b768d 100644 --- a/src/navigation/AppNavigator.tsx +++ b/src/navigation/AppNavigator.tsx @@ -45,6 +45,7 @@ import { MainTabParamList, } from './types'; import { useRegisteredScreens } from './screenRegistry'; +import { MemoryTabScreen } from '../screens/MemoryTabScreen'; const RootStack = createNativeStackNavigator(); const Tab = createBottomTabNavigator(); @@ -55,7 +56,7 @@ const TAB_ICON_MAP: Record = { ChatsTab: 'message-circle', ProjectsTab: 'folder', ModelsTab: 'cpu', - SettingsTab: 'settings', + MemoryTab: 'mic', }; const TabBarIcon: React.FC<{ name: string; focused: boolean }> = ({ name, focused }) => { @@ -173,9 +174,9 @@ const MainTabs: React.FC = () => { })} /> ({ tabPress: () => { triggerHaptic('selection'); }, })} @@ -232,6 +233,7 @@ export const AppNavigator: React.FC = () => { /> + diff --git a/src/navigation/types.ts b/src/navigation/types.ts index 9d8ffb75b..a02cbf6f0 100644 --- a/src/navigation/types.ts +++ b/src/navigation/types.ts @@ -13,6 +13,7 @@ export type RootStackParamList = { KnowledgeBase: { projectId: string }; DocumentPreview: { filePath: string; fileName: string; fileSize: number }; // Former SettingsStack + Settings: undefined; ModelSettings: undefined; RemoteServers: undefined; DeviceInfo: undefined; @@ -32,5 +33,5 @@ export type MainTabParamList = { ChatsTab: undefined; ProjectsTab: undefined; ModelsTab: { initialTab?: 'text' | 'image' | 'voice' | 'transcription'; repairModelId?: string } | undefined; - SettingsTab: undefined; + MemoryTab: undefined; }; diff --git a/src/screens/HomeScreen/index.tsx b/src/screens/HomeScreen/index.tsx index fca2503fa..ee7ca0769 100644 --- a/src/screens/HomeScreen/index.tsx +++ b/src/screens/HomeScreen/index.tsx @@ -134,9 +134,14 @@ export const HomeScreen: React.FC = ({ navigation }) => { Off Grid {showIcon && } - navigation.navigate('ProDetail')} hitSlop={8} style={styles.crownButton}> - - + + navigation.navigate('Settings')} hitSlop={8} style={styles.iconButton}> + + + navigation.navigate('ProDetail')} hitSlop={8} style={styles.crownButton}> + + + {/* Collapsed Models summary — tap to open the manager sheet. Both the diff --git a/src/screens/HomeScreen/styles.ts b/src/screens/HomeScreen/styles.ts index 8477e732c..7cbe24723 100644 --- a/src/screens/HomeScreen/styles.ts +++ b/src/screens/HomeScreen/styles.ts @@ -23,6 +23,18 @@ const createLayoutStyles = (colors: ThemeColors) => ({ alignItems: 'center' as const, gap: 8, }, + headerRight: { + flexDirection: 'row' as const, + alignItems: 'center' as const, + gap: 8, + }, + iconButton: { + width: 32, + height: 32, + borderRadius: 16, + alignItems: 'center' as const, + justifyContent: 'center' as const, + }, crownButton: { width: 32, height: 32, diff --git a/src/screens/MemoryTabScreen.tsx b/src/screens/MemoryTabScreen.tsx new file mode 100644 index 000000000..4603b3676 --- /dev/null +++ b/src/screens/MemoryTabScreen.tsx @@ -0,0 +1,100 @@ +import React from 'react'; +import { View, Text } from 'react-native'; +import { SafeAreaView } from 'react-native-safe-area-context'; +import { useNavigation } from '@react-navigation/native'; +import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; +import Icon from 'react-native-vector-icons/Feather'; +import { useThemedStyles, useTheme } from '../theme'; +import type { ThemeColors } from '../theme'; +import { TYPOGRAPHY, SPACING } from '../constants'; +import { Button } from '../components'; +import { useRegisteredScreens } from '../navigation/screenRegistry'; +import type { RootStackParamList } from '../navigation/types'; + +// Name the recorder registers itself under in pro.activate (screenRegistry). +// Present only when Pro is active; absent in the free build. This is the main +// recorder screen (start/stop controls + dashboard); it pushes to the full +// recordings archive (LocketRecordings) itself. +const RECORDER_SCREEN = 'AlwaysOnTranscription'; + +/** + * The Memory bottom tab. Renders the Pro recorder when it has been registered + * (pro.activate runs only behind the entitlement gate), otherwise a paywall. + * The lookup is reactive (useRegisteredScreens), so unlocking Pro at runtime + * swaps the paywall for the recorder with no app restart. The recorder screen + * uses useNavigation internally, so it works rendered as a tab root. + */ +export const MemoryTabScreen: React.FC = () => { + const recorder = useRegisteredScreens().find((s) => s.name === RECORDER_SCREEN); + if (recorder) { + const Recorder = recorder.component; + return ; + } + return ; +}; + +const MemoryPaywall: React.FC = () => { + const navigation = useNavigation>(); + const { colors } = useTheme(); + const styles = useThemedStyles(createStyles); + return ( + + + + + + Memory + + Record continuously and transcribe on your phone. The audio and the + transcript stay on the device - nothing is uploaded. + + + Tap-to-seek timestamps, resumable transcription, on-device Whisper. + + +