From 5f65e570055240fe9973112e44c65e33c0e3ad93 Mon Sep 17 00:00:00 2001 From: artus9033 Date: Fri, 6 Feb 2026 21:56:16 +0100 Subject: [PATCH 01/25] feat: support newest MLC on iOS --- packages/mlc/ATTRIBUTIONS.md | 10 +- packages/mlc/ios/MLCEngine.mm | 143 ++++++++++++++++++-- packages/mlc/ios/engine/BackgroundWorker.h | 14 -- packages/mlc/ios/engine/BackgroundWorker.mm | 32 ----- packages/mlc/ios/engine/EngineState.h | 24 ---- packages/mlc/ios/engine/EngineState.mm | 73 ---------- packages/mlc/ios/engine/JSONFFIEngine.h | 45 ------ packages/mlc/ios/engine/JSONFFIEngine.mm | 123 ----------------- packages/mlc/ios/engine/LLMEngine.h | 27 ++-- packages/mlc/ios/engine/LLMEngine.mm | 140 ++++++++++++------- 10 files changed, 247 insertions(+), 384 deletions(-) delete mode 100644 packages/mlc/ios/engine/BackgroundWorker.h delete mode 100644 packages/mlc/ios/engine/BackgroundWorker.mm delete mode 100644 packages/mlc/ios/engine/EngineState.h delete mode 100644 packages/mlc/ios/engine/EngineState.mm delete mode 100644 packages/mlc/ios/engine/JSONFFIEngine.h delete mode 100644 packages/mlc/ios/engine/JSONFFIEngine.mm diff --git a/packages/mlc/ATTRIBUTIONS.md b/packages/mlc/ATTRIBUTIONS.md index 692f47b9..fe424393 100644 --- a/packages/mlc/ATTRIBUTIONS.md +++ b/packages/mlc/ATTRIBUTIONS.md @@ -1,8 +1,7 @@ -Third-Party Notices -=================== +# Third-Party Notices + +## MLC-LLM (mlc-ai/mlc-llm) -MLC-LLM (mlc-ai/mlc-llm) ------------------------ Portions of the iOS engine implementation are derived from the MLC-LLM project, and the prebuilt runtime binaries shipped in this package are based on MLC-LLM. @@ -14,7 +13,6 @@ License: Apache License, Version 2.0 License URL: https://www.apache.org/licenses/LICENSE-2.0 Derived source files in this package: + - packages/mlc/ios/engine/LLMEngine.h - packages/mlc/ios/engine/LLMEngine.mm -- packages/mlc/ios/engine/JSONFFIEngine.h -- packages/mlc/ios/engine/JSONFFIEngine.mm diff --git a/packages/mlc/ios/MLCEngine.mm b/packages/mlc/ios/MLCEngine.mm index ac4d6364..5bed48f0 100644 --- a/packages/mlc/ios/MLCEngine.mm +++ b/packages/mlc/ios/MLCEngine.mm @@ -6,12 +6,16 @@ #import "LLMEngine.h" +typedef void (^MLCStreamCallback)(NSDictionary* response); + @interface MLCEngine : NativeMLCEngineSpecBase -@property(nonatomic, strong) LLMEngine* engine; +@property(nonatomic, strong) JSONFFIEngine* engine; @property(nonatomic, strong) NSURL* bundleURL; @property(nonatomic, strong) NSDictionary* cachedAppConfig; @property(nonatomic, strong) NSArray* cachedModelList; +@property(nonatomic, strong) NSMutableDictionary* pendingRequests; +@property(nonatomic) dispatch_queue_t streamCallbackQueue; @end @@ -26,7 +30,9 @@ + (NSString *)moduleName { - (instancetype)init { self = [super init]; if (self) { - _engine = [[LLMEngine alloc] init]; + _engine = [[JSONFFIEngine alloc] init]; + _pendingRequests = [NSMutableDictionary new]; + _streamCallbackQueue = dispatch_queue_create("com.callstack.mlcegine.stream", DISPATCH_QUEUE_SERIAL); // Get the Documents directory path for downloaded models NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); @@ -39,6 +45,16 @@ - (instancetype)init { if (dirError) { NSLog(@"Error creating bundle directory: %@", dirError); } + + [self.engine initBackgroundEngine:^(NSString* responseJSON) { + [self handleStreamCallback:responseJSON]; + }]; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + [self.engine runBackgroundLoop]; + }); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + [self.engine runBackgroundStreamBackLoop]; + }); } return self; } @@ -113,6 +129,7 @@ - (NSDictionary*)findModelById:(NSString*)modelId { // Helper method to build complete request with messages and options - (NSDictionary*)buildRequestWithMessages:(NSArray*)messages options:(const JS::NativeMLCEngine::GenerationOptions &)options { NSMutableDictionary *request = [@{@"messages": messages, @"stream": @(YES)} mutableCopy]; + request[@"stream_options"] = @{@"include_usage": @(YES)}; if (options.temperature().has_value()) { request[@"temperature"] = @(options.temperature().value()); @@ -147,6 +164,81 @@ - (NSDictionary*)buildRequestWithMessages:(NSArray*)messages options:(const JS:: return request; } +- (NSString*)jsonStringFromDictionary:(NSDictionary*)dictionary error:(NSError**)error { + NSData* jsonData = [NSJSONSerialization dataWithJSONObject:dictionary options:0 error:error]; + if (!jsonData) { + return nil; + } + return [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding]; +} + +- (NSString*)requestIdFromResponse:(NSDictionary*)response { + NSString* requestId = response[@"request_id"]; + if (!requestId) { + requestId = response[@"requestId"]; + } + if (!requestId) { + requestId = response[@"id"]; + } + return requestId; +} + +- (void)handleStreamCallback:(NSString*)responseJSON { + dispatch_async(self.streamCallbackQueue, ^{ + NSData* jsonData = [responseJSON dataUsingEncoding:NSUTF8StringEncoding]; + if (!jsonData) { + NSLog(@"Failed to decode stream response data"); + return; + } + + NSError* parseError; + id jsonObject = [NSJSONSerialization JSONObjectWithData:jsonData options:0 error:&parseError]; + if (parseError || ![jsonObject isKindOfClass:[NSDictionary class]]) { + NSLog(@"Invalid stream response JSON: %@", parseError.localizedDescription); + return; + } + + NSDictionary* response = (NSDictionary*)jsonObject; + NSString* requestId = [self requestIdFromResponse:response]; + if (!requestId && self.pendingRequests.count == 1) { + requestId = self.pendingRequests.allKeys.firstObject; + } + if (!requestId) { + NSLog(@"Missing request id in stream response"); + return; + } + + MLCStreamCallback callback = self.pendingRequests[requestId]; + if (!callback) { + return; + } + + dispatch_async(dispatch_get_main_queue(), ^{ + callback(response); + }); + + if (response[@"usage"]) { + [self.pendingRequests removeObjectForKey:requestId]; + } + }); +} + +- (NSString*)startChatCompletionWithRequest:(NSDictionary*)request + completion:(MLCStreamCallback)completion + error:(NSError**)error { + NSString* requestJSON = [self jsonStringFromDictionary:request error:error]; + if (!requestJSON) { + return nil; + } + + NSString* requestId = [NSUUID UUID].UUIDString; + if (completion) { + self.pendingRequests[requestId] = [completion copy]; + } + [self.engine chatCompletion:requestJSON requestID:requestId]; + return requestId; +} + - (void)generateText:(NSArray*)messages options:(JS::NativeMLCEngine::GenerationOptions &)options resolve:(RCTPromiseResolveBlock)resolve @@ -159,9 +251,9 @@ - (void)generateText:(NSArray*)messages __block NSString* finalFinishReason = nil; __block NSString* finalRole = nil; - [self.engine chatCompletionWithMessages:messages - options:request - completion:^(NSDictionary* response) { + NSError* requestError; + NSString* requestId = [self startChatCompletionWithRequest:request + completion:^(NSDictionary* response) { if (response[@"usage"]) { resolve(@{ @"role": finalRole, @@ -189,7 +281,11 @@ - (void)generateText:(NSArray*)messages finalFinishReason = choice[@"finish_reason"]; } } - }]; + } error:&requestError]; + + if (!requestId) { + reject(@"MLCEngine", requestError.localizedDescription ?: @"Failed to start generation", nil); + } } - (void)streamText:(NSArray*)messages @@ -202,9 +298,9 @@ - (void)streamText:(NSArray*)messages __block NSString* finalFinishReason = nil; @try { - NSString *requestId = [self.engine chatCompletionWithMessages:messages - options:request - completion:^(NSDictionary* response) { + NSError* requestError; + NSString *requestId = [self startChatCompletionWithRequest:request + completion:^(NSDictionary* response) { if (response[@"usage"]) { [self emitOnChatComplete:@{ @"usage": response[@"usage"], @@ -219,7 +315,13 @@ - (void)streamText:(NSArray*)messages } [self emitOnChatUpdate:choice]; - }]; + } error:&requestError]; + + if (!requestId) { + @throw [NSException exceptionWithName:@"MLCEngine" + reason:requestError.localizedDescription ?: @"Failed to start generation" + userInfo:nil]; + } resolve(requestId); } @catch (NSException* exception) { @@ -274,7 +376,19 @@ - (void)prepareModel:(NSString*)modelId return; } - [self.engine reloadWithModelPath:modelLocalPath modelLib:modelLib]; + NSDictionary* modelConfig = [self readModelConfig:modelId error:nil]; + NSMutableDictionary* engineConfig = modelConfig ? [modelConfig mutableCopy] : [NSMutableDictionary new]; + engineConfig[@"model"] = modelLocalPath; + engineConfig[@"model_lib"] = modelLib; + + NSError* configError; + NSString* engineConfigJSON = [self jsonStringFromDictionary:engineConfig error:&configError]; + if (!engineConfigJSON) { + reject(@"MLCEngine", configError.localizedDescription ?: @"Failed to build engine config", nil); + return; + } + + [self.engine reload:engineConfigJSON]; resolve([NSString stringWithFormat:@"Model prepared: %@", modelId]); } @catch (NSException* exception) { @@ -519,9 +633,14 @@ - (void)unloadModel:(RCTPromiseResolveBlock)resolve } - (void)cancelStream:(nonnull NSString *)streamId resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { - [self.engine cancelRequest:streamId]; + [self.pendingRequests removeObjectForKey:streamId]; + [self.engine abort:streamId]; resolve(nil); } +- (void)dealloc { + [self.engine exitBackgroundLoop]; +} + @end diff --git a/packages/mlc/ios/engine/BackgroundWorker.h b/packages/mlc/ios/engine/BackgroundWorker.h deleted file mode 100644 index a3899242..00000000 --- a/packages/mlc/ios/engine/BackgroundWorker.h +++ /dev/null @@ -1,14 +0,0 @@ -// -// BackgroundWorker.h -// Pods -// - -#import - -NS_ASSUME_NONNULL_BEGIN - -@interface BackgroundWorker : NSThread -- (instancetype)initWithTask:(void (^)(void))task; -@end - -NS_ASSUME_NONNULL_END diff --git a/packages/mlc/ios/engine/BackgroundWorker.mm b/packages/mlc/ios/engine/BackgroundWorker.mm deleted file mode 100644 index e07bfb05..00000000 --- a/packages/mlc/ios/engine/BackgroundWorker.mm +++ /dev/null @@ -1,32 +0,0 @@ -// -// BackgroundWorker.mm -// Pods -// - -#import "BackgroundWorker.h" - -/** - * BackgroundWorker manages background thread execution for the MLC engine. - * This class provides a simple interface to run long-running tasks on separate threads, - * ensuring the main thread remains responsive while the LLM engine processes requests. - * It's used to run the engine's background loop and stream processing loop concurrently. - */ -@implementation BackgroundWorker { - void (^_task)(void); -} - -- (instancetype)initWithTask:(void (^)(void))task { - self = [super init]; - if (self) { - _task = [task copy]; - } - return self; -} - -- (void)main { - if (_task) { - _task(); - } -} - -@end diff --git a/packages/mlc/ios/engine/EngineState.h b/packages/mlc/ios/engine/EngineState.h deleted file mode 100644 index e9f8dfd9..00000000 --- a/packages/mlc/ios/engine/EngineState.h +++ /dev/null @@ -1,24 +0,0 @@ -// -// MLCEngine.h -// Pods -// -// Created by Szymon Rybczak on 19/07/2024. -// - -#import "JSONFFIEngine.h" -#import - -NS_ASSUME_NONNULL_BEGIN - -@interface EngineState : NSObject -@property(nonatomic, strong) NSMutableDictionary *requestStateMap; - -- (NSString*)chatCompletionWithJSONFFIEngine:(JSONFFIEngine *)jsonFFIEngine - request:(NSDictionary *)request - completion:(void (^)(NSDictionary* response))completion; -- (void)streamCallbackWithResult:(NSString *)result; -- (void)cancelRequest:(NSString *)requestId - withJSONFFIEngine:(JSONFFIEngine *)jsonFFIEngine; -@end - -NS_ASSUME_NONNULL_END diff --git a/packages/mlc/ios/engine/EngineState.mm b/packages/mlc/ios/engine/EngineState.mm deleted file mode 100644 index 52ea4066..00000000 --- a/packages/mlc/ios/engine/EngineState.mm +++ /dev/null @@ -1,73 +0,0 @@ -// -// EngineState.mm -// Pods -// - -#import "EngineState.h" -#import "JSONFFIEngine.h" - -/** - * EngineState manages the request lifecycle and callback routing for chat completions. - * It maintains a mapping between request IDs and their corresponding completion handlers, - * ensuring that streaming responses are properly routed back to the correct caller. - * This class handles JSON serialization/deserialization and coordinates between - * the high-level API and the low-level JSON FFI engine. - */ -@implementation EngineState - -- (instancetype)init { - self = [super init]; - if (self) { - _requestStateMap = [NSMutableDictionary new]; - } - return self; -} - -- (NSString*)chatCompletionWithJSONFFIEngine:(JSONFFIEngine*)jsonFFIEngine - request:(NSDictionary*)request - completion:(void (^)(NSDictionary* response))completion { - NSError* error; - NSData* jsonData = [NSJSONSerialization dataWithJSONObject:request options:0 error:&error]; - if (error) { - @throw [NSException exceptionWithName:@"JSONSerializationException" - reason:[NSString stringWithFormat:@"Failed to serialize request: %@", - error.localizedDescription] - userInfo:nil]; - } - - NSString* jsonRequest = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding]; - NSString* requestID = [[NSUUID UUID] UUIDString]; - - self.requestStateMap[requestID] = completion; - - [jsonFFIEngine chatCompletion:jsonRequest requestID:requestID]; - - return requestID; -} - -- (void)streamCallbackWithResult:(NSString*)result { - NSError* error; - NSArray* responses = [NSJSONSerialization JSONObjectWithData:[result dataUsingEncoding:NSUTF8StringEncoding] options:0 error:&error]; - if (error) { - NSLog(@"Error decoding JSON: %@", error); - return; - } - - for (NSDictionary* res in responses) { - NSString* requestID = res[@"id"]; - void (^completion)(NSDictionary*) = self.requestStateMap[requestID]; - if (completion) { - completion(res); - if (res[@"usage"]) { - [self.requestStateMap removeObjectForKey:requestID]; - } - } - } -} - -- (void)cancelRequest:(NSString *)requestId withJSONFFIEngine:(JSONFFIEngine *)jsonFFIEngine { - [self.requestStateMap removeObjectForKey:requestId]; - [jsonFFIEngine abort:requestId]; -} - -@end diff --git a/packages/mlc/ios/engine/JSONFFIEngine.h b/packages/mlc/ios/engine/JSONFFIEngine.h deleted file mode 100644 index 1e4fb441..00000000 --- a/packages/mlc/ios/engine/JSONFFIEngine.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) MLC-AI - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This file is derived from the MLC-LLM project: - * https://github.com/mlc-ai/mlc-llm - */ - -#import -#import - -/** - * This is an internal Raw JSON FFI Engine that redirects request to internal JSON FFI Engine in C++ - */ -@interface JSONFFIEngine : NSObject - -- (void)initBackgroundEngine:(void (^)(NSString *))streamCallback; - -- (void)reload:(NSString *)engineConfig; - -- (void)unload; - -- (void)reset; - -- (void)chatCompletion:(NSString *)requestJSON requestID:(NSString *)requestID; - -- (void)abort:(NSString *)requestID; - -- (void)runBackgroundLoop; - -- (void)runBackgroundStreamBackLoop; - -- (void)exitBackgroundLoop; - -@end diff --git a/packages/mlc/ios/engine/JSONFFIEngine.mm b/packages/mlc/ios/engine/JSONFFIEngine.mm deleted file mode 100644 index cec8778d..00000000 --- a/packages/mlc/ios/engine/JSONFFIEngine.mm +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) MLC-AI - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This file is derived from the MLC-LLM project: - * https://github.com/mlc-ai/mlc-llm - */ -#import -#import -#include - -#include "JSONFFIEngine.h" - -#define TVM_USE_LIBBACKTRACE 0 -#define DMLC_USE_LOGGING_LIBRARY - -#include -#include - -using namespace tvm::runtime; - -@implementation JSONFFIEngine { - // Internal c++ classes - // internal module backed by JSON FFI - Module json_ffi_engine_; - // member functions - PackedFunc init_background_engine_func_; - PackedFunc unload_func_; - PackedFunc reload_func_; - PackedFunc reset_func_; - PackedFunc chat_completion_func_; - PackedFunc abort_func_; - PackedFunc run_background_loop_func_; - PackedFunc run_background_stream_back_loop_func_; - PackedFunc exit_background_loop_func_; -} - -- (instancetype)init { - if (self = [super init]) { - // load chat module - const PackedFunc* f_json_ffi_create = Registry::Get("mlc.json_ffi.CreateJSONFFIEngine"); - ICHECK(f_json_ffi_create) << "Cannot find mlc.json_ffi.CreateJSONFFIEngine"; - json_ffi_engine_ = (*f_json_ffi_create)(); - init_background_engine_func_ = json_ffi_engine_->GetFunction("init_background_engine"); - reload_func_ = json_ffi_engine_->GetFunction("reload"); - unload_func_ = json_ffi_engine_->GetFunction("unload"); - reset_func_ = json_ffi_engine_->GetFunction("reset"); - chat_completion_func_ = json_ffi_engine_->GetFunction("chat_completion"); - abort_func_ = json_ffi_engine_->GetFunction("abort"); - run_background_loop_func_ = json_ffi_engine_->GetFunction("run_background_loop"); - run_background_stream_back_loop_func_ = - json_ffi_engine_->GetFunction("run_background_stream_back_loop"); - exit_background_loop_func_ = json_ffi_engine_->GetFunction("exit_background_loop"); - - ICHECK(init_background_engine_func_ != nullptr); - ICHECK(reload_func_ != nullptr); - ICHECK(unload_func_ != nullptr); - ICHECK(reset_func_ != nullptr); - ICHECK(chat_completion_func_ != nullptr); - ICHECK(abort_func_ != nullptr); - ICHECK(run_background_loop_func_ != nullptr); - ICHECK(run_background_stream_back_loop_func_ != nullptr); - ICHECK(exit_background_loop_func_ != nullptr); - } - return self; -} - -- (void)initBackgroundEngine:(void (^)(NSString*))streamCallback { - TypedPackedFunc internal_stream_callback([streamCallback](String value) { - streamCallback([NSString stringWithUTF8String:value.c_str()]); - }); - int device_type = kDLMetal; - int device_id = 0; - init_background_engine_func_(device_type, device_id, internal_stream_callback); -} - -- (void)reload:(NSString*)engineConfigJson { - std::string engine_config = engineConfigJson.UTF8String; - reload_func_(engine_config); -} - -- (void)unload { - unload_func_(); -} - -- (void)reset { - reset_func_(); -} - -- (void)chatCompletion:(NSString*)requestJSON requestID:(NSString*)requestID { - std::string request_json = requestJSON.UTF8String; - std::string request_id = requestID.UTF8String; - chat_completion_func_(request_json, request_id); -} - -- (void)abort:(NSString*)requestID { - std::string request_id = requestID.UTF8String; - abort_func_(request_id); -} - -- (void)runBackgroundLoop { - run_background_loop_func_(); -} - -- (void)runBackgroundStreamBackLoop { - run_background_stream_back_loop_func_(); -} - -- (void)exitBackgroundLoop { - exit_background_loop_func_(); -} - -@end diff --git a/packages/mlc/ios/engine/LLMEngine.h b/packages/mlc/ios/engine/LLMEngine.h index 6a3fb5fb..1e4fb441 100644 --- a/packages/mlc/ios/engine/LLMEngine.h +++ b/packages/mlc/ios/engine/LLMEngine.h @@ -17,20 +17,29 @@ */ #import +#import -NS_ASSUME_NONNULL_BEGIN +/** + * This is an internal Raw JSON FFI Engine that redirects request to internal JSON FFI Engine in C++ + */ +@interface JSONFFIEngine : NSObject -@interface LLMEngine : NSObject +- (void)initBackgroundEngine:(void (^)(NSString *))streamCallback; -- (instancetype)init; +- (void)reload:(NSString *)engineConfig; -- (void)reloadWithModelPath:(NSString *)modelPath modelLib:(NSString *)modelLib; -- (void)reset; - (void)unload; -- (NSString*)chatCompletionWithMessages:(NSArray *)messages options:(NSDictionary *)options completion:(void (^)(NSDictionary* response))completion; -- (void)cancelRequest:(NSString *)requestId; +- (void)reset; -@end +- (void)chatCompletion:(NSString *)requestJSON requestID:(NSString *)requestID; + +- (void)abort:(NSString *)requestID; -NS_ASSUME_NONNULL_END +- (void)runBackgroundLoop; + +- (void)runBackgroundStreamBackLoop; + +- (void)exitBackgroundLoop; + +@end diff --git a/packages/mlc/ios/engine/LLMEngine.mm b/packages/mlc/ios/engine/LLMEngine.mm index bda8f0ff..2f31be86 100644 --- a/packages/mlc/ios/engine/LLMEngine.mm +++ b/packages/mlc/ios/engine/LLMEngine.mm @@ -16,73 +16,121 @@ * https://github.com/mlc-ai/mlc-llm */ -#import "LLMEngine.h" -#import "BackgroundWorker.h" -#import "EngineState.h" +#import +#import +#include -@interface LLMEngine () +#include "LLMEngine.h" -@property(nonatomic, strong) EngineState* state; -@property(nonatomic, strong) JSONFFIEngine* jsonFFIEngine; -@property(nonatomic, strong) NSMutableArray* threads; +#define TVM_USE_LIBBACKTRACE 0 +#define DMLC_USE_LOGGING_LIBRARY -@end +#include +#include +#include +#include +#include + +using namespace tvm::runtime; +using tvm::ffi::Function; +using tvm::ffi::Module; +using tvm::ffi::Optional; +using tvm::ffi::String; +using tvm::ffi::TypedFunction; -@implementation LLMEngine +@implementation JSONFFIEngine { + // Internal c++ classes + // internal module backed by JSON FFI + Optional json_ffi_engine_; + // member functions + Function init_background_engine_func_; + Function unload_func_; + Function reload_func_; + Function reset_func_; + Function chat_completion_func_; + Function abort_func_; + Function run_background_loop_func_; + Function run_background_stream_back_loop_func_; + Function exit_background_loop_func_; +} - (instancetype)init { - self = [super init]; - if (self) { - _state = [[EngineState alloc] init]; - _jsonFFIEngine = [[JSONFFIEngine alloc] init]; - _threads = [NSMutableArray array]; - - [_jsonFFIEngine initBackgroundEngine:^(NSString* _Nullable result) { - [self.state streamCallbackWithResult:result]; - }]; - - BackgroundWorker* backgroundWorker = [[BackgroundWorker alloc] initWithTask:^{ - [NSThread setThreadPriority:1.0]; - [self.jsonFFIEngine runBackgroundLoop]; - }]; - - BackgroundWorker* backgroundStreamBackWorker = [[BackgroundWorker alloc] initWithTask:^{ - [self.jsonFFIEngine runBackgroundStreamBackLoop]; - }]; - - backgroundWorker.qualityOfService = NSQualityOfServiceUserInteractive; - [_threads addObject:backgroundWorker]; - [_threads addObject:backgroundStreamBackWorker]; - [backgroundWorker start]; - [backgroundStreamBackWorker start]; + if (self = [super init]) { + // load chat module + Function f_json_ffi_create = Function::GetGlobalRequired("mlc.json_ffi.CreateJSONFFIEngine"); + json_ffi_engine_ = f_json_ffi_create().cast(); + init_background_engine_func_ = + json_ffi_engine_.value()->GetFunction("init_background_engine").value_or(Function(nullptr)); + reload_func_ = json_ffi_engine_.value()->GetFunction("reload").value_or(Function(nullptr)); + unload_func_ = json_ffi_engine_.value()->GetFunction("unload").value_or(Function(nullptr)); + reset_func_ = json_ffi_engine_.value()->GetFunction("reset").value_or(Function(nullptr)); + chat_completion_func_ = + json_ffi_engine_.value()->GetFunction("chat_completion").value_or(Function(nullptr)); + abort_func_ = json_ffi_engine_.value()->GetFunction("abort").value_or(Function(nullptr)); + run_background_loop_func_ = + json_ffi_engine_.value()->GetFunction("run_background_loop").value_or(Function(nullptr)); + run_background_stream_back_loop_func_ = json_ffi_engine_.value() + ->GetFunction("run_background_stream_back_loop") + .value_or(Function(nullptr)); + exit_background_loop_func_ = + json_ffi_engine_.value()->GetFunction("exit_background_loop").value_or(Function(nullptr)); + + ICHECK(init_background_engine_func_ != nullptr); + ICHECK(reload_func_ != nullptr); + ICHECK(unload_func_ != nullptr); + ICHECK(reset_func_ != nullptr); + ICHECK(chat_completion_func_ != nullptr); + ICHECK(abort_func_ != nullptr); + ICHECK(run_background_loop_func_ != nullptr); + ICHECK(run_background_stream_back_loop_func_ != nullptr); + ICHECK(exit_background_loop_func_ != nullptr); } return self; } -- (void)dealloc { - [self.jsonFFIEngine exitBackgroundLoop]; +- (void)initBackgroundEngine:(void (^)(NSString*))streamCallback { + TypedFunction internal_stream_callback([streamCallback](String value) { + streamCallback([NSString stringWithUTF8String:value.c_str()]); + }); + int device_type = kDLMetal; + int device_id = 0; + init_background_engine_func_(device_type, device_id, internal_stream_callback); +} + +- (void)reload:(NSString*)engineConfigJson { + std::string engine_config = engineConfigJson.UTF8String; + reload_func_(engine_config); } -- (void)reloadWithModelPath:(NSString*)modelPath modelLib:(NSString*)modelLib { - NSString* engineConfig = - [NSString stringWithFormat:@"{\"model\": \"%@\", \"model_lib\": \"system://%@\", \"mode\": \"interactive\"}", modelPath, modelLib]; - [self.jsonFFIEngine reload:engineConfig]; +- (void)unload { + unload_func_(); } - (void)reset { - [self.jsonFFIEngine reset]; + reset_func_(); } -- (void)unload { - [self.jsonFFIEngine unload]; +- (void)chatCompletion:(NSString*)requestJSON requestID:(NSString*)requestID { + std::string request_json = requestJSON.UTF8String; + std::string request_id = requestID.UTF8String; + chat_completion_func_(request_json, request_id); +} + +- (void)abort:(NSString*)requestID { + std::string request_id = requestID.UTF8String; + abort_func_(request_id); +} + +- (void)runBackgroundLoop { + run_background_loop_func_(); } -- (NSString*)chatCompletionWithMessages:(NSArray*)messages options:(NSDictionary*)options completion:(void (^)(NSDictionary* response))completion { - return [self.state chatCompletionWithJSONFFIEngine:self.jsonFFIEngine request:options completion:completion]; +- (void)runBackgroundStreamBackLoop { + run_background_stream_back_loop_func_(); } -- (void)cancelRequest:(NSString *)requestId { - [self.state cancelRequest:requestId withJSONFFIEngine:self.jsonFFIEngine]; +- (void)exitBackgroundLoop { + exit_background_loop_func_(); } @end From 2698748c8b216a2b2f5c7b75fec1cec86a428b8a Mon Sep 17 00:00:00 2001 From: Artur Morys - Magiera Date: Fri, 6 Feb 2026 00:19:54 +0100 Subject: [PATCH 02/25] fix: improve demo app UI on Android (#191) * fix: improve demo app UI on Android * chore: revert accidental change * chore: changes after CR * chore: drop obsolete import * fix: comply with typecheck --- apps/expo-example/app.json | 1 - apps/expo-example/src/App.tsx | 2 +- .../RecordButton/RecordButtonUIBase.tsx | 14 +++++++--- .../components/adapters/appleSetupAdapter.ts | 1 - .../adapters/mlcModelSetupAdapter.ts | 8 +++--- .../src/config/providers.common.ts | 2 +- .../src/screens/ChatScreen/ChatInputBar.tsx | 14 +++++----- .../screens/ChatScreen/ModelPickerSheet.tsx | 2 +- .../src/screens/ChatScreen/SettingsSheet.tsx | 27 +++++++++++++++++-- apps/expo-example/src/store/chatStore.ts | 5 +++- apps/expo-example/src/store/providerStore.ts | 4 +-- apps/expo-example/src/theme/colors.android.ts | 4 +-- apps/expo-example/src/utils/storage.ts | 1 + apps/expo-example/tsconfig.json | 3 +-- 14 files changed, 60 insertions(+), 28 deletions(-) diff --git a/apps/expo-example/app.json b/apps/expo-example/app.json index 52acc6bb..4fa48af3 100644 --- a/apps/expo-example/app.json +++ b/apps/expo-example/app.json @@ -17,7 +17,6 @@ "bundleIdentifier": "com.callstack.ai.example" }, "android": { - "forceDarkAllowed": false, "adaptiveIcon": { "foregroundImage": "./assets/adaptive-icon.png", "backgroundColor": "#ffffff" diff --git a/apps/expo-example/src/App.tsx b/apps/expo-example/src/App.tsx index 87f35034..c789eee3 100644 --- a/apps/expo-example/src/App.tsx +++ b/apps/expo-example/src/App.tsx @@ -190,7 +190,7 @@ const styles = StyleSheet.create({ }, drawer: { width: 280, - backgroundColor: colors.systemBackground, + backgroundColor: '#fff', }, drawerScroll: { flex: 1, diff --git a/apps/expo-example/src/components/RecordButton/RecordButtonUIBase.tsx b/apps/expo-example/src/components/RecordButton/RecordButtonUIBase.tsx index 8f65f449..dbbf5fd5 100644 --- a/apps/expo-example/src/components/RecordButton/RecordButtonUIBase.tsx +++ b/apps/expo-example/src/components/RecordButton/RecordButtonUIBase.tsx @@ -186,11 +186,13 @@ export function RecordButtonUIBase({ ) } + const uiDisabled = disabled || isProcessing || !transcriptionModel + return ( {isProcessing ? ( @@ -202,9 +204,15 @@ export function RecordButtonUIBase({ } + fallback={ + + } /> )} diff --git a/apps/expo-example/src/components/adapters/appleSetupAdapter.ts b/apps/expo-example/src/components/adapters/appleSetupAdapter.ts index 08b43152..499813cf 100644 --- a/apps/expo-example/src/components/adapters/appleSetupAdapter.ts +++ b/apps/expo-example/src/components/adapters/appleSetupAdapter.ts @@ -8,7 +8,6 @@ export const createAppleLanguageSetupAdapter = ( tools: ToolSet = {} ): SetupAdapter => { const apple = createAppleProvider({ - // @ts-expect-error availableTools: tools, }) const model = apple.languageModel() diff --git a/apps/expo-example/src/components/adapters/mlcModelSetupAdapter.ts b/apps/expo-example/src/components/adapters/mlcModelSetupAdapter.ts index a6108653..7352dddc 100644 --- a/apps/expo-example/src/components/adapters/mlcModelSetupAdapter.ts +++ b/apps/expo-example/src/components/adapters/mlcModelSetupAdapter.ts @@ -1,6 +1,6 @@ import type { LanguageModelV3 } from '@ai-sdk/provider' import { mlc } from '@react-native-ai/mlc' -import RNBlobUtil from 'react-native-blob-util' +import { File, Paths } from 'expo-file-system' import type { Availability, SetupAdapter } from '../../config/providers.common' @@ -18,10 +18,8 @@ export const createMLCLanguageSetupAdapter = ( icon: 'cpu', }, builtIn: false, - async isAvailable(): Promise { - return (await RNBlobUtil.fs.exists( - RNBlobUtil.fs.dirs.SDCardDir + `/${modelId}/tensor-cache.json` - )) + isAvailable(): Availability { + return new File(Paths.document, model.modelId, 'tensor-cache.json').exists ? 'yes' : 'availableForDownload' }, diff --git a/apps/expo-example/src/config/providers.common.ts b/apps/expo-example/src/config/providers.common.ts index 1b564643..cf861b29 100644 --- a/apps/expo-example/src/config/providers.common.ts +++ b/apps/expo-example/src/config/providers.common.ts @@ -25,7 +25,7 @@ export interface SetupAdapter { // Whether the model is built-in (true) or downloadable (false) builtIn: boolean // Check if model is ready, unavailable, or downloadable - isAvailable: () => Availability | Promise + isAvailable: () => Availability // Download the model with progress callback download: (onProgress: (percentage: number) => void) => Promise // Remove the downloaded model from storage diff --git a/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx b/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx index 86242a59..0bb571bc 100644 --- a/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx +++ b/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx @@ -31,18 +31,20 @@ export function ChatInputBar({ onSend, isGenerating }: ChatInputBarProps) {