fix: can not chat with remote model (#907)

Author: marknguyen1302

cortex-js/src/extensions/anthropic.engine.ts

@@ -93,7 +93,7 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
       return {
         choices: [
           {
-            delta: {
+            message: {
               content: data.content[0].text,
             },
           },

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -19,12 +19,13 @@ import { Engines } from './types/engine.interface';
 import { join } from 'path';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { FileManagerService } from '../services/file-manager/file-manager.service';
-import { isLocalModel } from '@/utils/normalize-model-id';
+import { isLocalModel, isRemoteEngine } from '@/utils/normalize-model-id';
 
 type ChatOptions = {
   threadId?: string;
   message?: string;
   attach: boolean;
+  preset?: string;
 };
 
 @SubCommand({
@@ -89,6 +90,7 @@ export class ChatCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);
@@ -106,7 +108,7 @@ export class ChatCommand extends BaseCommand {
     );
     return this.cortexUsecases
       .startCortex()
-      .then(() => this.modelsCliUsecases.startModel(modelId))
+      .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
       .then(() =>
         this.chatCliUsecases.chat(
           modelId,
@@ -156,4 +158,12 @@ export class ChatCommand extends BaseCommand {
   parseAttach() {
     return true;
   }
+
+  @Option({
+    flags: '-p, --preset <preset>',
+    description: 'Apply a chat preset to the chat session',
+  })
+  parsePreset(value: string) {
+    return value;
+  }
 }

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -59,7 +59,6 @@ export class CortexCommand extends CommandRunner {
     const port = options?.port || defaultCortexJsPort;
     const showLogs = options?.logs || false;
     const dataFolderPath = options?.dataFolder;
-
     return this.startServer(host, port, showLogs, dataFolderPath);
   }
 
@@ -114,13 +113,15 @@ export class CortexCommand extends CommandRunner {
         apiServerPort: port,
         dataFolderPath: dataFolderPath || config.dataFolderPath,
       });
+      process.exit(1);
     } catch (e) {
       console.error(e);
       // revert the data folder path if it was set
       await this.fileManagerService.writeConfigFile({
         ...config,
       });
       console.error(`Failed to start server. Is port ${port} in use?`);
+      process.exit(0);
     }
   }
 

cortex-js/src/infrastructure/commanders/models/model-start.command.ts

@@ -12,6 +12,7 @@ import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { BaseCommand } from '../base.command';
+import { isRemoteEngine } from '@/utils/normalize-model-id';
 
 type ModelStartOptions = {
   attach: boolean;
@@ -72,6 +73,7 @@ export class ModelStartCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);

cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts

@@ -12,6 +12,7 @@ import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { BaseCommand } from '../base.command';
+import { isLocalModel, isRemoteEngine } from '@/utils/normalize-model-id';
 
 type RunOptions = {
   threadId?: string;
@@ -69,11 +70,7 @@ export class RunCommand extends BaseCommand {
 
     // Second check if model is available
     const existingModel = await this.modelsCliUsecases.getModel(modelId);
-    if (
-      !existingModel ||
-      !Array.isArray(existingModel.files) ||
-      /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0])
-    ) {
+    if (!existingModel || !isLocalModel(existingModel.files)) {
       checkingSpinner.fail(`Model is not available`);
       process.exit(1);
     }
@@ -82,6 +79,7 @@ export class RunCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);

cortex-js/src/infrastructure/commanders/types/engine.interface.ts

@@ -17,3 +17,10 @@ export const EngineNamesMap: {
   [Engines.onnx]: 'onnx',
   [Engines.tensorrtLLM]: 'tensorrt-llm',
 };
+
+export const RemoteEngines: Engines[] = [
+  Engines.groq,
+  Engines.mistral,
+  Engines.openai,
+  Engines.anthropic,
+];

cortex-js/src/utils/normalize-model-id.ts

@@ -1,5 +1,9 @@
 import { ModelArtifact } from '@/domain/models/model.interface';
 import { getGpuInfo } from './cuda';
+import {
+  Engines,
+  RemoteEngines,
+} from '@/infrastructure/commanders/types/engine.interface';
 
 export const normalizeModelId = (modelId: string): string => {
   return modelId.replace(':main', '').replace(/[:/]/g, '-');
@@ -15,6 +19,10 @@ export const isLocalModel = (
   );
 };
 
+export const isRemoteEngine = (engine: string): boolean => {
+  return RemoteEngines.includes(engine as Engines);
+};
+
 /**
  * Parse the model hub engine branch
  * @param branch