chore: inference chat preset support (#931)

Author: louis-jan

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -108,12 +108,13 @@ export class ChatCommand extends BaseCommand {
     );
 
     const preset = await this.fileService.getPreset(options.preset);
-    
+      
     return this.cortex.models.start(modelId, preset).then(() =>
       this.chatClient.chat(
         modelId,
         options.threadId,
         message, // Accept both message from inputs or arguments
+        preset ? preset : {},
       ),
     );
   }

cortex-js/src/infrastructure/commanders/services/chat-client.ts

@@ -18,6 +18,7 @@ export class ChatClient {
     modelId: string,
     threadId?: string,
     message?: string,
+    settings?: Partial<Cortex.ChatCompletionCreateParamsStreaming>,
   ): Promise<void> {
     console.log(`In order to exit, type '${this.exitClause}'.`);
     const thread = await this.getOrCreateNewThread(modelId, threadId);
@@ -36,8 +37,16 @@ export class ChatClient {
       output: stdout,
       prompt: this.userIndicator,
     });
+
     if (message)
-      this.sendCompletionMessage(message, messages, modelId, thread.id, rl);
+      this.sendCompletionMessage(
+        message,
+        messages,
+        modelId,
+        thread.id,
+        rl,
+        settings,
+      );
     rl.prompt();
 
     rl.on('close', async () => {
@@ -46,7 +55,7 @@ export class ChatClient {
     });
 
     rl.on('line', (input) =>
-      this.sendCompletionMessage(input, messages, modelId, thread.id, rl),
+      this.sendCompletionMessage(input, messages, modelId, thread.id, rl, settings),
     );
   }
 
@@ -56,6 +65,7 @@ export class ChatClient {
     modelId: string,
     threadId: string,
     rl: readline.Interface,
+    settings: Partial<Cortex.ChatCompletionCreateParamsStreaming> = {},
   ) {
     if (!userInput || userInput.trim() === '') return;
 
@@ -64,6 +74,19 @@ export class ChatClient {
       return;
     }
 
+    if (
+      'pre_prompt' in settings &&
+      !messages.some((m) => m.role === 'system')
+    ) {
+      messages = [
+        {
+          content: settings.pre_prompt as string,
+          role: 'system',
+        },
+        ...messages,
+      ];
+    }
+
     const model = await this.cortex.models.retrieve(modelId);
 
     messages.push({
@@ -89,6 +112,7 @@ export class ChatClient {
       model: modelId,
       max_tokens: 4098,
       temperature: 0.7,
+      ...settings,
       // Override with model inference params
       ...parser.parseModelInferenceParams(model),
       stream: true,

cortex-js/src/infrastructure/services/download-manager/download-manager.service.ts

@@ -234,17 +234,25 @@ export class DownloadManagerService {
         );
         if (downloadItem) {
           downloadItem.size.transferred = transferredBytes;
-          downloadItem.progress = Math.floor(
+          downloadItem.progress = Math.round(
             (transferredBytes / totalBytes) * 100,
           );
           bar.update(downloadItem.progress);
         }
         const lastProgress = currentDownloadState.progress;
-        currentDownloadState.progress = Math.floor(
-          currentDownloadState.children.reduce(
-            (pre, curr) => pre + curr.progress,
+        currentDownloadState.progress = Math.round(
+          (currentDownloadState.children.reduce(
+            (pre, curr) => pre + curr.size.transferred,
             0,
-          ) / Math.max(currentDownloadState.children.length, 1),
+          ) /
+            Math.max(
+              currentDownloadState.children.reduce(
+                (pre, curr) => pre + curr.size.total,
+                0,
+              ),
+              1,
+            )) *
+            100,
         );
         // console.log(currentDownloadState.progress);
         if (currentDownloadState.progress !== lastProgress)