fix: empty mimeType error on Windows (#1141)

Author: toshikwa

packages/cdk/lambda/utils/media.ts

@@ -3,7 +3,10 @@ import {
   ImageFormat,
   VideoFormat,
 } from '@aws-sdk/client-bedrock-runtime';
-import { SupportedMimeType } from '@generative-ai-use-cases/common';
+import {
+  extensionToMimeType,
+  SupportedMimeType,
+} from '@generative-ai-use-cases/common';
 
 const SupportedFormat = {
   ...DocumentFormat,
@@ -27,3 +30,11 @@ export const getFormatFromMimeType = (mimeType: string) => {
   }
   throw new Error(`Unsupported MIME type: ${mimeType}`);
 };
+
+export const getMimeTypeFromFileName = (fileName: string) => {
+  const extension = fileName.split('.').pop()?.toLowerCase() || '';
+  if (extension in extensionToMimeType) {
+    return extensionToMimeType[extension];
+  }
+  throw new Error(`Unsupported file extension: ${fileName}`);
+};

packages/cdk/lambda/utils/models.ts

@@ -31,7 +31,7 @@ import {
   applyAutoCacheToMessages,
   applyAutoCacheToSystem,
 } from './promptCache';
-import { getFormatFromMimeType } from './media';
+import { getFormatFromMimeType, getMimeTypeFromFileName } from './media';
 
 // Default Models
 
@@ -358,10 +358,16 @@ const createConverseCommandInput = (
     // Put images, videos, and documents before the task, instruction, and user query
     if (message.extraData) {
       message.extraData.forEach((extra) => {
+        // Prior to v4.2.4, 'extra.source.mediaType' could be empty.
+        // For resumed conversations from older versions, we fallback to detecting mimeType based on the extension.
+        const mimeType =
+          extra.source.mediaType || getMimeTypeFromFileName(extra.name);
+        const format = getFormatFromMimeType(mimeType);
+
         if (extra.type === 'image' && extra.source.type === 'base64') {
           contentBlocks.push({
             image: {
-              format: getFormatFromMimeType(extra.source.mediaType),
+              format,
               source: {
                 bytes: Buffer.from(extra.source.data, 'base64'),
               },
@@ -370,7 +376,7 @@ const createConverseCommandInput = (
         } else if (extra.type === 'file' && extra.source.type === 'base64') {
           contentBlocks.push({
             document: {
-              format: getFormatFromMimeType(extra.source.mediaType),
+              format,
               name: extra.name
                 .split('.')[0]
                 .replace(/[^a-zA-Z0-9\s\-()[\]]/g, 'X'), // If the file name contains Japanese, it will cause an error, so convert it
@@ -382,7 +388,7 @@ const createConverseCommandInput = (
         } else if (extra.type === 'video' && extra.source.type === 'base64') {
           contentBlocks.push({
             video: {
-              format: getFormatFromMimeType(extra.source.mediaType),
+              format,
               source: {
                 bytes: Buffer.from(extra.source.data, 'base64'),
               },
@@ -391,7 +397,7 @@ const createConverseCommandInput = (
         } else if (extra.type === 'video' && extra.source.type === 's3') {
           contentBlocks.push({
             video: {
-              format: getFormatFromMimeType(extra.source.mediaType),
+              format,
               source: {
                 s3Location: {
                   uri: extra.source.data,

packages/common/src/application/media.ts

@@ -83,3 +83,10 @@ export const mimeTypeToExtensions: Record<SupportedMimeType, string[]> = {
   ...imageMimeTypeToExtensions,
   ...videoMimeTypeToExtensions,
 };
+
+export const extensionToMimeType: Record<string, SupportedMimeType> =
+  Object.fromEntries(
+    Object.entries(mimeTypeToExtensions).flatMap(([mimeType, extensions]) =>
+      extensions.map((ext) => [ext, mimeType as SupportedMimeType])
+    )
+  );

packages/web/src/utils/MediaUtils.ts

@@ -6,6 +6,7 @@ import {
   imageMimeTypeToExtensions,
   videoMimeTypeToExtensions,
   mimeTypeToExtensions,
+  extensionToMimeType,
 } from '@generative-ai-use-cases/common';
 import { fileTypeFromBuffer, fileTypeFromStream } from 'file-type';
 
@@ -36,12 +37,15 @@ export const getMimeTypeFromFileHeader = async (file: File) => {
       throw new Error('Error reading MIME type from file');
     }
   }
+
   // Some file types are not supported by the file-type library.
-  // In this case, we fall back to using 'file.type' for the MIME type, since the file-type library covers most binary file types.
+  // In this case, we fall back to using common MIME types based on the file extension.
   // https://github.com/sindresorhus/file-type/tree/main?tab=readme-ov-file#supported-file-types
   if (!mimeType || mimeType === 'application/x-cfb') {
-    mimeType = file.type;
+    const extension = file.name.split('.').pop()?.toLowerCase() || '';
+    mimeType = extensionToMimeType[extension] || '';
   }
+
   return (mimeTypeAlias[mimeType] || mimeType) as SupportedMimeType;
 };