feat(s3): implement multipart upload

Author: denolfe

.gitignore

@@ -1,3 +1,5 @@
+dev/tmp
+
 # Created by https://www.gitignore.io/api/node,macos,windows,webstorm,sublimetext,visualstudiocode
 
 ### macOS ###

README.md

@@ -45,6 +45,8 @@ This plugin supports the following adapters:
 
 However, you can create your own adapter for any third-party service you would like to use.
 
+All adapters are implemented `dev` directory's [Payload Config](https://github.com/payloadcms/plugin-cloud-storage/blob/master/dev/src/payload.config.ts). See this file for examples.
+
 ## Plugin options
 
 This plugin is configurable to work across many different Payload collections. A `*` denotes that the property is required.
@@ -84,7 +86,9 @@ const adapter = azureBlobStorageAdapter({
 
 ### S3 Adapter
 
-To use the S3 adapter, you need to have `@aws-sdk/client-s3` installed in your project dependencies. To do so, run `yarn add @aws-sdk/client-s3`.
+To use the S3 adapter, some peer dependencies need to be installed:
+
+`yarn add @aws-sdk/client-s3 @aws-sdk/lib-storage`.
 
 From there, create the adapter, passing in all of its required properties:
 
@@ -97,13 +101,18 @@ const adapter = s3Adapter({
       accessKeyId: process.env.S3_ACCESS_KEY_ID,
       secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
     }
+    // ... Other S3 configuration
   },
   bucket: process.env.S3_BUCKET,
 })
 
 // Now you can pass this adapter to the plugin
 ```
 
+Other S3 Client configuration is documented [here](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html).
+
+Any upload over 50MB will automatically be uploaded using S3's multi-part upload.
+
 #### Other S3-Compatible Storage
 
 If you're running an S3-compatible object storage such as MinIO or Digital Ocean Spaces, you'll have to set the `endpoint` appropriately for the provider.

dev/package.json

@@ -19,7 +19,7 @@
     "@google-cloud/storage": "^6.4.2",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
-    "payload": "^1.1.3"
+    "payload": "^1.6.6"
   },
   "devDependencies": {
     "@types/express": "^4.17.9",

dev/yarn.lock

@@ -13,14 +13,18 @@
   "prepublishOnly": "yarn build",
   "peerDependencies": {
     "@aws-sdk/client-s3": "^3.142.0",
+    "@aws-sdk/lib-storage": "^3.267.0",
     "@azure/storage-blob": "^12.11.0",
     "@google-cloud/storage": "^6.4.1",
-    "payload": "^1.0.27"
+    "payload": "^1.6.6"
   },
   "peerDependenciesMeta": {
     "@aws-sdk/client-s3": {
       "optional": true
     },
+    "@aws-sdk/lib-storage": {
+      "optional": true
+    },
     "@azure/storage-blob": {
       "optional": true
     },
@@ -49,7 +53,7 @@
     "eslint-plugin-import": "2.25.4",
     "eslint-plugin-prettier": "^4.0.0",
     "nodemon": "^2.0.6",
-    "payload": "^1.1.3",
+    "payload": "^1.6.6",
     "prettier": "^2.7.1",
     "ts-node": "^9.1.1",
     "typescript": "^4.1.3"

package.json

@@ -3,6 +3,8 @@ import path from 'path'
 import type * as AWS from '@aws-sdk/client-s3'
 import type { CollectionConfig } from 'payload/types'
 import type stream from 'stream'
+
+import { Upload } from '@aws-sdk/lib-storage'
 import type { HandleUpload } from '../../types'
 
 interface Args {
@@ -13,6 +15,8 @@ interface Args {
   getStorageClient: () => AWS.S3
 }
 
+const multipartThreshold = 1024 * 1024 * 50 // 50MB
+
 export const getHandleUpload = ({
   getStorageClient,
   bucket,
@@ -22,21 +26,37 @@ export const getHandleUpload = ({
   return async ({ data, file }) => {
     const fileKey = path.posix.join(prefix, file.filename)
 
-    let fileBufferOrStream: Buffer | stream.Readable
-    if (file.tempFilePath) {
-      fileBufferOrStream = fs.createReadStream(file.tempFilePath)
-    } else {
-      fileBufferOrStream = file.buffer
+    const fileBufferOrStream: Buffer | stream.Readable = file.tempFilePath
+      ? fs.createReadStream(file.tempFilePath)
+      : file.buffer
+
+    if (file.buffer.length > 0 && file.buffer.length < multipartThreshold) {
+      await getStorageClient().putObject({
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      })
+
+      return data
     }
 
-    await getStorageClient().putObject({
-      Bucket: bucket,
-      Key: fileKey,
-      Body: fileBufferOrStream,
-      ACL: acl,
-      ContentType: file.mimeType,
+    const parallelUploadS3 = new Upload({
+      client: getStorageClient(),
+      params: {
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      },
+      queueSize: 4,
+      partSize: multipartThreshold,
     })
 
+    await parallelUploadS3.done()
+
     return data
   }
 }

src/adapters/s3/handleUpload.ts

@@ -1,7 +1,8 @@
-exports.S3 = function () {
-  return null
-}
+exports.S3 = () => null
 
-exports.HeadObjectCommand = function () {
-  return null
-}
+exports.HeadObjectCommand = () => null
+exports.PutObjectCommand = () => null
+exports.UploadPartCommand = () => null
+exports.CreateMultipartUploadCommand = () => null
+exports.CompleteMultipartUploadCommand = () => null
+exports.PutObjectTaggingCommand = () => null

src/adapters/s3/mock.js

@@ -9,6 +9,7 @@ export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): Webpa
       alias: {
         ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
         '@aws-sdk/client-s3': path.resolve(__dirname, './mock.js'),
+        '@aws-sdk/lib-storage': path.resolve(__dirname, './mock.js'),
       },
     },
   }