feat(web, ci): upload compass-web assets to an s3 bucket during publish COMPASS-10132 (#7631)

* chore(webpack): make sure that images are always inlined in web build

* chore(web): add upload script for compass-web

* chore(ci): move downloads bucket env to compass-env anchor; move role_arn to project config

* feat(ci): add publish-web task to the pipeline

* chore(web): switch back to aws sdk2 to avoid too many dependencies being updated

* chore(web): add cache-control headers

* chore(web): adjust assets cache control policy

Author: gribnoysup

.evergreen/buildvariants-and-tasks.in.yml

@@ -252,6 +252,7 @@ buildvariants:
     run_on: ubuntu2204-large
     tasks:
       - name: publish
+      - name: publish-web
       - name: publish-dev-release-info
 
   - name: static-analysis
@@ -410,6 +411,21 @@ tasks:
       - func: get-all-artifacts
       - func: publish
 
+  - name: publish-web
+    # Publish both on PRs and on main (if it gets too noisy and won't be used
+    # much we can probably disable for PRs)
+    tags: ['run-on-pr']
+    depends_on:
+      - name: '.required-for-publish'
+        variant: '*'
+    commands:
+      - func: prepare
+      - func: install
+      - func: bootstrap
+        vars:
+          scope: "@mongodb-js/compass-web"
+      - func: publish-web
+
   - name: publish-dev-release-info
     tags: []
     depends_on:

.evergreen/buildvariants-and-tasks.yml

@@ -229,7 +229,7 @@ buildvariants:
   - name: test-packaged-app-macos-15-x64
     display_name: Test Packaged App MacOS x64 15
     run_on: macos-15-amd64-gui
-    patchable: true
+    patchable: false
     depends_on:
       - name: package-compass
         variant: package-macos-x64
@@ -242,6 +242,7 @@ buildvariants:
     run_on: ubuntu2204-large
     tasks:
       - name: publish
+      - name: publish-web
       - name: publish-dev-release-info
   - name: static-analysis
     display_name: Create Static Analysis Report
@@ -415,6 +416,19 @@ tasks:
           scope: mongodb-compass
       - func: get-all-artifacts
       - func: publish
+  - name: publish-web
+    tags:
+      - run-on-pr
+    depends_on:
+      - name: .required-for-publish
+        variant: '*'
+    commands:
+      - func: prepare
+      - func: install
+      - func: bootstrap
+        vars:
+          scope: '@mongodb-js/compass-web'
+      - func: publish-web
   - name: publish-dev-release-info
     tags: []
     depends_on:

.evergreen/functions.yml

@@ -48,6 +48,9 @@ variables:
     GITHUB_TOKEN: ${devtoolsbot_github_token}
     DOWNLOAD_CENTER_AWS_ACCESS_KEY_ID: ${aws_key_evergreen_integrations}
     DOWNLOAD_CENTER_AWS_SECRET_ACCESS_KEY: ${aws_secret_evergreen_integrations}
+    DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+    DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+    DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
     EVERGREEN_BUCKET_NAME: mciuploads
     EVERGREEN_BUCKET_KEY_PREFIX: ${project}/${revision}_${revision_order_id}
     MONGODB_RUNNER_LOG_DIR: ${workdir}/src/.testserver/
@@ -504,23 +507,39 @@ functions:
   publish:
     - command: ec2.assume_role
       params:
-        role_arn: 'arn:aws:iam::119629040606:role/s3-access.cdn-origin-compass'
+        role_arn: ${downloads_bucket_role_arn}
     - command: shell.exec
       params:
         working_dir: src
         shell: bash
         env:
           <<: *compass-env
-          DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-          DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-          DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
         script: |
           set -e
           # Load environment variables
           eval $(.evergreen/print-compass-env.sh)
           echo "Uploading release assets to S3 and GitHub if needed..."
           npm run --workspace mongodb-compass upload
 
+  publish-web:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${downloads_bucket_role_arn}
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          <<: *compass-env
+        script: |
+          set -e
+          # Load environment variables
+          eval $(.evergreen/print-compass-env.sh)
+          echo "Compiling compass-web"
+          npm run --workspace @mongodb-js/compass-web compile
+          echo "Uploading release assets to S3"
+          npm run --workspace @mongodb-js/compass-web upload
+
   publish-dev-release-info:
     - command: shell.exec
       params:

configs/webpack-config-compass/src/index.ts

@@ -22,7 +22,8 @@ import {
   sourceLoader,
   cssLoader,
   lessLoader,
-  assetsLoader,
+  fontLoader,
+  imageLoader,
   resourceLoader,
   sharedObjectLoader,
 } from './loaders';
@@ -225,7 +226,8 @@ export function createElectronRendererConfig(
         nodeLoader(opts),
         cssLoader(opts),
         lessLoader(opts),
-        assetsLoader(opts),
+        fontLoader(opts),
+        imageLoader(opts),
         sharedObjectLoader(opts),
         sourceLoader(opts),
       ],
@@ -370,7 +372,8 @@ export function createWebConfig(args: Partial<ConfigArgs>): WebpackConfig {
         nodeLoader(opts),
         cssLoader(opts, true),
         lessLoader(opts),
-        assetsLoader(opts),
+        fontLoader(opts),
+        imageLoader(opts),
         sourceLoader(opts),
       ],
       parser: {

configs/webpack-config-compass/src/loaders.ts

@@ -187,16 +187,18 @@ export const lessLoader = (args: ConfigArgs) => ({
 });
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export const assetsLoader = (_args: ConfigArgs) => ({
-  test: /\.(jpe?g|png|svg|gif|woff|woff2|ttf|eot|otf)(\?.+?)?$/,
-  // asset (or asset auto) will either compile as data-uri or to a file path
-  // based on the size, this is a good strategy for loading assets in the GUI
-  type: 'asset',
-  parser: {
-    dataUrlCondition: {
-      maxSize: 2 * 1024, // 2kb
-    },
-  },
+export const fontLoader = (_args: ConfigArgs) => ({
+  test: /\.(woff|woff2|ttf|eot|otf)(\?.+?)?$/,
+  // fonts are always big and should be emitted as a separate file
+  type: 'asset/resource',
+});
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export const imageLoader = (_args: ConfigArgs) => ({
+  test: /\.(jpe?g|png|svg|gif)(\?.+?)?$/,
+  // it's convenient to inline images as data-urls to make sure that publised
+  // library artifacts only produce importable javascript assets
+  type: 'asset/inline',
 });
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -223,7 +225,7 @@ export const sourceLoader = (args: ConfigArgs) => ({
     nodeLoader(args).test,
     cssLoader(args).test,
     lessLoader(args).test,
-    assetsLoader(args).test,
+    resourceLoader(args).test,
     sharedObjectLoader(args).test,
     // Produced by html-webpack-plugin and should not be handled
     /\.(ejs|html)$/,

package-lock.json

@@ -62,7 +62,8 @@
     "test-cov": "nyc --compact=false --produce-source-map=false -x \"**/*.spec.*\" --reporter=lcov --reporter=text --reporter=html npm run test",
     "test-watch": "npm run test -- --watch",
     "test-ci": "npm run test-cov",
-    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ."
+    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
+    "upload": "node --experimental-strip-types scripts/upload.mts"
   },
   "peerDependencies": {
     "react": "^17.0.2",
@@ -115,6 +116,7 @@
     "@types/react": "^17.0.5",
     "@types/react-dom": "^17.0.10",
     "@types/sinon-chai": "^3.2.5",
+    "aws-sdk": "^2.1692.0",
     "browser-process-hrtime": "^1.0.0",
     "bson": "^6.10.4",
     "buffer": "^6.0.3",

packages/compass-web/package.json

@@ -0,0 +1,95 @@
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import { brotliCompressSync } from 'zlib';
+import { promisify } from 'util';
+import S3 from 'aws-sdk/clients/s3.js';
+
+// TODO(SRE-4971): replace with a compass-web-only bucket when provisioned
+const DOWNLOADS_BUCKET = 'cdn-origin-compass';
+
+const DIST_DIR = path.resolve(import.meta.dirname, '..', 'dist');
+
+const HEAD = child_process
+  .spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' })
+  .stdout.trim();
+
+function getCredentials() {
+  if (
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID ||
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY ||
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN
+  ) {
+    throw new Error('Missing required env variables');
+  }
+  return {
+    accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY,
+    sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN,
+  };
+}
+
+const artifacts = await fs.promises.readdir(DIST_DIR);
+
+if (!artifacts.length) {
+  throw new Error('No artifact files found');
+}
+
+const contentTypeForExt: Record<string, string> = {
+  '.mjs': 'text/javascript',
+  '.txt': 'text/plain', // extracted third party license info
+  '.ts': 'text/typescript', // type definitions
+  '.json': 'application/json', // tsdoc meta
+};
+
+const ALLOWED_EXTS = Object.keys(contentTypeForExt);
+
+for (const file of artifacts) {
+  if (!ALLOWED_EXTS.includes(path.extname(file))) {
+    throw new Error(`Unexpected artifact file extension for ${file}`);
+  }
+}
+
+const s3Client = new S3({
+  credentials: getCredentials(),
+});
+
+const IMMUTABLE_CACHE_MAX_AGE = 1000 * 60 * 60 * 24 * 7; // a week
+
+for (const file of artifacts) {
+  const filePath = path.join(DIST_DIR, file);
+  // TODO(SRE-4971): while we're uploading to the downloads bucket, the object
+  // key always needs to start with `compass/`
+  const objectKey = `compass/web/${HEAD}/${file}`;
+
+  console.log(
+    'Uploading compass-web/dist/%s to s3://%s/%s ...',
+    file,
+    DOWNLOADS_BUCKET,
+    objectKey
+  );
+
+  const fileContent = fs.readFileSync(filePath, 'utf8');
+  const compressedFileContent = brotliCompressSync(fileContent);
+
+  const asyncPutObject: (
+    params: S3.Types.PutObjectRequest
+  ) => Promise<S3.Types.PutObjectOutput> = promisify(
+    s3Client.putObject.bind(s3Client)
+  );
+
+  const res = await asyncPutObject({
+    ACL: 'private',
+    Bucket: DOWNLOADS_BUCKET,
+    Key: objectKey,
+    Body: compressedFileContent,
+    ContentType: contentTypeForExt[path.extname(file)],
+    ContentEncoding: 'br',
+    ContentLength: compressedFileContent.byteLength,
+    // Assets stored under the commit hash never change after upload, so the
+    // cache-control setting for them can be quite generous
+    CacheControl: `public, max-age=${IMMUTABLE_CACHE_MAX_AGE}, immutable`,
+  });
+
+  console.log('Successfully uploaded %s (ETag: %s)', file, res.ETag);
+}