Skip to content

Commit 828087f

Browse files
authored
feat(web, ci): add publish script; automatically publish for dev on every merge to main COMPASS-10156 (#7637)
* chore(web): move upload script to release dir; move some reusable code to its own file; adjust tsconfig * feat(web, ci): add publish script; add publish-web step to publish CI flow * fix(web): max-age is seconds, not millis * chore(web): update object key resolution to invalidate latest cache * chore(web): add more logging to publish script * chore(ci): only run publish on merges to main
1 parent 984776f commit 828087f

File tree

10 files changed

+297
-100
lines changed

10 files changed

+297
-100
lines changed

.evergreen/buildvariants-and-tasks.in.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,7 @@ tasks:
424424
- func: bootstrap
425425
vars:
426426
scope: "@mongodb-js/compass-web"
427+
- func: upload-web
427428
- func: publish-web
428429

429430
- name: publish-dev-release-info

.evergreen/buildvariants-and-tasks.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,7 @@ tasks:
428428
- func: bootstrap
429429
vars:
430430
scope: '@mongodb-js/compass-web'
431+
- func: upload-web
431432
- func: publish-web
432433
- name: publish-dev-release-info
433434
tags: []

.evergreen/functions.yml

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
# The variables are with the functions because they are only used by the
2-
# functions and also because you can't use variables across includes.
1+
# The variables and paratemers are with the functions because they are only used
2+
# by the functions and also because you can't use variables across includes.
33
variables:
44
- &save-artifact-params-private
55
aws_key: ${aws_key}
@@ -77,6 +77,13 @@ variables:
7777
E2E_TESTS_ATLAS_CS_WITHOUT_SEARCH: ${e2e_tests_atlas_cs_without_search}
7878
E2E_TESTS_ATLAS_CS_WITH_SEARCH: ${e2e_tests_atlas_cs_with_search}
7979

80+
parameters:
81+
- key: compass_web_publish_environment
82+
value: dev
83+
description: Atlas Cloud environment that compass-web will be deployed for during a CI run
84+
- key: compass_web_release_commit
85+
description: Exact commit that will be deployed as a latest release for provided compass_web_publish_environment
86+
8087
# This is here with the variables because anchors aren't supported across includes
8188
post:
8289
- command: archive.targz_pack
@@ -521,7 +528,7 @@ functions:
521528
echo "Uploading release assets to S3 and GitHub if needed..."
522529
npm run --workspace mongodb-compass upload
523530
524-
publish-web:
531+
upload-web:
525532
- command: ec2.assume_role
526533
params:
527534
role_arn: ${downloads_bucket_role_arn}
@@ -540,6 +547,29 @@ functions:
540547
echo "Uploading release assets to S3"
541548
npm run --workspace @mongodb-js/compass-web upload
542549
550+
publish-web:
551+
- command: ec2.assume_role
552+
params:
553+
role_arn: ${downloads_bucket_role_arn}
554+
- command: shell.exec
555+
params:
556+
working_dir: src
557+
shell: bash
558+
env:
559+
<<: *compass-env
560+
COMPASS_WEB_PUBLISH_ENVIRONMENT: '${compass_web_publish_environment}'
561+
COMPASS_WEB_RELEASE_COMMIT: '${compass_web_release_commit}'
562+
script: |
563+
set -e
564+
# Load environment variables
565+
eval $(.evergreen/print-compass-env.sh)
566+
# Deploy to dev on every commit to main, do nothing for release branches
567+
if [[ "$EVERGREEN_PROJECT" == '10gen-compass-main' && "$EVERGREEN_IS_PATCH" != "true" ]]; then
568+
npm run --workspace @mongodb-js/compass-web publish
569+
else
570+
echo "Skipping publish: wrong project ($EVERGREEN_PROJECT) or patch ($EVERGREEN_IS_PATCH)"
571+
fi
572+
543573
publish-dev-release-info:
544574
- command: shell.exec
545575
params:

packages/compass-web/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@
6363
"test-watch": "npm run test -- --watch",
6464
"test-ci": "npm run test-cov",
6565
"reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
66-
"upload": "node --experimental-strip-types scripts/upload.mts"
66+
"upload": "node --experimental-strip-types scripts/release/upload.mts",
67+
"publish": "node --experimental-strip-types scripts/release/publish.mts"
6768
},
6869
"peerDependencies": {
6970
"react": "^17.0.2",
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import path from 'path';
2+
import { brotliCompressSync } from 'zlib';
3+
import { inspect } from 'util';
4+
import {
5+
ALLOWED_PUBLISH_ENVIRONMENTS,
6+
DOWNLOADS_BUCKET,
7+
DOWNLOADS_BUCKET_PUBLIC_HOST,
8+
ENTRYPOINT_FILENAME,
9+
MANIFEST_FILENAME,
10+
PUBLISH_ENVIRONMENT,
11+
RELEASE_COMMIT,
12+
asyncPutObject,
13+
getObjectKey,
14+
} from './utils.mts';
15+
16+
console.log(
17+
'Publishing compass-web@%s to %s environment',
18+
RELEASE_COMMIT,
19+
PUBLISH_ENVIRONMENT
20+
);
21+
22+
if (!ALLOWED_PUBLISH_ENVIRONMENTS.includes(PUBLISH_ENVIRONMENT ?? '')) {
23+
throw new Error(
24+
`Unknown publish environment: expected ${inspect(
25+
ALLOWED_PUBLISH_ENVIRONMENTS
26+
)}, got ${inspect(PUBLISH_ENVIRONMENT)}`
27+
);
28+
}
29+
30+
const publicManifestUrl = new URL(
31+
getObjectKey(MANIFEST_FILENAME),
32+
DOWNLOADS_BUCKET_PUBLIC_HOST
33+
);
34+
35+
const publicEntrypointUrl = new URL(
36+
getObjectKey(ENTRYPOINT_FILENAME),
37+
DOWNLOADS_BUCKET_PUBLIC_HOST
38+
);
39+
40+
let assets: URL[];
41+
42+
function assertResponseIsOk(res: Response) {
43+
if (res.status !== 200) {
44+
throw new Error(
45+
`Request returned a non-OK response: ${res.status} ${res.statusText}`
46+
);
47+
}
48+
}
49+
50+
console.log('Fetching manifest from %s', publicManifestUrl);
51+
52+
try {
53+
const res = await fetch(publicManifestUrl);
54+
assertResponseIsOk(res);
55+
const manifest = await res.json();
56+
57+
if (
58+
!(
59+
Array.isArray(manifest) &&
60+
manifest.every((asset) => {
61+
return typeof asset === 'string';
62+
})
63+
)
64+
) {
65+
throw new Error(
66+
`Manifest schema is not matching: expected string[], got ${inspect(
67+
manifest
68+
)}`
69+
);
70+
}
71+
72+
console.log('Checking that assets in manifest exist');
73+
74+
assets = manifest.map((asset) => {
75+
return new URL(getObjectKey(asset), DOWNLOADS_BUCKET_PUBLIC_HOST);
76+
});
77+
78+
await Promise.all(
79+
assets.map(async (assetUrl) => {
80+
const res = await fetch(assetUrl, { method: 'HEAD' });
81+
assertResponseIsOk(res);
82+
})
83+
);
84+
} catch (err) {
85+
throw new AggregateError(
86+
[err],
87+
`Aborting publish, failed to resolve manifest ${publicManifestUrl}`
88+
);
89+
}
90+
91+
function buildProxyEntrypointFile() {
92+
return (
93+
assets
94+
.map((asset) => {
95+
return `import ${JSON.stringify(asset)};`;
96+
})
97+
.concat(
98+
`export * from ${JSON.stringify(publicEntrypointUrl)};`,
99+
`/** Compass version: https://github.com/mongodb-js/compass/tree/${RELEASE_COMMIT} */`
100+
)
101+
.join('\n') + '\n'
102+
);
103+
}
104+
105+
const fileKey = getObjectKey('index.mjs', PUBLISH_ENVIRONMENT);
106+
const fileContent = buildProxyEntrypointFile();
107+
const compressedFileContent = brotliCompressSync(fileContent);
108+
109+
console.log(
110+
'Uploading entrypoint to s3://%s/%s ...',
111+
DOWNLOADS_BUCKET,
112+
fileKey
113+
);
114+
115+
const ENTRYPOINT_CACHE_MAX_AGE_SECONDS = 1 * 60 * 3; // 3mins
116+
117+
const res = await asyncPutObject({
118+
ACL: 'private',
119+
Bucket: DOWNLOADS_BUCKET,
120+
Key: fileKey,
121+
Body: compressedFileContent,
122+
ContentType: 'text/javascript',
123+
ContentEncoding: 'br',
124+
ContentLength: compressedFileContent.byteLength,
125+
// "Latest" entrypoint files can change quite often, so max-age is quite
126+
// short and browser should always revalidate on stale
127+
CacheControl: `public, max-age=${ENTRYPOINT_CACHE_MAX_AGE_SECONDS}, must-revalidate`,
128+
});
129+
130+
console.log(
131+
'Successfully uploaded %s (ETag: %s)',
132+
path.basename(fileKey),
133+
res.ETag
134+
);
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import fs from 'fs';
2+
import path from 'path';
3+
import { brotliCompressSync } from 'zlib';
4+
import {
5+
DIST_DIR,
6+
DOWNLOADS_BUCKET,
7+
asyncPutObject,
8+
getObjectKey,
9+
} from './utils.mts';
10+
11+
const artifacts = await fs.promises.readdir(DIST_DIR);
12+
13+
if (!artifacts.length) {
14+
throw new Error('No artifact files found');
15+
}
16+
17+
const contentTypeForExt: Record<string, string> = {
18+
'.mjs': 'text/javascript',
19+
'.txt': 'text/plain', // extracted third party license info
20+
'.ts': 'text/typescript', // type definitions
21+
'.json': 'application/json', // tsdoc / assets meta
22+
};
23+
24+
const ALLOWED_EXTS = Object.keys(contentTypeForExt);
25+
26+
for (const file of artifacts) {
27+
if (!ALLOWED_EXTS.includes(path.extname(file))) {
28+
throw new Error(`Unexpected artifact file extension for ${file}`);
29+
}
30+
}
31+
32+
const IMMUTABLE_CACHE_MAX_AGE_SECONDS = 1 * 60 * 60 * 24 * 7; // a week
33+
34+
for (const file of artifacts) {
35+
const filePath = path.join(DIST_DIR, file);
36+
const objectKey = getObjectKey(file);
37+
38+
console.log(
39+
'Uploading compass-web/dist/%s to s3://%s/%s ...',
40+
file,
41+
DOWNLOADS_BUCKET,
42+
objectKey
43+
);
44+
45+
const fileContent = fs.readFileSync(filePath, 'utf8');
46+
const compressedFileContent = brotliCompressSync(fileContent);
47+
48+
const res = await asyncPutObject({
49+
ACL: 'private',
50+
Bucket: DOWNLOADS_BUCKET,
51+
Key: objectKey,
52+
Body: compressedFileContent,
53+
ContentType: contentTypeForExt[path.extname(file)],
54+
ContentEncoding: 'br',
55+
ContentLength: compressedFileContent.byteLength,
56+
// Assets stored under the commit hash never change after upload, so the
57+
// cache-control setting for them can be quite generous
58+
CacheControl: `public, max-age=${IMMUTABLE_CACHE_MAX_AGE_SECONDS}, immutable`,
59+
});
60+
61+
console.log('Successfully uploaded %s (ETag: %s)', file, res.ETag);
62+
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import S3 from 'aws-sdk/clients/s3.js';
2+
import child_process from 'child_process';
3+
import path from 'path';
4+
import { promisify } from 'util';
5+
6+
// TODO(SRE-4971): replace with a compass-web-only bucket when provisioned
7+
export const DOWNLOADS_BUCKET = 'cdn-origin-compass';
8+
9+
export const DOWNLOADS_BUCKET_PUBLIC_HOST = 'https://downloads.mongodb.com';
10+
11+
export const ENTRYPOINT_FILENAME = 'compass-web.mjs';
12+
13+
export const MANIFEST_FILENAME = 'assets-manifest.json';
14+
15+
export const DIST_DIR = path.resolve(import.meta.dirname, '..', '..', 'dist');
16+
17+
export const ALLOWED_PUBLISH_ENVIRONMENTS = ['dev', 'qa', 'staging', 'prod'];
18+
19+
export const PUBLISH_ENVIRONMENT = process.env.COMPASS_WEB_PUBLISH_ENVIRONMENT;
20+
21+
export const RELEASE_COMMIT =
22+
process.env.COMPASS_WEB_RELEASE_COMMIT ||
23+
child_process
24+
.spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' })
25+
.stdout.trim();
26+
27+
function getAWSCredentials() {
28+
if (
29+
!process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID ||
30+
!process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY ||
31+
!process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN
32+
) {
33+
throw new Error('Missing required env variables');
34+
}
35+
return {
36+
accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID,
37+
secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY,
38+
sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN,
39+
};
40+
}
41+
const s3Client = new S3({
42+
credentials: getAWSCredentials(),
43+
});
44+
45+
export const asyncPutObject: (
46+
params: S3.Types.PutObjectRequest
47+
) => Promise<S3.Types.PutObjectOutput> = promisify(
48+
s3Client.putObject.bind(s3Client)
49+
);
50+
51+
export function getObjectKey(filename: string, release = RELEASE_COMMIT) {
52+
// TODO(SRE-4971): while we're uploading to the downloads bucket, the object
53+
// key always needs to start with `compass/`
54+
return `compass/compass-web/${release}/${filename}`;
55+
}

0 commit comments

Comments
 (0)