WIP Compress add brotli support

js2bin.js

@@ -11,23 +11,25 @@ command: --build, --ci, --help
 command-args: take the form of --name=value
 
 --build: embed your application into the precompiled NodeJS binary.
-  --node:     NodeJS version(s) to use, can specify more than one. 
+  --node:     NodeJS version(s) to use, can specify more than one.
               e.g. --node=10.16.0 --node=12.4.0
   --platform: Platform(s) to build for, can specify more than one.
               e.g. --platform=linux --platform=darwin
-  --app:      Path to your (bundled) application. 
+  --app:      Path to your (bundled) application.
               e.g. --app=/path/to/app/index.js
   --name:     (opt) Application name
               e.g --name=MyAppSoCool
   --dir:      (opt) Working directory, if not specified use cwd
               e.g. --dir=/tmp/js2bin
   --cache     (opt) Cache any pre-built binaries used, to avoid redownload
   --arch:     (opt) Architecture to build for
+  --compress: (opt) Compression type to use, brotli or gzip
+              e.g. --compress=brotli
 
 --ci: build NodeJS with preallocated space for embedding applications
-  --node: NodeJS version to build from source, can specify more than one. 
+  --node: NodeJS version to build from source, can specify more than one.
           e.g. --node=10.16.0
-  --size: Amount of preallocated space, can specify more than one. 
+  --size: Amount of preallocated space, can specify more than one.
           e.g. --size=2MB --size=4MB
   --dir:       (opt) Working directory, if not specified use cwd
   --cache:     (opt) whether to keep build in the cache (to be reused by --build)
@@ -94,7 +96,7 @@ if (args.build) {
   const plats = asArray(args.platform);
   versions.forEach(version => {
     plats.forEach(plat => {
-      const builder = new NodeJsBuilder(args.dir, version, app, args.name);
+      const builder = new NodeJsBuilder(args.dir, version, app, args.name, { compressionAlgo: args.compress });
       p = p.then(() => {
         const arch = args.arch || 'x64';
         log(`building for version=${version}, plat=${plat} app=${app}} arch=${arch}`);
@@ -112,7 +114,7 @@ if (args.build) {
     let lastBuilder;
     sizes.forEach(size => {
       archs.forEach(arch => {
-        const builder = new NodeJsBuilder(args.dir, version, size);
+        const builder = new NodeJsBuilder(args.dir, version, size, undefined, { compressionAlgo: args.compress });
         lastBuilder = builder;
         p = p.then(() => {
           log(`building for version=${version}, size=${size} arch=${arch}`);

src/NodeBuilder.js

@@ -1,6 +1,12 @@
 
 const { log, download, upload, deleteArtifact, getAssetIdByName, fetch, mkdirp, rmrf, copyFileAsync, runCommand, renameAsync, patchFile } = require('./util');
-const { gzipSync, createGunzip } = require('zlib');
+const {
+  brotliCompressSync,
+  gzipSync,
+  createGunzip,
+  gzip,
+  constants: { BROTLI_PARAM_QUALITY, BROTLI_MAX_QUALITY, Z_BEST_COMPRESSION },
+} = require('zlib');
 const { join, dirname, basename, parse, resolve } = require('path');
 const fs = require('fs');
 const os = require('os');
@@ -48,7 +54,7 @@ function buildName(platform, arch, placeHolderSizeMB, version) {
 }
 
 class NodeJsBuilder {
-  constructor(cwd, version, mainAppFile, appName, patchDir) {
+  constructor(cwd, version, mainAppFile, appName, { patchDir, compressionAlgo } = {}) {
     this.version = version;
     this.appFile = resolve(mainAppFile);
     this.appName = appName;
@@ -73,6 +79,7 @@ class NodeJsBuilder {
     this.resultFile = isWindows ? join(this.nodeSrcDir, 'Release', 'node.exe') : join(this.nodeSrcDir, 'out', 'Release', 'node');
     this.placeHolderSizeMB = -1;
     this.builderImageVersion = 3;
+    this.compressionAlgo = compressionAlgo || 'gzip';
   }
 
   static platform() {
@@ -192,8 +199,35 @@ class NodeJsBuilder {
   }
 
   getAppContentToBundle() {
-    const mainAppFileCont = gzipSync(fs.readFileSync(this.appFile), {level: 9}).toString('base64');
-    return Buffer.from(this.appName).toString('base64') + '\n' + mainAppFileCont;
+    const supportedCompression = {
+      brotli: () => this.getAppContentBrotli(),
+      br: () => this.getAppContentBrotli(),
+      gzip: () => this.getAppContentGzip(),
+      deflate: () => { throw new Error('deflate is not a supported compression type'); },
+    };
+    if (!supportedCompression[this.compressionAlgo]) {
+      throw new Error(`unknown compression type: ${this.compressionAlgo}`);
+    }
+    let mainAppBuffer = supportedCompression[this.compressionAlgo]();
+    return Buffer.from(this.appName).toString('base64') + '\n' + mainAppBuffer.toString('base64');
+  }
+
+  getAppContentGzip() {
+    const gzipBuffer = gzipSync(fs.readFileSync(this.appFile), {
+      level: Z_BEST_COMPRESSION,
+    });
+    return gzipBuffer;
+  }
+
+  getAppContentBrotli() {
+    const brotliBuffer = brotliCompressSync(fs.readFileSync(this.appFile), {
+      params: { [BROTLI_PARAM_QUALITY]: BROTLI_MAX_QUALITY },
+    });
+    // brotli does not use any official magic bytes or header signature
+    // prepend the unofficial header ce b2 cf 81
+    // see https://github.com/madler/brotli/blob/master/br-format-v3.txt
+    const brotliHeader = Buffer.from('ceb2cf81', 'hex');
+    return Buffer.concat([brotliHeader, brotliBuffer]);
   }
 
   prepareNodeJsBuild() {
@@ -286,7 +320,7 @@ class NodeJsBuilder {
   // 1. download node source
   // 2. expand node version
   // 3. install _third_party_main.js
-  // 4. process mainAppFile (gzip, base64 encode it) - could be a placeholder file
+  // 4. process mainAppFile (gzip/brotli, base64 encode it) - could be a placeholder file
   // 5. kick off ./configure & build
   buildFromSource(uploadBuild, cache, container, arch, ptrCompression) {
     const makeArgs = isWindows ? ['x64', 'no-cctest'] : [`-j${os.cpus().length}`];

src/_third_party_main.js

@@ -1,6 +1,6 @@
 
 const Module = require('module');
-const { gunzipSync } = require('zlib');
+const { brotliDecompressSync, gunzipSync } = require('zlib');
 const { join, dirname } = require('path');
 
 let source = process.binding('natives')._js2bin_app_main;
@@ -19,7 +19,15 @@ const parts = source.split('\n');
 const appName = Buffer.from(parts[0], 'base64').toString();
 const filename = join(dirname(process.execPath), `${appName.trim()}.js`);
 
-source = parts[1];
+const sourceBuffer = Buffer.from(parts[1], 'base64');
+let sourceString;
+if (sourceBuffer.toString('hex', 0, 4) === 'ceb2cf81') {
+  sourceString = brotliDecompressSync(sourceBuffer.subarray(4)).toString();
+} else if (sourceBuffer.tostring('hex', 0, 2) === '1f8b') {
+  sourceString = gunzipSync(sourceBuffer, {chunkSize: 128*1024*1024}).toString();
+} else {
+  throw new Error('unknown compression format')
+}
 
 // here we turn what looks like an internal module to an non-internal one
 // that way the module is loaded exactly as it would by: node app_main.js
@@ -32,14 +40,14 @@ mod._compile(`
 // initialize clustering
 const cluster = require('cluster');
 if (cluster.worker) {
-   // NOOP - cluster worker already initialized, likely Node 12.x+ 
+   // NOOP - cluster worker already initialized, likely Node 12.x+
 }else if (process.argv[1] && process.env.NODE_UNIQUE_ID) {
    cluster._setupWorker()
    delete process.env.NODE_UNIQUE_ID
 } else {
   process.argv.splice(1, 0, __filename); // don't mess with argv in clustering
 }
 
-${gunzipSync(Buffer.from(source, 'base64'), {chunkSize: 128*1024*1024}).toString()}
+${sourceString}
 
 `, filename);