fixed heap and mem issues (#813)

Co-authored-by: hickeydh <hickeydh@amazon.com>

Author: hickeydh-aws

src/core/cdk/src/initial-setup.ts

@@ -151,6 +151,7 @@ export namespace InitialSetup {
           ORGANIZATIONS_ITEM_ID: 'organizations',
           DYNAMODB_PARAMETERS_TABLE_NAME: parametersTable.tableName,
           DEPLOY_STACK_PAGE_SIZE: props.pageSize,
+          COMPUTE_TYPE: props.codebuildComputeType,
         },
       });
 

src/deployments/cdk/cdk.ts

@@ -3,6 +3,9 @@ import { CdkToolkit } from './toolkit';
 import * as app from './src/app';
 import microstats from 'microstats';
 import { debugModeEnabled } from '@aws-cdk/core/lib/debug';
+import * as v8 from 'v8';
+
+const PAGE_SIZE = parseInt(process.env.DEPLOY_STACK_PAGE_SIZE) || 850;
 
 process.on('unhandledRejection', (reason, _) => {
   console.error(reason);
@@ -17,16 +20,45 @@ microstats.on('disk', function (value) {
 microstats.on('cpu', function (value) {
   console.log('CPU:', value);
 });
-let microstatsOptions = { frequency: '5s' };
+let microstatsOptions = {
+  frequency: 'onalert',
+  memoryalert: { used: '>80%' },
+  cpualert: { load: '>90%' },
+  diskalert: { used: '>70%' },
+};
 /**
  * Entrypoint for bootstrapping, deploying and synthesizing CDK apps.
  */
 
+const convertToMegabytes = (num: number) => {
+  return `${(num / 1024 / 1024).toFixed(2)} MB`;
+};
+
+const getHeapStatistics = () => {
+  const heapstats = v8.getHeapStatistics();
+
+  return {
+    totalHeapSize: convertToMegabytes(heapstats.total_heap_size),
+    totalHeapSizeExecutable: convertToMegabytes(heapstats.total_heap_size_executable),
+    totalPhysicalSize: convertToMegabytes(heapstats.total_physical_size),
+    totalAvailableSize: convertToMegabytes(heapstats.total_available_size),
+    usedHeapSize: convertToMegabytes(heapstats.used_heap_size),
+    heapSizeLimit: convertToMegabytes(heapstats.heap_size_limit),
+    usedHeapSizePercentage: ((heapstats.used_heap_size / heapstats.heap_size_limit) * 100).toFixed(2),
+    mallocedMemory: convertToMegabytes(heapstats.malloced_memory),
+    peakMallocedMemory: convertToMegabytes(heapstats.peak_malloced_memory),
+    doesZapGarbage: heapstats.does_zap_garbage,
+    nativeContexts: heapstats.number_of_native_contexts,
+    detachedContexts: heapstats.number_of_detached_contexts,
+  };
+};
+
 async function main() {
   if (debugModeEnabled()) {
     microstats.start(microstatsOptions, err => {
       console.log(err);
     });
+    console.log(getHeapStatistics());
   }
   const usage = `Usage: cdk.ts <command> [<command>] --phase PHASE [--region REGION] [--account-key ACCOUNT_KEY] [--parallel]`;
   const args = mri(process.argv.slice(2), {
@@ -54,26 +86,29 @@ async function main() {
     phaseId: `${phase}`,
     region: args.region,
     accountKey: args['account-key'],
-    // Make sure templates and assets do not build in to the same directory
     useTempOutputDir: true,
   });
   console.log(`Total stacks = ${apps.length}`);
-  const appDeploymentPaging = [];
-
-  const toolkit = await CdkToolkit.create(apps);
-
-  if (commands.includes('bootstrap')) {
-    await toolkit.bootstrap();
-  }
-  if (commands.includes('synth')) {
-    await toolkit.synth();
+  let appsPage = [];
+  for (let i = 0; i < apps.length; i++) {
+    appsPage.push(apps[i]);
+    console.log(`deploying stack ${i + 1} of ${apps.length}`);
+    if (appsPage.length > PAGE_SIZE - 1 || i === apps.length - 1) {
+      const toolkit = await CdkToolkit.create(appsPage);
+      if (commands.includes('bootstrap')) {
+        await toolkit.bootstrap();
+      }
+      if (commands.includes('synth')) {
+        await toolkit.synth();
+      }
+      if (commands.includes('deploy')) {
+        await toolkit.deployAllStacks({
+          parallel,
+        });
+      }
+      appsPage = [];
+    }
   }
-  if (commands.includes('deploy')) {
-    await toolkit.deployAllStacks({
-      parallel,
-    });
-  }
-
   if (debugModeEnabled()) {
     microstats.stop();
   }

src/deployments/cdk/codebuild-deploy.sh

@@ -1,5 +1,15 @@
 #!/bin/sh
 
+if [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_LARGE" ]; then
+  echo "The codebuild build type has changed from default to ${COMPUTE_TYPE}, increasing the max_old_space_size to 8192"
+  export NODE_OPTIONS=--max_old_space_size=8192
+elif [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_2XLARGE" ]; then
+  echo "The codebuild build type has changed from default to ${COMPUTE_TYPE}, increasing the max_old_space_size to 16384"
+  export NODE_OPTIONS=--max_old_space_size=16384
+else
+  echo "The codebuild build type is ${COMPUTE_TYPE}"
+fi
+
 if [ -z "${ACCELERATOR_PHASE}" ]; then
   echo "The environment variable ACCELERATOR_PHASE has to be set to the path of the app you want to deploy."
   exit 1

src/deployments/cdk/toolkit.ts

@@ -147,26 +147,14 @@ export class CdkToolkit {
       console.log(`There are no stacks to be deployed`);
       return [];
     }
-    let combinedOutputs: StackOutput[] = [];
+
+    let combinedOutputs: StackOutput[];
     if (parallel) {
-      const pageSize = this.deploymentPageSize;
-      console.log(`Deploying ${pageSize} stacks at a time out of ${stacks.length} stacks.`);
-      let stackPromises: any[] = [];
-      for (let i = 0; i < stacks.length; i++) {
-        const stack = stacks[i];
-        console.log(`deploying stack ${i + 1} of ${stacks.length}`);
-
-        stackPromises.push(this.deployStack(stack));
-        if (stackPromises.length > pageSize - 1 || i === stacks.length - 1) {
-          const results = await Promise.all(stackPromises);
-          console.log(`Deployed stacks ${i + 1} of ${stacks.length}`);
-          stackPromises = [];
-          if (results) {
-            combinedOutputs.push(...results);
-          }
-        }
-      }
-      combinedOutputs = combinedOutputs.flat(2);
+      // Deploy all stacks in parallel
+      const promises = stacks.map(stack => this.deployStack(stack));
+      // Wait for all promises to be fulfilled
+      const outputsList = await fulfillAll(promises);
+      combinedOutputs = outputsList.reduce((result, output) => [...result, ...output]);
     } else {
       // Deploy all stacks sequentially
       combinedOutputs = [];
@@ -175,8 +163,8 @@ export class CdkToolkit {
         combinedOutputs.push(...output);
       }
     }
+
     // Merge all stack outputs
-    console.log(JSON.stringify(combinedOutputs, null, 4));
     return combinedOutputs;
   }
 
@@ -186,13 +174,20 @@ export class CdkToolkit {
     );
 
     const stackExists = await this.cloudFormation.stackExists({ stack });
-    console.log(`Stack ${stack.displayName} exists`, stackExists);
+    console.log(
+      `Stack ${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region} exists`,
+      stackExists,
+    );
 
     const resources = Object.keys(stack.template.Resources || {});
     if (resources.length === 0) {
-      console.warn(`${stack.displayName}: stack has no resources`);
+      console.warn(
+        `${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region}: stack has no resources`,
+      );
       if (stackExists) {
-        console.warn(`${stack.displayName}: deleting existing stack`);
+        console.warn(
+          `${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region}: deleting existing stack`,
+        );
         this.destroyStack(stack);
       }
       return [];

src/installer/cdk/src/index.ts

@@ -83,12 +83,7 @@ async function main() {
   const codebuildComputeType = new cdk.CfnParameter(stack, 'CodeBuild Compute Type', {
     description: 'The compute type of the build server for the Accelerator deployments.',
     default: codebuild.ComputeType.MEDIUM,
-    allowedValues: [
-      codebuild.ComputeType.SMALL,
-      codebuild.ComputeType.MEDIUM,
-      codebuild.ComputeType.LARGE,
-      codebuild.ComputeType.X2_LARGE,
-    ],
+    allowedValues: [codebuild.ComputeType.MEDIUM, codebuild.ComputeType.LARGE, codebuild.ComputeType.X2_LARGE],
   });
 
   const stackDeployPageSize = new cdk.CfnParameter(stack, 'Deployment Page Size', {