added pagination (#789)

Co-authored-by: Dustin Hickey <hickeydh@amazon.amazon.com>

Author: hickeydh-aws

reference-artifacts/Custom-Scripts/Developer-Scripts/src/load-outputs.ts

@@ -0,0 +1,58 @@
+import * as fs from 'fs';
+import * as aws from 'aws-sdk';
+import * as dynamodb from 'aws-sdk/clients/dynamodb';
+import { resourceLimits } from 'node:worker_threads';
+const DEV_FILE_PATH = '../../../src/deployments/cdk/';
+const DEV_OUTPUTS_FILE_PATH = `${DEV_FILE_PATH}outputs.json`;
+
+const env = process.env;
+const acceleratorPrefix = env.ACCELERATOR_PREFIX || 'ASEA-';
+const documentClient = new aws.DynamoDB.DocumentClient();
+
+export const scanDDBTable = async (tableName: string) => {
+  const items: dynamodb.ItemList = [];
+  let token: dynamodb.Key | undefined;
+  const props: dynamodb.ScanInput = {
+    TableName: tableName,
+  };
+  do {
+    const response = await documentClient.scan(props).promise();
+    token = response.LastEvaluatedKey;
+    props.ExclusiveStartKey = token!;
+    items.push(...response.Items!);
+  } while (token);
+
+  return items;
+};
+
+export const loadOutputs = (items: dynamodb.ItemList) => {
+  return items.reduce((outputList: any, item) => {
+    const output = JSON.parse(item.outputValue as string);
+    outputList.push(...output);
+    return outputList;
+  }, []);
+};
+
+export const loadConfigs = (items: dynamodb.ItemList) => {
+  return items.map(item => {
+    return { id: item.id, value: JSON.parse(item.value as string) };
+  });
+};
+
+export const writeConfigs = (configList: any) => {
+  for (const config of configList) {
+    if (config.id.includes('accounts/0')) {
+      config.id = 'accounts';
+    }
+    if (config.id !== 'accounts-items-count') {
+      fs.writeFileSync(`${DEV_FILE_PATH}${config.id}.json`, JSON.stringify(config.value, null, 2));
+    }
+  }
+};
+scanDDBTable(`${acceleratorPrefix}Outputs`)
+  .then(outputs => loadOutputs(outputs))
+  .then(parsedOutputs => fs.writeFileSync(DEV_OUTPUTS_FILE_PATH, JSON.stringify(parsedOutputs, null, 2)));
+
+scanDDBTable(`${acceleratorPrefix}Parameters`)
+  .then(params => loadConfigs(params))
+  .then(configs => writeConfigs(configs));

src/deployments/cdk/toolkit.ts

@@ -139,14 +139,24 @@ export class CdkToolkit {
       console.log(`There are no stacks to be deployed`);
       return [];
     }
-
-    let combinedOutputs: StackOutput[];
+    let combinedOutputs: StackOutput[] = [];
     if (parallel) {
-      // Deploy all stacks in parallel
-      const promises = stacks.map(stack => this.deployStack(stack));
-      // Wait for all promises to be fulfilled
-      const outputsList = await fulfillAll(promises);
-      combinedOutputs = outputsList.reduce((result, output) => [...result, ...output]);
+      const pageSize = 900;
+      let stackPromises: any[] = [];
+      for (let i = 0; i < stacks.length; i++) {
+        const stack = stacks[i];
+        console.log(`deploying stack ${i + 1} of ${stacks.length}`);
+        stackPromises.push(this.deployStack(stack));
+        if (stackPromises.length > pageSize - 1 || i === stacks.length - 1) {
+          const results = await Promise.all(stackPromises);
+          console.log(`Deployed stacks ${i + 1} of ${stacks.length}`);
+          stackPromises = [];
+          if (results) {
+            combinedOutputs.push(...results);
+          }
+        }
+      }
+      combinedOutputs = combinedOutputs.flat(2);
     } else {
       // Deploy all stacks sequentially
       combinedOutputs = [];
@@ -157,6 +167,7 @@ export class CdkToolkit {
     }
 
     // Merge all stack outputs
+    console.log(JSON.stringify(combinedOutputs, null, 4));
     return combinedOutputs;
   }
 

src/lib/custom-resources/cdk-security-hub-accept-invites/runtime/src/index.ts

@@ -36,12 +36,20 @@ async function onCreate(event: CloudFormationCustomResourceEvent) {
   }
 
   // Check for pending invitations from Master
-  const invitations = await throttlingBackOff(() => hub.listInvitations().promise());
-  if (!invitations.Invitations) {
+  let token: string | undefined;
+  let invitations: AWS.SecurityHub.InvitationList = [];
+  do {
+    const response = await throttlingBackOff(() => hub.listInvitations({ NextToken: token }).promise());
+    if (response.Invitations) {
+      invitations.push(...response.Invitations);
+    }
+    token = response.NextToken;
+  } while (token);
+  if (!invitations) {
     console.log(`No Security Hub invitations found`);
   } else {
     // Accepting Invitation from Master account
-    const ownerInvitation = invitations.Invitations.find(x => x.AccountId === masterAccountId);
+    const ownerInvitation = invitations.find(x => x.AccountId === masterAccountId);
     if (ownerInvitation) {
       const invitationId = ownerInvitation?.InvitationId!;
       await throttlingBackOff(() =>

src/lib/custom-resources/cdk-security-hub-disable-controls/runtime/src/index.ts

@@ -30,26 +30,18 @@ async function onEvent(event: CloudFormationCustomResourceEvent) {
 
 async function onCreate(event: CloudFormationCustomResourceEvent) {
   const standards = event.ResourceProperties.standards;
-  const standardsResponse = await throttlingBackOff(() => hub.describeStandards().promise());
-  const enabledStandardsResponse = await throttlingBackOff(() => hub.getEnabledStandards().promise());
+  const standardsResponse = await describeStandards();
+  const enabledStandardsResponse = await getEnabledStandards();
 
   // Getting standards and disabling specific Controls for each standard
   for (const standard of standards) {
-    const standardArn = standardsResponse.Standards?.find(x => x.Name === standard.name)?.StandardsArn;
-    const standardSubscriptionArn = enabledStandardsResponse.StandardsSubscriptions?.find(
-      s => s.StandardsArn === standardArn,
-    )?.StandardsSubscriptionArn;
+    const standardArn = standardsResponse?.find(x => x.Name === standard.name)?.StandardsArn;
+    const standardSubscriptionArn = enabledStandardsResponse?.find(s => s.StandardsArn === standardArn)
+      ?.StandardsSubscriptionArn;
 
-    const standardControls = await throttlingBackOff(() =>
-      hub
-        .describeStandardsControls({
-          StandardsSubscriptionArn: standardSubscriptionArn!,
-          MaxResults: 100,
-        })
-        .promise(),
-    );
+    const standardControls = await describeStandardsControls(standardSubscriptionArn);
     for (const disableControl of standard['controls-to-disable']) {
-      const standardControl = standardControls.Controls?.find(x => x.ControlId === disableControl);
+      const standardControl = standardControls?.find(x => x.ControlId === disableControl);
       if (!standardControl) {
         console.log(`Control "${disableControl}" not found for Standard "${standard.name}"`);
         continue;
@@ -87,16 +79,9 @@ async function onUpdate(event: CloudFormationCustomResourceUpdateEvent) {
       s => s.StandardsArn === standardArn,
     )?.StandardsSubscriptionArn;
 
-    const standardControls = await throttlingBackOff(() =>
-      hub
-        .describeStandardsControls({
-          StandardsSubscriptionArn: standardSubscriptionArn!,
-          MaxResults: 100,
-        })
-        .promise(),
-    );
+    const standardControls = await describeStandardsControls(standardSubscriptionArn);
     for (const disableControl of standard['controls-to-disable'] || []) {
-      const standardControl = standardControls.Controls?.find(x => x.ControlId === disableControl);
+      const standardControl = standardControls?.find(x => x.ControlId === disableControl);
       if (!standardControl) {
         console.log(`Control "${disableControl}" not found for Standard "${standard.name}"`);
         continue;
@@ -119,7 +104,7 @@ async function onUpdate(event: CloudFormationCustomResourceUpdateEvent) {
         c => !standard['controls-to-disable']?.includes(c),
       );
       for (const enableControl of enableControls || []) {
-        const standardControl = standardControls.Controls?.find(x => x.ControlId === enableControl);
+        const standardControl = standardControls?.find(x => x.ControlId === enableControl);
         if (!standardControl) {
           console.log(`Control "${enableControl}" not found for Standard "${standard.name}"`);
           continue;
@@ -140,6 +125,52 @@ async function onUpdate(event: CloudFormationCustomResourceUpdateEvent) {
   };
 }
 
+async function describeStandards() {
+  const standards = [];
+  let token: string | undefined;
+  do {
+    const response = await throttlingBackOff(() => hub.describeStandards().promise());
+    if (response.Standards) {
+      standards.push(...response.Standards);
+    }
+    token = response.NextToken;
+  } while (token);
+
+  return standards;
+}
+
+async function getEnabledStandards() {
+  const enabledStandards = [];
+  let token: string | undefined;
+  do {
+    const response = await throttlingBackOff(() => hub.getEnabledStandards().promise());
+    if (response.StandardsSubscriptions) {
+      enabledStandards.push(...response.StandardsSubscriptions);
+    }
+    token = response.NextToken;
+  } while (token);
+
+  return enabledStandards;
+}
+
+async function describeStandardsControls(subscriptionArn: string | undefined) {
+  let token: string | undefined;
+  const standardControls: any[] = [];
+  if (!subscriptionArn) {
+    return standardControls;
+  }
+  do {
+    const response = await throttlingBackOff(() =>
+      hub.describeStandardsControls({ StandardsSubscriptionArn: subscriptionArn, NextToken: token }).promise(),
+    );
+    if (response.Controls) {
+      standardControls.push(...response.Controls);
+    }
+    token = response.NextToken;
+  } while (token);
+  return standardControls;
+}
+
 async function onDelete(_: CloudFormationCustomResourceEvent) {
   console.log(`Nothing to do for delete...`);
 }