(Fix): SM jitter (#1050)

* exponential backoff fix

* exponential backoff fix

* Fixed backoff for lambdas

* added backoff in other location

* fixed lambda timeouts and added timeout aspect

* fixed typo

* tweak default max jitter delay to 2s from 5s

Co-authored-by: hickeydh-aws <hickeydh@amazon.com>
Co-authored-by: Brian969 <56414362+Brian969@users.noreply.github.com>

Author: hickeydh-aws

reference-artifacts/Add-ons/opensiem/lambdas/common/src/backoff.ts

@@ -20,8 +20,24 @@ export function throttlingBackOff<T>(
   request: () => Promise<T>,
   options?: Partial<Omit<IBackOffOptions, 'retry'>>,
 ): Promise<T> {
+  const defaultDelay = 500;
+  let maxDelayValue = 2000;
+
+  if (process.env.BACKOFF_START_DELAY) {
+    const backoffStartDelay = parseInt(process.env.BACKOFF_START_DELAY, 10);
+    if (Number.isInteger(backoffStartDelay)) {
+      maxDelayValue = backoffStartDelay;
+    }
+  }
+
+  // Add jitter to the starting delay
+  const startingDelay = Math.random() * (maxDelayValue - defaultDelay + 1) + defaultDelay;
+
+  console.log(`throttlingBackOff delay set to ${startingDelay}`);
+
   return backOff(request, {
-    startingDelay: 500,
+    startingDelay,
+    delayFirstAttempt: true,
     jitter: 'full',
     retry: isThrottlingError,
     ...options,
@@ -34,6 +50,7 @@ export const isThrottlingError = (e: any) =>
   e.code === 'ConcurrentModificationException' || // Retry for AWS Organizations
   e.code === 'InsufficientDeliveryPolicyException' || // Retry for ConfigService
   e.code === 'NoAvailableDeliveryChannelException' || // Retry for ConfigService
+  e.code === 'ConcurrentModifications' || // Retry for AssociateHostedZone
   e.code === 'TooManyRequestsException' ||
   e.code === 'Throttling' ||
   e.code === 'ThrottlingException' ||

src/core/cdk/src/index.ts

@@ -39,6 +39,7 @@ async function main() {
     env.CONFIG_S3_BUCKET || `${acceleratorPrefix.toLowerCase()}${cdk.Aws.ACCOUNT_ID}-${cdk.Aws.REGION}-config`;
   const codebuildComputeType = env.BUILD_COMPUTE_TYPE || 'BUILD_GENERAL1_LARGE';
   const pageSize = env.DEPLOY_STACK_PAGE_SIZE || '900';
+  const backoff = env.BACKOFF_START_DELAY;
   const enablePrebuiltProject = 'ENABLE_PREBUILT_PROJECT' in env;
   const notificationEmail = env.NOTIFICATION_EMAIL || 'notify@example.com';
   const installerCmk = env.INSTALLER_CMK || `alias/${acceleratorPrefix}Installer-Key`;
@@ -70,6 +71,7 @@ async function main() {
     installerCmk,
     codebuildComputeType,
     pageSize,
+    backoff,
   });
 }
 

src/core/cdk/src/initial-setup.ts

@@ -57,6 +57,7 @@ export namespace InitialSetup {
     // eslint-disable-next-line  @typescript-eslint/no-explicit-any
     codebuildComputeType: any;
     pageSize: string;
+    backoff: string | undefined;
     /**
      * Current Accelerator version
      */
@@ -264,6 +265,7 @@ export namespace InitialSetup {
           SUBNET_CIDR_ASSIGNED_POOL: subnetCidrPoolTable.tableName,
           CIDR_POOL: cidrPoolTable.tableName,
           DEPLOY_STACK_PAGE_SIZE: props.pageSize,
+          BACKOFF_START_DELAY: props.backoff || '',
           COMPUTE_TYPE: props.codebuildComputeType,
         },
       });

src/deployments/cdk/src/common/alb-ip-forwarding.ts

@@ -59,7 +59,7 @@ export class AlbIpForwarding extends Construct {
       runtime: lambda.Runtime.NODEJS_14_X,
       code: lambdaCode,
       handler: 'index.albIpMonitor',
-      timeout: Duration.seconds(30),
+      timeout: Duration.seconds(60),
       environment: {
         LOOKUP_TABLE: ddbTable.tableName,
       },
@@ -82,7 +82,7 @@ export class AlbIpForwarding extends Construct {
       runtime: lambda.Runtime.NODEJS_14_X,
       handler: 'index.albTargetRecordMonitor',
       code: lambdaCode,
-      timeout: Duration.seconds(30),
+      timeout: Duration.seconds(60),
       environment: {
         LOOKUP_TABLE: ddbTable.tableName,
       },

src/installer/cdk/src/index.ts

@@ -115,6 +115,12 @@ class Installer extends cdk.Stack {
       default: 680,
     });
 
+    const backoffStartDelay = new cdk.CfnParameter(this, 'Backoff Start Delay', {
+      description:
+        'The start delay for exponential backoff of API calls in milliseconds. Leave at the default of 2000 unless needed.',
+      default: 2000,
+    });
+
     const stateMachineName = `${acceleratorPrefix}MainStateMachine_sm`;
 
     // The state machine name has to match the name of the state machine in initial setup
@@ -322,6 +328,10 @@ class Installer extends cdk.Stack {
             type: codebuild.BuildEnvironmentVariableType.PLAINTEXT,
             value: stackDeployPageSize.valueAsString,
           },
+          BACKOFF_START_DELAY: {
+            type: codebuild.BuildEnvironmentVariableType.PLAINTEXT,
+            value: backoffStartDelay.valueAsString,
+          },
         },
       },
       cache: codebuild.Cache.local(codebuild.LocalCacheMode.SOURCE),

src/lib/cdk-accelerator/src/core/accelerator-stack.ts

@@ -12,7 +12,7 @@
  */
 
 import * as cdk from '@aws-cdk/core';
-import { AcceleratorNameTagger, AcceleratorProtectedTagger } from '.';
+import { AcceleratorNameTagger, AcceleratorProtectedTagger, LambdaEnvironmentVariables, LambdaDefaultTimeout } from '.';
 
 export interface AcceleratorStackProps extends cdk.StackProps {
   acceleratorName: string;
@@ -34,6 +34,8 @@ export class AcceleratorStack extends cdk.Stack {
     cdk.Aspects.of(this).add(new cdk.Tag('Accelerator', this.acceleratorName));
     cdk.Aspects.of(this).add(new AcceleratorNameTagger());
     cdk.Aspects.of(this).add(new AcceleratorProtectedTagger(this.acceleratorName));
+    cdk.Aspects.of(this).add(new LambdaEnvironmentVariables());
+    cdk.Aspects.of(this).add(new LambdaDefaultTimeout());
   }
 
   static of(construct: cdk.IConstruct): AcceleratorStack {

src/lib/cdk-accelerator/src/core/index.ts

@@ -14,3 +14,5 @@
 export * from './cfn-include';
 export * from './name-tagger';
 export * from './accelerator-protected-tagger';
+export * from './lambda-env-variables';
+export * from './lambda-default-timeout';

src/lib/cdk-accelerator/src/core/lambda-default-timeout.ts

@@ -0,0 +1,12 @@
+import * as cdk from '@aws-cdk/core';
+import * as lambda from '@aws-cdk/aws-lambda';
+
+export class LambdaDefaultTimeout implements cdk.IAspect {
+  visit(node: cdk.IConstruct): void {
+    if (node instanceof lambda.CfnFunction) {
+      if (!node.timeout) {
+        node.timeout = 60;
+      }
+    }
+  }
+}

src/lib/cdk-accelerator/src/core/lambda-env-variables.ts

@@ -0,0 +1,10 @@
+import * as cdk from '@aws-cdk/core';
+import * as lambda from '@aws-cdk/aws-lambda';
+
+export class LambdaEnvironmentVariables implements cdk.IAspect {
+  visit(node: cdk.IConstruct): void {
+    if (node instanceof lambda.Function) {
+      node.addEnvironment('BACKOFF_START_DELAY', process.env['BACKOFF_START_DELAY'] || '2000');
+    }
+  }
+}

src/lib/common/src/aws/backoff.ts

@@ -20,8 +20,24 @@ export function throttlingBackOff<T>(
   request: () => Promise<T>,
   options?: Partial<Omit<IBackOffOptions, 'retry'>>,
 ): Promise<T> {
+  const defaultDelay = 500;
+  let maxDelayValue = 2000;
+
+  if (process.env.BACKOFF_START_DELAY) {
+    const backoffStartDelay = parseInt(process.env.BACKOFF_START_DELAY, 10);
+    if (Number.isInteger(backoffStartDelay)) {
+      maxDelayValue = backoffStartDelay;
+    }
+  }
+
+  // Add jitter to the starting delay
+  const startingDelay = Math.random() * (maxDelayValue - defaultDelay + 1) + defaultDelay;
+
+  console.log(`throttlingBackOff delay set to ${startingDelay}`);
+
   return backOff(request, {
-    startingDelay: 500,
+    startingDelay,
+    delayFirstAttempt: true,
     jitter: 'full',
     retry: isThrottlingError,
     ...options,
@@ -34,6 +50,7 @@ export const isThrottlingError = (e: any) =>
   e.code === 'ConcurrentModificationException' || // Retry for AWS Organizations
   e.code === 'InsufficientDeliveryPolicyException' || // Retry for ConfigService
   e.code === 'NoAvailableDeliveryChannelException' || // Retry for ConfigService
+  e.code === 'ConcurrentModifications' || // Retry for AssociateHostedZone
   e.code === 'TooManyRequestsException' ||
   e.code === 'Throttling' ||
   e.code === 'ThrottlingException' ||