Skip to content

Commit d3a7850

Browse files
naveenkoppulanaveenkoppula
authored
fix(core): Moving outputs from S3 to DynamoDB (#371)
* Initial Commit * Moving outputs from s3 to dynamodb * Prettier * Sending payload to store outputs * sending params to map sm * Incresing maxConcurrency to 10 for accounts and regions * Interm Push * Moving Store outputs to seperate SM * Adding encryption to Outputs Table
1 parent 02b88dc commit d3a7850

File tree

15 files changed

+302
-324
lines changed

15 files changed

+302
-324
lines changed

‎src/core/cdk/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
"@aws-cdk/aws-stepfunctions": "1.46.0",
4141
"@aws-cdk/aws-stepfunctions-tasks": "1.46.0",
4242
"@aws-cdk/aws-secretsmanager": "1.46.0",
43+
"@aws-cdk/aws-dynamodb": "1.46.0",
4344
"@aws-cdk/core": "1.46.0",
4445
"@aws-accelerator/accelerator-runtime": "workspace:^0.0.1",
4546
"@aws-accelerator/cdk-accelerator": "workspace:^0.0.1",

‎src/core/cdk/src/initial-setup.ts‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as iam from '@aws-cdk/aws-iam';
44
import * as lambda from '@aws-cdk/aws-lambda';
55
import * as s3assets from '@aws-cdk/aws-s3-assets';
66
import * as secrets from '@aws-cdk/aws-secretsmanager';
7+
import * as dynamodb from '@aws-cdk/aws-dynamodb';
78
import * as sfn from '@aws-cdk/aws-stepfunctions';
89
import * as tasks from '@aws-cdk/aws-stepfunctions-tasks';
910
import { CdkDeployProject, PrebuiltCdkDeployProject } from '@aws-accelerator/cdk-accelerator/src/codebuild';
@@ -19,6 +20,7 @@ import { CreateStackTask } from './tasks/create-stack-task';
1920
import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task';
2021
import * as fs from 'fs';
2122
import * as sns from '@aws-cdk/aws-sns';
23+
import { StoreOutputsTask } from './tasks/store-outputs-task';
2224

2325
export namespace InitialSetup {
2426
export interface CommonProps {
@@ -84,6 +86,18 @@ export namespace InitialSetup {
8486
});
8587
setSecretValue(organizationsSecret, '[]');
8688

89+
const outputsTable = new dynamodb.Table(this, 'Outputs', {
90+
tableName: createName({
91+
name: 'Outputs',
92+
suffixLength: 0,
93+
}),
94+
partitionKey: {
95+
name: 'id',
96+
type: dynamodb.AttributeType.STRING,
97+
},
98+
encryption: dynamodb.TableEncryption.DEFAULT,
99+
});
100+
87101
// This is the maximum time before a build times out
88102
// The role used by the build should allow this session duration
89103
const buildTimeout = cdk.Duration.hours(4);
@@ -469,9 +483,7 @@ export namespace InitialSetup {
469483
'configCommitId.$': '$.configCommitId',
470484
'organizationalUnits.$': '$.organizationalUnits',
471485
'accounts.$': '$.accounts',
472-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
473-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
474-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
486+
outputTableName: outputsTable.tableName,
475487
},
476488
resultPath: 'DISCARD',
477489
});
@@ -527,12 +539,8 @@ export namespace InitialSetup {
527539
ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName,
528540
ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName,
529541
CONFIG_BRANCH_NAME: props.configBranchName,
542+
STACK_OUTPUT_TABLE_NAME: outputsTable.tableName,
530543
};
531-
if (loadOutputs) {
532-
environment['STACK_OUTPUT_BUCKET_NAME.$'] = '$.storeOutput.outputBucketName';
533-
environment['STACK_OUTPUT_BUCKET_KEY.$'] = '$.storeOutput.outputBucketKey';
534-
environment['STACK_OUTPUT_VERSION.$'] = '$.storeOutput.outputVersion';
535-
}
536544
const deployTask = new sfn.Task(this, `Deploy Phase ${phase}`, {
537545
// tslint:disable-next-line: deprecation
538546
task: new tasks.StartExecution(codeBuildStateMachine, {
@@ -547,21 +555,32 @@ export namespace InitialSetup {
547555
return deployTask;
548556
};
549557

550-
const createStoreOutputTask = (phase: number) =>
551-
new CodeTask(this, `Store Phase ${phase} Output`, {
552-
functionProps: {
553-
code: lambdaCode,
554-
handler: 'index.storeStackOutputStep',
555-
role: pipelineRole,
556-
},
557-
functionPayload: {
558-
acceleratorPrefix: props.acceleratorPrefix,
559-
assumeRoleName: props.stateMachineExecutionRole,
560-
'accounts.$': '$.accounts',
561-
'regions.$': '$.regions',
562-
},
563-
resultPath: '$.storeOutput',
558+
const storeOutputsStateMachine = new sfn.StateMachine(this, `${props.acceleratorPrefix}StoreOutputs_sm`, {
559+
stateMachineName: `${props.acceleratorPrefix}StoreOutputs_sm`,
560+
definition: new StoreOutputsTask(this, 'StoreOutputs', {
561+
lambdaCode,
562+
role: pipelineRole,
563+
}),
564+
});
565+
566+
const createStoreOutputTask = (phase: number) => {
567+
const storeOutputsTask = new sfn.Task(this, `Store Phase ${phase} Outputs`, {
568+
// tslint:disable-next-line: deprecation
569+
task: new tasks.StartExecution(storeOutputsStateMachine, {
570+
integrationPattern: sfn.ServiceIntegrationPattern.SYNC,
571+
input: {
572+
'accounts.$': '$.accounts',
573+
'regions.$': '$.regions',
574+
acceleratorPrefix: props.acceleratorPrefix,
575+
assumeRoleName: props.stateMachineExecutionRole,
576+
outputsTable: outputsTable.tableName,
577+
phaseNumber: phase,
578+
},
579+
}),
580+
resultPath: 'DISCARD',
564581
});
582+
return storeOutputsTask;
583+
};
565584

566585
// TODO Create separate state machine for deployment
567586
const deployPhaseRolesTask = createDeploymentTask(-1, false);
@@ -587,9 +606,7 @@ export namespace InitialSetup {
587606
lambdaPath: 'index.createConfigRecorder',
588607
name: 'Create Config Recorder',
589608
functionPayload: {
590-
'stackOutputBucketName.$': '$.stackOutputBucketName',
591-
'stackOutputBucketKey.$': '$.stackOutputBucketKey',
592-
'stackOutputVersion.$': '$.stackOutputVersion',
609+
outputTableName: outputsTable.tableName,
593610
},
594611
}),
595612
});
@@ -604,9 +621,7 @@ export namespace InitialSetup {
604621
'configFilePath.$': '$.configFilePath',
605622
'configCommitId.$': '$.configCommitId',
606623
'baseline.$': '$.baseline',
607-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
608-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
609-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
624+
outputTableName: outputsTable.tableName,
610625
acceleratorPrefix: props.acceleratorPrefix,
611626
},
612627
}),
@@ -626,9 +641,7 @@ export namespace InitialSetup {
626641
'configRepositoryName.$': '$.configRepositoryName',
627642
'configFilePath.$': '$.configFilePath',
628643
'configCommitId.$': '$.configCommitId',
629-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
630-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
631-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
644+
outputTableName: outputsTable.tableName,
632645
},
633646
resultPath: 'DISCARD',
634647
});
@@ -648,9 +661,7 @@ export namespace InitialSetup {
648661
'configRepositoryName.$': '$.configRepositoryName',
649662
'configFilePath.$': '$.configFilePath',
650663
'configCommitId.$': '$.configCommitId',
651-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
652-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
653-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
664+
outputTableName: outputsTable.tableName,
654665
rdgwScripts,
655666
},
656667
resultPath: 'DISCARD',
@@ -668,9 +679,7 @@ export namespace InitialSetup {
668679
'configRepositoryName.$': '$.configRepositoryName',
669680
'configFilePath.$': '$.configFilePath',
670681
'configCommitId.$': '$.configCommitId',
671-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
672-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
673-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
682+
outputTableName: outputsTable.tableName,
674683
},
675684
resultPath: 'DISCARD',
676685
});
@@ -683,9 +692,7 @@ export namespace InitialSetup {
683692
},
684693
functionPayload: {
685694
assumeRoleName: props.stateMachineExecutionRole,
686-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
687-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
688-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
695+
outputTableName: outputsTable.tableName,
689696
},
690697
resultPath: 'DISCARD',
691698
});
@@ -702,9 +709,7 @@ export namespace InitialSetup {
702709
'configRepositoryName.$': '$.configRepositoryName',
703710
'configFilePath.$': '$.configFilePath',
704711
'configCommitId.$': '$.configCommitId',
705-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
706-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
707-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
712+
outputTableName: outputsTable.tableName,
708713
},
709714
resultPath: 'DISCARD',
710715
});
@@ -728,9 +733,7 @@ export namespace InitialSetup {
728733
'configRepositoryName.$': '$.configRepositoryName',
729734
'configFilePath.$': '$.configFilePath',
730735
'configCommitId.$': '$.configCommitId',
731-
'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
732-
'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
733-
'stackOutputVersion.$': '$.storeOutput.outputVersion',
736+
outputTableName: outputsTable.tableName,
734737
},
735738
}),
736739
resultPath: 'DISCARD',

‎src/core/cdk/src/tasks/store-outputs-task.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import * as cdk from '@aws-cdk/core';
2+
import * as iam from '@aws-cdk/aws-iam';
3+
import * as lambda from '@aws-cdk/aws-lambda';
4+
import * as sfn from '@aws-cdk/aws-stepfunctions';
5+
import { CodeTask } from '@aws-accelerator/cdk-accelerator/src/stepfunction-tasks';
6+
7+
export namespace StoreOutputsTask {
8+
export interface Props {
9+
role: iam.IRole;
10+
lambdaCode: lambda.Code;
11+
functionPayload?: { [key: string]: unknown };
12+
waitSeconds?: number;
13+
}
14+
}
15+
16+
export class StoreOutputsTask extends sfn.StateMachineFragment {
17+
readonly startState: sfn.State;
18+
readonly endStates: sfn.INextable[];
19+
20+
constructor(scope: cdk.Construct, id: string, props: StoreOutputsTask.Props) {
21+
super(scope, id);
22+
23+
const { role, lambdaCode, functionPayload, waitSeconds = 10 } = props;
24+
25+
role.addToPrincipalPolicy(
26+
new iam.PolicyStatement({
27+
effect: iam.Effect.ALLOW,
28+
resources: ['*'],
29+
actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'],
30+
}),
31+
);
32+
33+
const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs`, {
34+
itemsPath: `$.accounts`,
35+
resultPath: 'DISCARD',
36+
maxConcurrency: 10,
37+
parameters: {
38+
'account.$': '$$.Map.Item.Value',
39+
'regions.$': '$.regions',
40+
'acceleratorPrefix.$': '$.acceleratorPrefix',
41+
'assumeRoleName.$': '$.assumeRoleName',
42+
'outputsTable.$': '$.outputsTable',
43+
'phaseNumber.$': '$.phaseNumber',
44+
},
45+
});
46+
47+
const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, {
48+
itemsPath: `$.regions`,
49+
resultPath: 'DISCARD',
50+
maxConcurrency: 10,
51+
parameters: {
52+
'account.$': '$.account',
53+
'region.$': '$$.Map.Item.Value',
54+
'acceleratorPrefix.$': '$.acceleratorPrefix',
55+
'assumeRoleName.$': '$.assumeRoleName',
56+
'outputsTable.$': '$.outputsTable',
57+
'phaseNumber.$': '$.phaseNumber',
58+
},
59+
});
60+
61+
const startTaskResultPath = '$.storeOutputsOutput';
62+
const storeOutputsTask = new CodeTask(scope, `Store Outputs`, {
63+
resultPath: startTaskResultPath,
64+
functionPayload,
65+
functionProps: {
66+
role,
67+
code: lambdaCode,
68+
handler: 'index.storeStackOutputStep',
69+
},
70+
});
71+
72+
const pass = new sfn.Pass(this, 'Store Outputs Success');
73+
storeAccountOutputs.iterator(storeAccountRegionOutputs);
74+
storeAccountRegionOutputs.iterator(storeOutputsTask);
75+
const chain = sfn.Chain.start(storeAccountOutputs).next(pass);
76+
77+
this.startState = chain.startState;
78+
this.endStates = chain.endStates;
79+
}
80+
}

‎src/core/runtime/src/account-default-settings-step.ts‎

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as aws from 'aws-sdk';
22
import { Account } from '@aws-accelerator/common-outputs/src/accounts';
33
import { STS } from '@aws-accelerator/common/src/aws/sts';
4+
import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
45
import {
56
StackOutput,
67
getStackOutput,
@@ -11,38 +12,21 @@ import { CloudTrail } from '@aws-accelerator/common/src/aws/cloud-trail';
1112
import { PutEventSelectorsRequest, UpdateTrailRequest } from 'aws-sdk/clients/cloudtrail';
1213
import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
1314
import { LoadConfigurationInput } from './load-configuration-step';
14-
import { S3 } from '@aws-accelerator/common/src/aws/s3';
15+
import { loadOutputs } from './utils/load-outputs';
1516

1617
interface AccountDefaultSettingsInput extends LoadConfigurationInput {
1718
assumeRoleName: string;
1819
accounts: Account[];
19-
stackOutputBucketName: string;
20-
stackOutputBucketKey: string;
21-
stackOutputVersion: string;
20+
outputTableName: string;
2221
}
2322

24-
const s3 = new S3();
23+
const dynamodb = new DynamoDB();
2524

2625
export const handler = async (input: AccountDefaultSettingsInput) => {
2726
console.log('Setting account level defaults for all accounts in an organization ...');
2827
console.log(JSON.stringify(input, null, 2));
2928

30-
const {
31-
assumeRoleName,
32-
accounts,
33-
configRepositoryName,
34-
configFilePath,
35-
configCommitId,
36-
stackOutputBucketName,
37-
stackOutputBucketKey,
38-
stackOutputVersion,
39-
} = input;
40-
41-
const outputsString = await s3.getObjectBodyAsString({
42-
Bucket: stackOutputBucketName,
43-
Key: stackOutputBucketKey,
44-
VersionId: stackOutputVersion,
45-
});
29+
const { assumeRoleName, accounts, configRepositoryName, configFilePath, configCommitId, outputTableName } = input;
4630

4731
// Retrieve Configuration from Code Commit with specific commitId
4832
const acceleratorConfig = await loadAcceleratorConfig({
@@ -53,7 +37,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => {
5337

5438
const logAccountKey = acceleratorConfig.getMandatoryAccountKey('central-log');
5539

56-
const outputs = JSON.parse(outputsString) as StackOutput[];
40+
const outputs = await loadOutputs(outputTableName, dynamodb);
5741

5842
const sts = new STS();
5943

‎src/core/runtime/src/add-scp-step.ts‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,19 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts';
22
import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations';
33
import { LoadConfigurationInput } from './load-configuration-step';
44
import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
5-
import { S3 } from '@aws-accelerator/common/src/aws/s3';
6-
import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output';
5+
import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
76
import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts';
87
import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp';
8+
import { loadOutputs } from './utils/load-outputs';
99

1010
interface AddScpInput extends LoadConfigurationInput {
1111
acceleratorPrefix: string;
1212
accounts: Account[];
1313
organizationalUnits: OrganizationalUnit[];
14-
stackOutputBucketName: string;
15-
stackOutputBucketKey: string;
16-
stackOutputVersion: string;
14+
outputTableName: string;
1715
}
1816

19-
const s3 = new S3();
17+
const dynamodb = new DynamoDB();
2018

2119
export const handler = async (input: AddScpInput) => {
2220
console.log(`Adding service control policy to organization...`);
@@ -29,9 +27,7 @@ export const handler = async (input: AddScpInput) => {
2927
configRepositoryName,
3028
configFilePath,
3129
configCommitId,
32-
stackOutputBucketName,
33-
stackOutputBucketKey,
34-
stackOutputVersion,
30+
outputTableName,
3531
} = input;
3632

3733
// Retrieve Configuration from Code Commit with specific commitId
@@ -43,12 +39,7 @@ export const handler = async (input: AddScpInput) => {
4339
const organizationAdminRole = config['global-options']['organization-admin-role']!;
4440
const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole);
4541

46-
const outputsString = await s3.getObjectBodyAsString({
47-
Bucket: stackOutputBucketName,
48-
Key: stackOutputBucketKey,
49-
VersionId: stackOutputVersion,
50-
});
51-
const outputs = JSON.parse(outputsString) as StackOutput[];
42+
const outputs = await loadOutputs(outputTableName, dynamodb);
5243

5344
// Find the SCP artifact output
5445
const artifactOutput = ArtifactOutputFinder.findOneByName({

0 commit comments

Comments
 (0)