feature:(core): z120 move secrets to dynamo db (#372)

- Mergin this to start working on Moving SM input to DynamoDB

Author: nachundu-amzn

src/core/cdk/package.json

@@ -23,10 +23,13 @@
     "typescript": "3.8.3"
   },
   "dependencies": {
+    "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1",
+    "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1",
     "@aws-cdk/aws-cloudformation": "1.46.0",
     "@aws-cdk/aws-codebuild": "1.46.0",
     "@aws-cdk/aws-codepipeline": "1.46.0",
     "@aws-cdk/aws-codepipeline-actions": "1.46.0",
+    "@aws-cdk/aws-dynamodb": "1.46.0",
     "@aws-cdk/aws-events": "1.46.0",
     "@aws-cdk/aws-events-targets": "1.46.0",
     "@aws-cdk/aws-iam": "1.46.0",
@@ -36,14 +39,11 @@
     "@aws-cdk/aws-s3": "1.46.0",
     "@aws-cdk/aws-s3-assets": "1.46.0",
     "@aws-cdk/aws-s3-deployment": "1.46.0",
+    "@aws-cdk/aws-secretsmanager": "1.46.0",
     "@aws-cdk/aws-sns": "1.46.0",
     "@aws-cdk/aws-stepfunctions": "1.46.0",
     "@aws-cdk/aws-stepfunctions-tasks": "1.46.0",
-    "@aws-cdk/aws-secretsmanager": "1.46.0",
-    "@aws-cdk/aws-dynamodb": "1.46.0",
     "@aws-cdk/core": "1.46.0",
-    "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1",
-    "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1",
     "@types/cfn-response": "^1.0.3",
     "aws-sdk": "2.668.0",
     "cfn-response": "^1.0.1",

src/core/cdk/src/initial-setup.ts

@@ -69,22 +69,14 @@ export namespace InitialSetup {
 
       const stack = cdk.Stack.of(this);
 
-      const accountsSecret = new secrets.Secret(this, 'Accounts', {
-        secretName: 'accelerator/accounts',
-        description: 'This secret contains the information about the accounts that are used for deployment.',
-      });
-      setSecretValue(accountsSecret, '[]');
-
-      const limitsSecret = new secrets.Secret(this, 'Limits', {
-        secretName: 'accelerator/limits',
-        description: 'This secret contains a copy of the service limits of the Accelerator accounts.',
-      });
-
-      const organizationsSecret = new secrets.Secret(this, 'Organizations', {
-        secretName: 'accelerator/organizations',
-        description: 'This secret contains the information about the organizations that are used for deployment.',
+      const parametersTable = new dynamodb.Table(this, 'ParametersTable', {
+        tableName: createName({
+          name: 'Parameters',
+          suffixLength: 0,
+        }),
+        partitionKey: { name: 'id', type: dynamodb.AttributeType.STRING },
+        encryption: dynamodb.TableEncryption.DEFAULT,
       });
-      setSecretValue(organizationsSecret, '[]');
 
       const outputsTable = new dynamodb.Table(this, 'Outputs', {
         tableName: createName({
@@ -135,9 +127,10 @@ export namespace InitialSetup {
           ACCELERATOR_EXECUTION_ROLE_NAME: props.stateMachineExecutionRole,
           CDK_PLUGIN_ASSUME_ROLE_NAME: props.stateMachineExecutionRole,
           CDK_PLUGIN_ASSUME_ROLE_DURATION: `${buildTimeout.toSeconds()}`,
-          ACCOUNTS_SECRET_ID: accountsSecret.secretArn,
-          LIMITS_SECRET_ID: limitsSecret.secretArn,
-          ORGANIZATIONS_SECRET_ID: organizationsSecret.secretArn,
+          ACCOUNTS_ITEM_ID: 'accounts',
+          LIMITS_ITEM_ID: 'limits',
+          ORGANIZATIONS_ITEM_ID: 'organizations',
+          DYNAMODB_PARAMETERS_TABLE_NAME: parametersTable.tableName,
         },
       });
 
@@ -314,7 +307,8 @@ export namespace InitialSetup {
           role: pipelineRole,
         },
         functionPayload: {
-          organizationsSecretId: organizationsSecret.secretArn,
+          parametersTableName: parametersTable.tableName,
+          itemId: 'organizations',
           configRepositoryName: props.configRepositoryName,
           'configFilePath.$': '$.configuration.configFilePath',
           'configCommitId.$': '$.configuration.configCommitId',
@@ -329,7 +323,9 @@ export namespace InitialSetup {
           role: pipelineRole,
         },
         functionPayload: {
-          accountsSecretId: accountsSecret.secretArn,
+          parametersTableName: parametersTable.tableName,
+          itemId: 'accounts',
+          accountsItemsCountId: 'accounts-items-count',
           'configuration.$': '$.configuration',
         },
         resultPath: '$',
@@ -444,7 +440,8 @@ export namespace InitialSetup {
           'configRepositoryName.$': '$.configRepositoryName',
           'configFilePath.$': '$.configFilePath',
           'configCommitId.$': '$.configCommitId',
-          limitsSecretId: limitsSecret.secretArn,
+          parametersTableName: parametersTable.tableName,
+          itemId: 'limits',
           assumeRoleName: props.stateMachineExecutionRole,
           'accounts.$': '$.accounts',
         },
@@ -462,8 +459,9 @@ export namespace InitialSetup {
           'configFilePath.$': '$.configuration.configFilePath',
           'configCommitId.$': '$.configuration.configCommitId',
           acceleratorPrefix: props.acceleratorPrefix,
-          accountsSecretId: accountsSecret.secretArn,
-          organizationsSecretId: organizationsSecret.secretArn,
+          parametersTableName: parametersTable.tableName,
+          organizationsItemId: 'organizations',
+          accountsItemId: 'accounts',
           configBranch: props.configBranchName,
           'configRootFilePath.$': '$.configuration.configRootFilePath',
         },

src/core/runtime/src/load-accounts-step.ts

@@ -1,11 +1,14 @@
 import { Organizations } from '@aws-accelerator/common/src/aws/organizations';
-import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager';
 import { Account } from '@aws-accelerator/common-outputs/src/accounts';
 import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step';
 import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
+import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests';
 
 export interface LoadAccountsInput {
-  accountsSecretId: string;
+  accountsItemsCountId: string;
+  parametersTableName: string;
+  itemId: string;
   configuration: LoadConfigurationOutput;
 }
 
@@ -15,18 +18,26 @@ export interface LoadAccountsOutput {
   regions: string[];
 }
 
+const dynamoDB = new DynamoDB();
+
 export const handler = async (input: LoadAccountsInput): Promise<LoadAccountsOutput> => {
   console.log(`Loading accounts...`);
   console.log(JSON.stringify(input, null, 2));
 
-  const { accountsSecretId, configuration } = input;
+  const { parametersTableName, configuration, itemId, accountsItemsCountId } = input;
 
   // The first step is to load all the execution roles
   const organizations = new Organizations();
   const organizationAccounts = await organizations.listAccounts();
   const activeAccounts = organizationAccounts.filter(account => account.Status === 'ACTIVE');
 
   const accounts = [];
+
+  const chunk = (totalAccounts: Account[], size: number) =>
+    Array.from({ length: Math.ceil(totalAccounts.length / size) }, (v, i) =>
+      totalAccounts.slice(i * size, i * size + size),
+    );
+
   for (const accountConfig of configuration.accounts) {
     let organizationAccount;
     organizationAccount = activeAccounts.find(a => {
@@ -68,12 +79,26 @@ export const handler = async (input: LoadAccountsInput): Promise<LoadAccountsOut
     });
   }
 
-  // Store the accounts configuration in the accounts secret
-  const secrets = new SecretsManager();
-  await secrets.putSecretValue({
-    SecretId: accountsSecretId,
-    SecretString: JSON.stringify(accounts),
-  });
+  const accountItemsCountItem = await dynamoDB.getItem(getItemInput(parametersTableName, accountsItemsCountId));
+  const itemsCount = !accountItemsCountItem.Item ? 0 : Number(accountItemsCountItem.Item.value.S);
+
+  // Removing existing accounts from dynamodb table
+  for (let index = 0; index < itemsCount; index++) {
+    await dynamoDB.deleteItem(getItemInput(parametersTableName, `${itemId}/${index}`));
+  }
+
+  // Splitting the accounts array to chunks of size 100
+  const accountsChunk = chunk(accounts, 100);
+  // Store the accounts configuration in the dynamodb
+  for (const [index, accountChunk] of Object.entries(accountsChunk)) {
+    await dynamoDB.updateItem(
+      getUpdateItemInput(parametersTableName, `${itemId}/${index}`, JSON.stringify(accountChunk)),
+    );
+  }
+
+  await dynamoDB.updateItem(
+    getUpdateItemInput(parametersTableName, accountsItemsCountId, JSON.stringify(accountsChunk.length)),
+  );
 
   // Find all relevant accounts in the organization
   return {

src/core/runtime/src/load-limits-step.ts

@@ -2,12 +2,14 @@ import { ServiceQuotas } from '@aws-accelerator/common/src/aws/service-quotas';
 import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts';
 import { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits';
 import { STS } from '@aws-accelerator/common/src/aws/sts';
-import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager';
 import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
 import { LoadConfigurationInput } from './load-configuration-step';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
+import { getUpdateItemInput } from './utils/dynamodb-requests';
 
 export interface LoadLimitsInput extends LoadConfigurationInput {
-  limitsSecretId: string;
+  parametersTableName: string;
+  itemId: string;
   accounts: Account[];
   assumeRoleName: string;
 }
@@ -51,11 +53,21 @@ const LIMITS: { [limitKey: string]: LimitCode } = {
   },
 };
 
+const dynamoDB = new DynamoDB();
+
 export const handler = async (input: LoadLimitsInput) => {
   console.log(`Loading limits...`);
   console.log(JSON.stringify(input, null, 2));
 
-  const { configRepositoryName, configFilePath, limitsSecretId, accounts, assumeRoleName, configCommitId } = input;
+  const {
+    configRepositoryName,
+    configFilePath,
+    parametersTableName,
+    accounts,
+    assumeRoleName,
+    configCommitId,
+    itemId,
+  } = input;
 
   // Retrieve Configuration from Code Commit with specific commitId
   const config = await loadAcceleratorConfig({
@@ -147,10 +159,6 @@ export const handler = async (input: LoadLimitsInput) => {
     }
   }
 
-  // Store the limits in the secrets manager
-  const secrets = new SecretsManager();
-  await secrets.putSecretValue({
-    SecretId: limitsSecretId,
-    SecretString: JSON.stringify(limits, null, 2),
-  });
+  // Store the limits in the dynamodb
+  await dynamoDB.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(limits, null, 2)));
 };

src/core/runtime/src/load-organizations-step.ts

@@ -1,26 +1,28 @@
 import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations';
-import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager';
 import { LoadConfigurationInput } from './load-configuration-step';
 import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
 import { Organizations } from '@aws-accelerator/common/src/aws/organizations';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
+import { getUpdateItemInput } from './utils/dynamodb-requests';
 
 export interface LoadOrganizationsInput extends LoadConfigurationInput {
-  organizationsSecretId: string;
+  parametersTableName: string;
+  itemId: string;
 }
 
 export type LoadOrganizationsOutput = {
   organizationalUnits: OrganizationalUnit[];
 };
 
-const secrets = new SecretsManager();
 const organizations = new Organizations();
+const dynamoDB = new DynamoDB();
 
 export const handler = async (input: LoadOrganizationsInput): Promise<OrganizationalUnit[]> => {
   console.log('Load Organizations ...');
   console.log(JSON.stringify(input, null, 2));
 
   const organizationalUnits: OrganizationalUnit[] = [];
-  const { organizationsSecretId, configCommitId, configFilePath, configRepositoryName } = input;
+  const { configCommitId, configFilePath, configRepositoryName, parametersTableName, itemId } = input;
   // Retrieve Configuration from Code Commit with specific commitId
   const config = await loadAcceleratorConfig({
     repositoryName: configRepositoryName,
@@ -44,11 +46,8 @@ export const handler = async (input: LoadOrganizationsInput): Promise<Organizati
     });
   }
 
-  // Store the organizational units configuration in the accounts secret
-  await secrets.putSecretValue({
-    SecretId: organizationsSecretId,
-    SecretString: JSON.stringify(organizationalUnits),
-  });
+  // Store the organizations into the dynamodb
+  await dynamoDB.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(organizationalUnits)));
 
   // Find all relevant accounts in the organization
   return organizationalUnits;

src/core/runtime/src/ou-validation.ts

@@ -4,24 +4,26 @@ import { AcceleratorConfig, AcceleratorUpdateConfig, AccountsConfig } from '@aws
 import { ServiceControlPolicy, FULL_AWS_ACCESS_POLICY_NAME } from '@aws-accelerator/common/src/scp';
 import { Account } from '@aws-accelerator/common-outputs/src/accounts';
 import { OrganizationalUnit as ConfigOrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations';
-import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager';
 import { CodeCommit } from '@aws-accelerator/common/src/aws/codecommit';
 import { LoadConfigurationInput } from './load-configuration-step';
 import { FormatType, pretty } from '@aws-accelerator/common/src/util/perttier';
 import { getFormattedObject, getStringFromObject, equalIgnoreCase } from '@aws-accelerator/common/src/util/common';
 import { PutFileEntry } from 'aws-sdk/clients/codecommit';
 import { JSON_FORMAT, YAML_FORMAT } from '@aws-accelerator/common/src/util/constants';
 import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
+import { getItemInput } from './utils/dynamodb-requests';
 
 export interface ValdationInput extends LoadConfigurationInput {
   acceleratorPrefix: string;
-  accountsSecretId: string;
-  organizationsSecretId: string;
+  parametersTableName: string;
+  organizationsItemId: string;
+  accountsItemId: string;
   configBranch: string;
 }
 
 const organizations = new Organizations();
-const secrets = new SecretsManager();
+const dynamoDB = new DynamoDB();
 const codecommit = new CodeCommit();
 
 /**
@@ -42,8 +44,9 @@ export const handler = async (input: ValdationInput): Promise<string> => {
     configRepositoryName,
     configCommitId,
     acceleratorPrefix,
-    accountsSecretId,
-    organizationsSecretId,
+    parametersTableName,
+    organizationsItemId,
+    accountsItemId,
     configBranch,
     configRootFilePath,
   } = input;
@@ -60,8 +63,8 @@ export const handler = async (input: ValdationInput): Promise<string> => {
   let rootConfig = getFormattedObject(rootConfigString, format);
 
   let config = previousConfig;
-  const previousAccounts = await loadAccounts(accountsSecretId);
-  const previousOrganizationalUnits = await loadOrganizations(organizationsSecretId);
+  const previousAccounts = await loadAccounts(parametersTableName, accountsItemId);
+  const previousOrganizationalUnits = await loadOrganizations(parametersTableName, organizationsItemId);
   const organizationAdminRole = config['global-options']['organization-admin-role'];
   const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole, organizations);
 
@@ -294,20 +297,27 @@ function updateAccountConfig(accountConfig: any, accountInfo: UpdateAccountOutpu
   }
   return accountConfig;
 }
-async function loadAccounts(accountsSecretId: string): Promise<Account[]> {
-  const secret = await secrets.getSecret(accountsSecretId);
-  if (!secret) {
-    throw new Error(`Cannot find secret with ID "${accountsSecretId}"`);
+async function loadAccounts(tableName: string, itemId: string): Promise<Account[]> {
+  let index = 0;
+  const accounts: Account[] = [];
+  while (true) {
+    const item = await dynamoDB.getItem(getItemInput(tableName, `${itemId}/${index}`));
+    if (!item.Item) {
+      break;
+    }
+    accounts.push(...JSON.parse(item.Item.value.S!));
+    index++;
   }
-  return JSON.parse(secret.SecretString!);
+  return accounts;
 }
 
-async function loadOrganizations(organizationsSecretId: string): Promise<ConfigOrganizationalUnit[]> {
-  const secret = await secrets.getSecret(organizationsSecretId);
-  if (!secret) {
-    throw new Error(`Cannot find secret with ID "${organizationsSecretId}"`);
+async function loadOrganizations(tableName: string, itemId: string): Promise<ConfigOrganizationalUnit[]> {
+  const organizationalUnits: ConfigOrganizationalUnit[] = [];
+  const organizationsOutput = await dynamoDB.getItem(getItemInput(tableName, itemId));
+  if (!organizationsOutput.Item) {
+    return organizationalUnits;
   }
-  return JSON.parse(secret.SecretString!);
+  return JSON.parse(organizationsOutput.Item.value.S!);
 }
 
 interface UpdateAccountsOutput {