feat(core): regionalize cwl to s3 functionality (#346)

* Regionalize Amazon CloudWatch Logging to s3 functionality

Author: naveenkoppula

reference-artifacts/config.example.json

@@ -11,7 +11,7 @@
     "workloadaccounts-suffix" : 1,
     "workloadaccounts-prefix" : "config",
     "workloadaccounts-param-filename": "config.json",
-	"ignored-ous": [],
+	  "ignored-ous": [],
     "supported-regions": [
       "ap-northeast-1",
       "ap-northeast-2",
@@ -64,6 +64,7 @@
       "ssm-to-s3": true,
       "ssm-to-cwl": true
     },
+    "additional-cwl-regions": {},
     "reports": {
       "cost-and-usage-report": {
         "additional-schema-elements": ["RESOURCES"],

reference-artifacts/master-config-sample-snippets/sample_snippets.json

@@ -16,6 +16,12 @@ Config File Examples:  {Not in sample config file}
           "exclusions": ["def/*"]
         }
       ],
+************************************Additional regions for Amazon CloudWatch Central Logging to S3
+    "additional-cwl-regions": {
+      "us-east-1": {
+        "kinesis-stream-shard-count": 1
+      }
+    },
 ************************************cert REQUEST format
       "certificates": [
         {

src/deployments/cdk/src/apps/phase--1.ts

@@ -9,6 +9,9 @@ import * as customResourceRoles from '../deployments/iam';
  *   - Creating required roles for guardDuty custom resources
  *   - Creating required roles for securityHub custom resources
  *   - Creating required roles for IamCreateRole custom resource
+ *   - Creating required roles for createSSMDocument custom resource
+ *   - Creating required roles for createLogGroup custom resource
+ *   - Creating required roles for CWLCentralLoggingSubscriptionFilterRole custom resource
  */
 export async function deploy({ acceleratorConfig, accountStacks, accounts }: PhaseInput) {
   // creates roles for macie custom resources
@@ -35,15 +38,23 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts }: Pha
     accounts,
   });
 
-  // Creates roles for IamCreateRole custom resource
+  // Creates roles for createSSMDocument custom resource
   await customResourceRoles.createSSMDocumentRoles({
     accountStacks,
     accounts,
     config: acceleratorConfig,
   });
 
+  // Creates roles for createLogGroup custom resource
   await customResourceRoles.createLogGroupRole({
     accountStacks,
     accounts,
   });
+
+  // Creates roles for createCwlSubscriptionFilter custom resource
+  await customResourceRoles.createCwlAddSubscriptionFilterRoles({
+    accountStacks,
+    accounts,
+    config: acceleratorConfig,
+  });
 }

src/deployments/cdk/src/apps/phase-0.ts

@@ -167,6 +167,14 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte
     config: acceleratorConfig,
   });
 
+  // Creating roles required for CWL Central Logging
+  await iamDeployment.createCwlCentralLoggingRoles({
+    acceleratorPrefix: context.acceleratorPrefix,
+    accountStacks,
+    config: acceleratorConfig,
+    logBucket,
+  });
+
   /**
    * Code to create LogGroups required for DNS Logging
    */

src/deployments/cdk/src/apps/phase-1.ts

@@ -433,13 +433,12 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte
   }
 
   // Central Services step 1
-  const shardCount = acceleratorConfig['global-options']['central-log-services']['kinesis-stream-shard-count'];
-  const logsAccountStack = accountStacks.getOrCreateAccountStack(logAccountKey);
   await cwlCentralLoggingToS3.step1({
-    accountStack: logsAccountStack,
+    accountStacks,
     accounts,
     logBucket,
-    shardCount,
+    outputs,
+    config: acceleratorConfig,
   });
 
   await vpcDeployment.step1({

src/deployments/cdk/src/deployments/central-services/central-logging-s3/outputs.ts

@@ -0,0 +1,5 @@
+import * as t from 'io-ts';
+import { createCfnStructuredOutput } from '../../../common/structured-output';
+import { LogDestinationOutput } from '@aws-accelerator/common-outputs/src/log-destination';
+
+export const CfnLogDestinationOutput = createCfnStructuredOutput(LogDestinationOutput);

src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-1.ts

@@ -1,36 +1,77 @@
 import * as cdk from '@aws-cdk/core';
-import * as iam from '@aws-cdk/aws-iam';
-import { createRoleName, createName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator';
+import { createName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator';
 import * as kinesis from '@aws-cdk/aws-kinesis';
 import * as s3 from '@aws-cdk/aws-s3';
 import * as logs from '@aws-cdk/aws-logs';
 import * as kinesisfirehose from '@aws-cdk/aws-kinesisfirehose';
-import { AccountStack } from '../../../common/account-stacks';
+import { AccountStacks } from '../../../common/account-stacks';
 import { Account } from '../../../utils/accounts';
 import { JsonOutputValue } from '../../../common/json-output';
 import { CLOUD_WATCH_CENTRAL_LOGGING_BUCKET_PREFIX } from '@aws-accelerator/common/src/util/constants';
+import * as c from '@aws-accelerator/common-config';
+import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output';
+import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role';
+import { CfnLogDestinationOutput } from './outputs';
 
 export interface CentralLoggingToS3Step1Props {
-  accountStack: AccountStack;
+  accountStacks: AccountStacks;
   accounts: Account[];
   logBucket: s3.IBucket;
-  shardCount?: number;
+  outputs: StackOutput[];
+  config: c.AcceleratorConfig;
 }
 
 /**
  * Enable Central Logging to S3 in "log-archive" account Step 1
  */
 export async function step1(props: CentralLoggingToS3Step1Props) {
-  const { accountStack, accounts, logBucket, shardCount } = props;
+  const { accountStacks, accounts, logBucket, config, outputs } = props;
   // Setup for CloudWatch logs storing in logs account
   const allAccountIds = accounts.map(account => account.id);
-  await cwlSettingsInLogArchive({
-    scope: accountStack,
-    accountIds: allAccountIds,
-    bucketArn: logBucket.bucketArn,
-    encryptionKey: logBucket.encryptionKey?.keyArn!,
-    shardCount,
+  const centralLogServices = config['global-options']['central-log-services'];
+  const cwlRegionsConfig = config['global-options']['additional-cwl-regions'];
+  if (!cwlRegionsConfig[centralLogServices.region]) {
+    cwlRegionsConfig[centralLogServices.region] = {
+      'kinesis-stream-shard-count': centralLogServices['kinesis-stream-shard-count'],
+    };
+  }
+
+  const cwlLogStreamRoleOutput = IamRoleOutputFinder.tryFindOneByName({
+    outputs,
+    accountKey: centralLogServices.account,
+    roleKey: 'CWLLogsStreamRole',
+  });
+
+  const cwlKinesisStreamRoleOutput = IamRoleOutputFinder.tryFindOneByName({
+    outputs,
+    accountKey: centralLogServices.account,
+    roleKey: 'CWLKinesisStreamRole',
   });
+
+  if (!cwlLogStreamRoleOutput || !cwlKinesisStreamRoleOutput) {
+    console.error(`Skipping CWL Central logging setup due to unavailability of roles in output`);
+    return;
+  }
+
+  // Setting up in default "central-log-services" and "additional-cwl-regions" region
+  for (const [region, regionConfig] of Object.entries(cwlRegionsConfig)) {
+    // Setup CWL Central logging in default region
+    const logAccountStack = accountStacks.tryGetOrCreateAccountStack(centralLogServices.account, region);
+    if (!logAccountStack) {
+      console.error(
+        `Cannot find account stack ${centralLogServices.account}: ${region} while setting up cloudWatch central logging to S3`,
+      );
+      continue;
+    }
+    await cwlSettingsInLogArchive({
+      scope: logAccountStack,
+      accountIds: allAccountIds,
+      bucketArn: logBucket.bucketArn,
+      shardCount: regionConfig['kinesis-stream-shard-count'],
+      logStreamRoleArn: cwlLogStreamRoleOutput.roleArn,
+      kinesisStreamRoleArn: cwlKinesisStreamRoleOutput.roleArn,
+    });
+  }
 }
 
 /**
@@ -41,10 +82,11 @@ async function cwlSettingsInLogArchive(props: {
   scope: cdk.Construct;
   accountIds: string[];
   bucketArn: string;
-  encryptionKey: string;
+  logStreamRoleArn: string;
+  kinesisStreamRoleArn: string;
   shardCount?: number;
 }) {
-  const { scope, accountIds, bucketArn, encryptionKey, shardCount } = props;
+  const { scope, accountIds, bucketArn, logStreamRoleArn, kinesisStreamRoleArn, shardCount } = props;
 
   // Create Kinesis Stream for Logs streaming
   const logsStream = new kinesis.Stream(scope, 'Logs-Stream', {
@@ -56,27 +98,6 @@ async function cwlSettingsInLogArchive(props: {
     shardCount,
   });
 
-  // Create IAM Role for reading logs from stream and push to destination
-  const logsRole = new iam.Role(scope, 'CloudWatch-Logs-Stream-Role', {
-    roleName: createRoleName('CWL-Logs-Stream-Role'),
-    assumedBy: new iam.ServicePrincipal('logs.amazonaws.com'),
-  });
-
-  // Create IAM Policy for reading logs from stream and push to destination
-  const logsRolePolicy = new iam.Policy(scope, 'CWL-Logs-Stream-Policy', {
-    roles: [logsRole],
-    statements: [
-      new iam.PolicyStatement({
-        resources: [logsStream.streamArn],
-        actions: ['kinesis:PutRecord'],
-      }),
-      new iam.PolicyStatement({
-        resources: [logsRole.roleArn],
-        actions: ['iam:PassRole'],
-      }),
-    ],
-  });
-
   const destinationName = createName({
     name: 'LogDestination',
     suffixLength: 0,
@@ -100,57 +121,18 @@ async function cwlSettingsInLogArchive(props: {
   const logDestination = new logs.CfnDestination(scope, 'Log-Destination', {
     destinationName,
     targetArn: logsStream.streamArn,
-    roleArn: logsRole.roleArn,
+    roleArn: logStreamRoleArn,
     destinationPolicy: destinationPolicyStr,
   });
-  logDestination.node.addDependency(logsRolePolicy);
-
-  // Creating IAM role for Kinesis Delivery Stream Role
-  const kinesisStreamRole = new iam.Role(scope, 'CWL-Kinesis-Stream-Role', {
-    roleName: createRoleName('Kinesis-Stream-Role'),
-    assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'),
-  });
 
-  const kinesisStreamPolicy = new iam.Policy(scope, 'CWL-Kinesis-Stream-Policy', {
-    roles: [kinesisStreamRole],
-    statements: [
-      new iam.PolicyStatement({
-        resources: [encryptionKey],
-        actions: ['kms:DescribeKey', 'kms:GenerateDataKey*', 'kms:Decrypt', 'kms:Encrypt', 'kms:ReEncrypt*'],
-      }),
-      new iam.PolicyStatement({
-        resources: [bucketArn, `${bucketArn}/*`],
-        actions: [
-          's3:PutObject',
-          's3:PutObjectAcl',
-          's3:GetEncryptionConfiguration',
-          's3:AbortMultipartUpload',
-          's3:GetBucketLocation',
-          's3:GetObject',
-          's3:ListBucket',
-          's3:ListBucketMultipartUploads',
-          's3:PutObject',
-        ],
-      }),
-      new iam.PolicyStatement({
-        resources: ['*'],
-        actions: ['kinesis:DescribeStream', 'kinesis:GetShardIterator', 'kinesis:GetRecords', 'kinesis:ListShards'],
-      }),
-      new iam.PolicyStatement({
-        resources: ['arn:aws:logs:*:*:*'],
-        actions: ['logs:PutLogEvents'],
-      }),
-    ],
-  });
-
-  const kinesisDeliveryStream = new kinesisfirehose.CfnDeliveryStream(scope, 'Kinesis-Firehouse-Stream', {
+  new kinesisfirehose.CfnDeliveryStream(scope, 'Kinesis-Firehouse-Stream', {
     deliveryStreamName: createName({
       name: 'Kinesis-Delivery-Stream',
     }),
     deliveryStreamType: 'KinesisStreamAsSource',
     kinesisStreamSourceConfiguration: {
       kinesisStreamArn: logsStream.streamArn,
-      roleArn: kinesisStreamRole.roleArn,
+      roleArn: kinesisStreamRoleArn,
     },
     extendedS3DestinationConfiguration: {
       bucketArn,
@@ -159,18 +141,15 @@ async function cwlSettingsInLogArchive(props: {
         sizeInMBs: 50,
       },
       compressionFormat: 'UNCOMPRESSED',
-      roleArn: kinesisStreamRole.roleArn,
+      roleArn: kinesisStreamRoleArn,
       prefix: CLOUD_WATCH_CENTRAL_LOGGING_BUCKET_PREFIX,
     },
   });
-  kinesisDeliveryStream.node.addDependency(kinesisStreamPolicy);
-  kinesisDeliveryStream.node.addDependency(logsRolePolicy);
 
   // Store LogDestination ARN in output so that subsequent phases can access the output
-  new JsonOutputValue(scope, `CloudWatchCentralLoggingOutput`, {
-    type: 'CloudWatchCentralLogging',
-    value: {
-      logDestination: logDestination.attrArn,
-    },
+  new CfnLogDestinationOutput(scope, `CloudWatchCentralLoggingOutput`, {
+    destinationArn: logDestination.attrArn,
+    destinationName: logDestination.destinationName,
+    destinationKey: 'CwlCentralLogDestination',
   });
 }