fix(core): File copy mechanism broken (#591)

* Fixing file copy to have latest file from both s3 and repo
- Performing s3 deployment using aws sync to get latest from repo
- After bucket deployment s3copyfiles to get latest file from customer s3 bucket.
* Removing time check for copy to copy everytime when we find in bucket

Author: naveenkoppula

src/deployments/cdk/src/deployments/artifacts/step-1.ts

@@ -5,6 +5,8 @@ import { AcceleratorConfig } from '@aws-accelerator/common-config/src';
 import { AccountStacks, AccountStack } from '../../common/account-stacks';
 import { JsonOutputValue } from '../../common/json-output';
 import { ArtifactName, CfnArtifactOutput } from './outputs';
+import { createRoleName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator';
+import { S3CopyFiles } from '@aws-accelerator/custom-resource-s3-copy-files';
 
 export interface ArtifactsStep1Props {
   accountStacks: AccountStacks;
@@ -18,8 +20,14 @@ export async function step1(props: ArtifactsStep1Props) {
   const masterAccountKey = config.getMandatoryAccountKey('master');
   const masterAccountStack = accountStacks.getOrCreateAccountStack(masterAccountKey);
 
+  // Get the location of the original central bucket
+  const centralConfigBucketName = config['global-options']['central-bucket'];
+  const centralConfigBucket = s3.Bucket.fromBucketAttributes(masterAccountStack, 'CentralBucket', {
+    bucketName: centralConfigBucketName,
+  });
+
   // upload SCP Artifacts
-  uploadArtifacts({
+  const scpUpload = uploadArtifacts({
     accountStack: masterAccountStack,
     artifactName: 'SCP',
     artifactFolderName: 'SCPs',
@@ -29,7 +37,7 @@ export async function step1(props: ArtifactsStep1Props) {
   });
 
   // upload IAM-Policies Artifacts
-  uploadArtifacts({
+  const iamUpload = uploadArtifacts({
     accountStack: masterAccountStack,
     artifactName: 'IamPolicy',
     artifactFolderName: 'iam-policies',
@@ -39,7 +47,7 @@ export async function step1(props: ArtifactsStep1Props) {
   });
 
   // upload RDGW Artifacts
-  uploadArtifacts({
+  const rdgwUpload = uploadArtifacts({
     accountStack: masterAccountStack,
     artifactName: 'Rdgw',
     artifactFolderName: 'scripts',
@@ -49,7 +57,7 @@ export async function step1(props: ArtifactsStep1Props) {
   });
 
   // upload Rsyslog Artifacts
-  uploadArtifacts({
+  const rsyslogUpload = uploadArtifacts({
     accountStack: masterAccountStack,
     artifactName: 'Rsyslog',
     artifactFolderName: 'rsyslog',
@@ -59,7 +67,7 @@ export async function step1(props: ArtifactsStep1Props) {
   });
 
   // upload SSM-Document Artifacts
-  uploadArtifacts({
+  const ssmUpload = uploadArtifacts({
     accountStack: masterAccountStack,
     artifactName: 'SsmDocument',
     artifactFolderName: 'ssm-documents',
@@ -68,6 +76,21 @@ export async function step1(props: ArtifactsStep1Props) {
     destinationKeyPrefix: 'ssm-documents',
     keepExistingFiles: true,
   });
+
+  // Copy files from source to destination
+  const copyFiles = new S3CopyFiles(masterAccountStack, 'CopyFiles', {
+    roleName: createRoleName('S3CopyFiles'),
+    sourceBucket: centralConfigBucket,
+    destinationBucket: centralBucket,
+    deleteSourceObjects: false,
+    deleteSourceBucket: false,
+    forceUpdate: true,
+  });
+  copyFiles.node.addDependency(ssmUpload);
+  copyFiles.node.addDependency(rsyslogUpload);
+  copyFiles.node.addDependency(rdgwUpload);
+  copyFiles.node.addDependency(iamUpload);
+  copyFiles.node.addDependency(scpUpload);
 }
 
 function uploadArtifacts(props: {
@@ -78,7 +101,7 @@ function uploadArtifacts(props: {
   centralBucket: s3.IBucket;
   destinationKeyPrefix?: string;
   keepExistingFiles?: boolean;
-}) {
+}): s3deployment.BucketDeployment {
   const {
     accountStack,
     artifactName,
@@ -102,15 +125,16 @@ function uploadArtifacts(props: {
     artifactFolderName,
   );
 
-  // TODO Leave existing files in the folder
-  // TODO Do not override existing files
-  // See https://github.com/aws/aws-cdk/issues/953
-  new s3deployment.BucketDeployment(accountStack, `${artifactName}ArtifactsDeployment${accountKey}`, {
-    sources: [s3deployment.Source.asset(artifactsFolderPath)],
-    destinationBucket: centralBucket,
-    destinationKeyPrefix,
-    prune: !keepExistingFiles,
-  });
+  const s3Deployment = new s3deployment.BucketDeployment(
+    accountStack,
+    `${artifactName}ArtifactsDeployment${accountKey}`,
+    {
+      sources: [s3deployment.Source.asset(artifactsFolderPath)],
+      destinationBucket: centralBucket,
+      destinationKeyPrefix,
+      prune: !keepExistingFiles,
+    },
+  );
 
   // outputs to store reference artifacts s3 bucket information
   new JsonOutputValue(accountStack, `${artifactName}ArtifactsOutput${accountKey}`, {
@@ -130,4 +154,5 @@ function uploadArtifacts(props: {
     bucketName: centralBucket.bucketName,
     keyPrefix: artifactKeyPrefix,
   });
+  return s3Deployment;
 }

src/deployments/cdk/src/deployments/defaults/step-1.ts

@@ -79,12 +79,6 @@ function createCentralBucketCopy(props: DefaultsStep1Props) {
   const masterAccountConfig = config['global-options']['aws-org-master'];
   const masterAccountStack = accountStacks.getOrCreateAccountStack(masterAccountConfig.account);
 
-  // Get the location of the original central bucket
-  const centralBucketName = config['global-options']['central-bucket'];
-  const centralBucket = s3.Bucket.fromBucketAttributes(masterAccountStack, 'CentralBucket', {
-    bucketName: centralBucketName,
-  });
-
   const keyAlias = createEncryptionKeyName('Config-Key');
   const encryptionKey = new kms.Key(masterAccountStack, 'CentralBucketKey', {
     alias: `alias/${keyAlias}`,
@@ -131,17 +125,6 @@ function createCentralBucketCopy(props: DefaultsStep1Props) {
     }),
   );
 
-  // Copy files from source to destination
-  const copyFiles = new S3CopyFiles(masterAccountStack, 'CopyFiles', {
-    roleName: createRoleName('S3CopyFiles'),
-    sourceBucket: centralBucket,
-    destinationBucket: bucket,
-    deleteSourceObjects: false,
-    deleteSourceBucket: false,
-    forceUpdate: true,
-  });
-  copyFiles.node.addDependency(bucket);
-
   new CfnCentralBucketOutput(masterAccountStack, 'CentralBucketOutput', {
     bucketArn: bucket.bucketArn,
     bucketName: bucket.bucketName,

src/lib/custom-resources/cdk-s3-copy-files/runtime/src/index.ts

@@ -132,55 +132,32 @@ async function copyObject(props: {
   const { sourceBucketName, destinationBucketName, deleteSourceObjects, sourceObject } = props;
   const sourceKey = sourceObject.Key!;
 
-  let destinationLastModified;
+  let object: AWS.S3.GetObjectOutput;
   try {
-    const headObject = await throttlingBackOff(() =>
+    object = await throttlingBackOff(() =>
       s3
-        .headObject({
-          Bucket: destinationBucketName,
+        .getObject({
+          Bucket: sourceBucketName,
           Key: sourceKey,
         })
         .promise(),
     );
-    destinationLastModified = headObject.LastModified;
   } catch (e) {
-    console.debug(`Unable to head S3 object s3://${destinationBucketName}/${sourceKey}: ${e}`);
+    throw new Error(`Unable to get S3 object s3://${sourceBucketName}/${sourceKey}: ${e}`);
   }
 
-  if (
-    !destinationLastModified ||
-    !sourceObject.LastModified ||
-    compareDate(destinationLastModified, sourceObject.LastModified) < 0
-  ) {
-    let object: AWS.S3.GetObjectOutput;
-    try {
-      object = await throttlingBackOff(() =>
-        s3
-          .getObject({
-            Bucket: sourceBucketName,
-            Key: sourceKey,
-          })
-          .promise(),
-      );
-    } catch (e) {
-      throw new Error(`Unable to get S3 object s3://${sourceBucketName}/${sourceKey}: ${e}`);
-    }
-
-    try {
-      await throttlingBackOff(() =>
-        s3
-          .putObject({
-            Bucket: destinationBucketName,
-            Key: sourceKey,
-            Body: object.Body,
-          })
-          .promise(),
-      );
-    } catch (e) {
-      throw new Error(`Unable to put S3 object s3://${destinationBucketName}/${sourceKey}: ${e}`);
-    }
-  } else {
-    console.debug(`Skipping copy of s3://${sourceBucketName}/${sourceKey}`);
+  try {
+    await throttlingBackOff(() =>
+      s3
+        .putObject({
+          Bucket: destinationBucketName,
+          Key: sourceKey,
+          Body: object.Body,
+        })
+        .promise(),
+    );
+  } catch (e) {
+    throw new Error(`Unable to put S3 object s3://${destinationBucketName}/${sourceKey}: ${e}`);
   }
 
   if (deleteSourceObjects) {