Skip to content

Commit 5ecb4bd

Browse files
hickeydh-awshickeydh-aws
authored
Added memory checks and verbose logging. (#817)
Co-authored-by: hickeydh <hickeydh@amazon.com>
1 parent 08f45ed commit 5ecb4bd

File tree

4 files changed

+63
-76
lines changed

4 files changed

+63
-76
lines changed

‎src/deployments/cdk/cdk.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as app from './src/app';
44
import microstats from 'microstats';
55
import { debugModeEnabled } from '@aws-cdk/core/lib/debug';
66
import * as v8 from 'v8';
7+
const fs = require('fs').promises;
78

89
const PAGE_SIZE = parseInt(process.env.DEPLOY_STACK_PAGE_SIZE) || 850;
910

@@ -54,6 +55,7 @@ const getHeapStatistics = () => {
5455
};
5556

5657
async function main() {
58+
await fs.writeFile('/tmp/buildStatus.txt', 'started', 'utf8');
5759
if (debugModeEnabled()) {
5860
microstats.start(microstatsOptions, err => {
5961
console.log(err);
@@ -95,6 +97,9 @@ async function main() {
9597
console.log(`deploying stack ${i + 1} of ${apps.length}`);
9698
if (appsPage.length > PAGE_SIZE - 1 || i === apps.length - 1) {
9799
const toolkit = await CdkToolkit.create(appsPage);
100+
if (debugModeEnabled()) {
101+
console.log(getHeapStatistics());
102+
}
98103
if (commands.includes('bootstrap')) {
99104
await toolkit.bootstrap();
100105
}
@@ -112,6 +117,7 @@ async function main() {
112117
if (debugModeEnabled()) {
113118
microstats.stop();
114119
}
120+
await fs.writeFile('/tmp/buildStatus.txt', 'complete', 'utf8');
115121
}
116122

117123
// eslint-disable-next-line @typescript-eslint/no-floating-promises

‎src/deployments/cdk/codebuild-deploy.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
#!/bin/sh
22

3-
if [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_LARGE" ]; then
3+
4+
if [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_MEDIUM" ]; then
5+
echo "Increasing the max_old_space_size to 2048"
6+
export NODE_OPTIONS=--max_old_space_size=2048
7+
elif [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_LARGE" ]; then
48
echo "The codebuild build type has changed from default to ${COMPUTE_TYPE}, increasing the max_old_space_size to 8192"
59
export NODE_OPTIONS=--max_old_space_size=8192
610
elif [ "${COMPUTE_TYPE}" = "BUILD_GENERAL1_2XLARGE" ]; then

‎src/deployments/cdk/toolkit.ts‎

Lines changed: 46 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -167,27 +167,28 @@ export class CdkToolkit {
167167
// Merge all stack outputs
168168
return combinedOutputs;
169169
}
170-
171-
async deployStack(stack: CloudFormationStackArtifact): Promise<StackOutput[]> {
172-
console.log(
173-
`Deploying stack ${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region}`,
174-
);
175-
170+
async deploymentLog(stack: CloudFormationStackArtifact, message: string, messageType: string = 'INFO') {
171+
const stackLoggingInfo = {
172+
stackName: stack.displayName,
173+
stackEnvironment: stack.environment,
174+
assumeRoleArn: stack.assumeRoleArn || 'N/A',
175+
message,
176+
messageType,
177+
};
178+
179+
console.log(JSON.stringify(stackLoggingInfo, null, 4));
180+
}
181+
async deployStack(stack: CloudFormationStackArtifact, retries: number = 0): Promise<StackOutput[]> {
182+
this.deploymentLog(stack, 'Deploying Stack');
176183
const stackExists = await this.cloudFormation.stackExists({ stack });
177-
console.log(
178-
`Stack ${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region} exists`,
179-
stackExists,
180-
);
184+
this.deploymentLog(stack, `Stack Exists: ${stackExists}`);
181185

182186
const resources = Object.keys(stack.template.Resources || {});
187+
183188
if (resources.length === 0) {
184-
console.warn(
185-
`${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region}: stack has no resources`,
186-
);
189+
this.deploymentLog(stack, 'Stack has no resources');
187190
if (stackExists) {
188-
console.warn(
189-
`${stack.displayName} in account ${stack.environment.account} in region ${stack.environment.region}: deleting existing stack`,
190-
);
191+
this.deploymentLog(stack, 'Deleting existing stack');
191192
this.destroyStack(stack);
192193
}
193194
return [];
@@ -197,51 +198,37 @@ export class CdkToolkit {
197198
if (debugModeEnabled()) {
198199
cfn.config.logger = console;
199200
}
200-
201-
console.log(
202-
`Calling describeStacks API for ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
203-
);
201+
this.deploymentLog(stack, 'Describing Stack');
204202
const existingStack = await cfn
205203
.describeStacks({
206204
StackName: stack.id,
207205
})
208206
.promise();
209-
console.log(`Finding status of ${stack.displayName} stack`);
210207
const stackStatus = existingStack.Stacks[0].StackStatus;
211-
console.log(
212-
`${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region} status`,
213-
stackStatus,
214-
);
208+
this.deploymentLog(stack, `Stack Status: ${stackStatus}`);
209+
215210
try {
216211
if (stackStatus === 'ROLLBACK_COMPLETE') {
217-
console.log(
218-
`Calling updateTerminationProtection API on ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
219-
);
212+
this.deploymentLog(stack, 'Disabling termination protection');
220213
await cfn
221214
.updateTerminationProtection({
222215
StackName: stack.id,
223216
EnableTerminationProtection: false,
224217
})
225218
.promise();
226-
console.log(
227-
`Successfully disabled termination protection on ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
228-
);
219+
this.deploymentLog(stack, 'Disabled termination protection');
229220
}
230221
} catch (e) {
231-
console.warn(
232-
`Failed to disable termination protection for ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}: ${e}`,
233-
);
222+
this.deploymentLog(stack, 'Could not disable termination protection');
223+
console.log(e);
234224
}
235225
}
236226

237227
try {
238228
// Add stack tags to the tags list
239229
// const tags = this.tags || [];
240230
const tags = [...tagsForStack(stack)];
241-
242-
console.log(
243-
`Calling deployStack API on ${stack.displayName} stack in account ${stack.environment.account} and region ${stack.environment.region}`,
244-
);
231+
this.deploymentLog(stack, 'Calling deployStack API');
245232
const result = await this.cloudFormation.deployStack({
246233
stack,
247234
deployName: stack.stackName,
@@ -256,16 +243,13 @@ export class CdkToolkit {
256243
});
257244

258245
if (result.noOp) {
259-
console.log(
260-
`${stack.displayName} in account ${stack.environment.account} and region ${stack.environment.region} has no changes`,
261-
);
246+
this.deploymentLog(stack, 'No changes to deploy');
262247
} else {
263-
console.log(
264-
` in account ${stack.environment.account} and region ${stack.environment.region} deployment successful`,
265-
);
248+
this.deploymentLog(stack, 'Deployment Successful');
266249
}
267-
250+
this.deploymentLog(stack, 'Deleting assembly directory');
268251
this.deleteAssemblyDir(stack.assembly.directory);
252+
this.deploymentLog(stack, 'Deleted assembly directory');
269253

270254
return Object.entries(result.outputs).map(([name, value]) => ({
271255
stack: stack.stackName,
@@ -275,17 +259,16 @@ export class CdkToolkit {
275259
value,
276260
}));
277261
} catch (e) {
278-
console.log(
279-
`${stack.displayName} stack in account ${stack.environment.account} and region ${stack.environment.region} failed to deploy`,
280-
);
262+
this.deploymentLog(stack, `Failed to deploy: ${e}`, 'ERROR');
281263
if (!stackExists) {
282-
console.warn(
283-
`deleting newly created failed stack ${stack.displayName} stack in account ${stack.environment.account} and region ${stack.environment.region}`,
284-
);
264+
this.deploymentLog(stack, 'Deleting failed stack');
285265
await this.destroyStack(stack);
286-
console.warn(
287-
`${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}: deleted newly created failed stack`,
288-
);
266+
this.deploymentLog(stack, 'Deleted failed stack');
267+
}
268+
if (retries < 1) {
269+
console.log(e);
270+
this.deploymentLog(stack, 'Deployment failed because of error. Retrying deployment');
271+
return await this.deployStack(stack, retries + 1);
289272
}
290273
throw e;
291274
}
@@ -295,48 +278,37 @@ export class CdkToolkit {
295278
* Destroy the given stack. It skips deletion when stack termination is turned on.
296279
*/
297280
private async destroyStack(stack: CloudFormationStackArtifact): Promise<void> {
281+
this.deploymentLog(stack, 'Destroying stack');
298282
console.log(
299283
`Destroying ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
300284
);
301285
try {
302286
const sdk = await this.props.sdkProvider.forEnvironment(stack.environment, Mode.ForWriting);
303287
const cfn = sdk.cloudFormation();
304-
console.log(
305-
`Trying to disable termination protection before destroying ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
306-
);
288+
this.deploymentLog(stack, 'Disabling termination protection');
307289
await cfn
308290
.updateTerminationProtection({
309291
StackName: stack.id,
310292
EnableTerminationProtection: false,
311293
})
312294
.promise();
313-
console.log(
314-
`Successfully disabled termination protection on ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
315-
);
295+
this.deploymentLog(stack, 'Disabled termination protection');
316296
} catch (e) {
317-
console.warn(
318-
`${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}: cannot disable stack termination protection`,
319-
);
297+
this.deploymentLog(stack, 'Cloud not disable termination protection');
320298
}
321299
try {
322-
console.log(
323-
`Calling destroyStack API on ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
324-
);
300+
this.deploymentLog(stack, 'Destroying stack');
325301
await this.cloudFormation.destroyStack({
326302
stack,
327303
deployName: stack.stackName,
328304
roleArn: undefined,
329305
force: true,
330306
});
331-
console.log(
332-
`Successfully destroyed/deleted the ${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}`,
333-
);
307+
this.deploymentLog(stack, 'Successfully destroyed stack');
334308
} catch (e) {
335-
const errorMessage = `${e}`;
336-
if (errorMessage.includes('it may need to be manually deleted')) {
337-
console.warn(
338-
`${stack.displayName} stack in account ${stack.environment.account} in region ${stack.environment.region}: ${e}`,
339-
);
309+
this.deploymentLog(stack, 'Could not destroy stack');
310+
console.log(e);
311+
if (e.errorMessage.includes('it may need to be manually deleted')) {
340312
return;
341313
}
342314
throw e;
@@ -345,7 +317,6 @@ export class CdkToolkit {
345317
private async deleteAssemblyDir(assemblyPath: string) {
346318
try {
347319
await fsp.rmdir(assemblyPath, { recursive: true });
348-
console.log('Removing AssemblyDir for succesfully deployed stack', assemblyPath);
349320
} catch (err) {
350321
console.warn(err);
351322
console.log('Could not remove AssemblyDir for succesfully deployed stack', assemblyPath);

‎src/lib/cdk-accelerator/src/codebuild/cdk-deploy-project.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,12 @@ export class PrebuiltCdkDeployProject extends CdkDeployProjectBase {
158158
build: {
159159
commands: [`cd ${appDir}`, `sh ${entrypointFileName}`],
160160
},
161+
post_build: {
162+
commands: [
163+
'buildComplete=`cat /tmp/buildStatus.txt`',
164+
'if [ $buildComplete = "complete" ]; then echo "Build Finished"; else echo "Build did not finish. please review logs for errors!" && exit 1; fi',
165+
],
166+
},
161167
},
162168
}),
163169
environment: {

0 commit comments

Comments
 (0)