diff --git a/docs/guides/functions.md b/docs/guides/functions.md index 172c1248e..84b7266b1 100644 --- a/docs/guides/functions.md +++ b/docs/guides/functions.md @@ -722,15 +722,24 @@ functions: By default, osls creates function versions for every deploy. This behavior is optional, and can be turned off in cases where you don't invoke past versions by their qualifier. If you would like to do this, you can invoke your functions as `arn:aws:lambda:....:function/myFunc:3` to invoke version 3 for example. -Versions are not cleaned up by osls, so make sure you use a plugin or other tool to prune sufficiently old versions. osls can't clean up versions because it doesn't have information about whether older versions are invoked or not. This feature adds to the number of total stack outputs and resources because a function version is a separate resource from the function it refers to. +Older versions are not removed automatically unless you enable `provider.pruneFunctionVersions`. When enabled, osls deletes function and layer versions beyond the configured limit after each deploy, while keeping versions referenced by aliases. Aliased versions are never deleted. -To turn off function versioning, set the provider-level option `versionFunctions`. +To turn off function versioning, set the provider-level option `versionFunctions`. `pruneFunctionVersions` cannot be used when `versionFunctions` is `false`. ```yml provider: versionFunctions: false ``` +Enable automatic pruning after deploy: + +```yml +provider: + pruneFunctionVersions: true # keeps 10 versions (default) + # pruneFunctionVersions: + # number: 20 +``` + ## Dead Letter Queue (DLQ) When AWS lambda functions fail, they are [retried](http://docs.aws.amazon.com/lambda/latest/dg/retries-on-errors.html). If the retries also fail, AWS has a feature to send information about the failed request to a SNS topic or SQS queue, called the [Dead Letter Queue](http://docs.aws.amazon.com/lambda/latest/dg/dlq.html), which you can use to track and diagnose and react to lambda failures. diff --git a/docs/guides/serverless.yml.md b/docs/guides/serverless.yml.md index 17e7f0eb2..28a07ba0f 100644 --- a/docs/guides/serverless.yml.md +++ b/docs/guides/serverless.yml.md @@ -123,6 +123,10 @@ provider: kmsKeyArn: arn:aws:kms:us-east-1:XXXXXX:key/some-hash # Use function versioning (enabled by default) versionFunctions: false + # After deploy, delete older Lambda function and layer versions (disabled by default) + pruneFunctionVersions: true # keeps 10 versions + # pruneFunctionVersions: + # number: 20 # explicit number of versions to keep # Processor architecture: 'x86_64' or 'arm64' via Graviton2 (default: x86_64) architecture: x86_64 ``` diff --git a/lib/plugins/aws/provider.js b/lib/plugins/aws/provider.js index 5a817a6d5..ccc2edbc5 100644 --- a/lib/plugins/aws/provider.js +++ b/lib/plugins/aws/provider.js @@ -562,6 +562,19 @@ class AwsProvider { awsLambdaTimeout: { type: 'integer', minimum: 1, maximum: 900 }, awsLambdaTracing: { anyOf: [{ enum: ['Active', 'PassThrough'] }, { type: 'boolean' }] }, awsLambdaVersioning: { type: 'boolean' }, + awsPruneFunctionVersions: { + anyOf: [ + { type: 'boolean' }, + { + type: 'object', + properties: { + number: { type: 'integer', minimum: 0 }, + }, + additionalProperties: false, + required: ['number'], + }, + ], + }, awsLambdaVpcConfig: { type: 'object', properties: { @@ -1237,6 +1250,7 @@ class AwsProvider { vpc: { $ref: '#/definitions/awsLambdaVpcConfig' }, vpcEndpointIds: { $ref: '#/definitions/awsCfArrayInstruction' }, versionFunctions: { $ref: '#/definitions/awsLambdaVersioning' }, + pruneFunctionVersions: { $ref: '#/definitions/awsPruneFunctionVersions' }, websocket: { type: 'object', properties: { diff --git a/lib/plugins/aws/prune.js b/lib/plugins/aws/prune.js new file mode 100644 index 000000000..c2cc2d7ef --- /dev/null +++ b/lib/plugins/aws/prune.js @@ -0,0 +1,257 @@ +'use strict'; + +const { + LambdaClient, + DeleteFunctionCommand, + DeleteLayerVersionCommand, + ListAliasesCommand, + ListLayerVersionsCommand, + ListVersionsByFunctionCommand, +} = require('@aws-sdk/client-lambda'); +const ServerlessError = require('../../serverless-error'); +const { log } = require('../../utils/serverless-utils/log'); +const { + getAwsErrorMessage, + getAwsErrorStatusCode, + isLambdaResourceNotFoundError, +} = require('../../aws/aws-sdk-v3-error'); + +const DEFAULT_PRUNE_VERSIONS = 10; + +class AwsPrune { + constructor(serverless, options) { + this.serverless = serverless; + this.options = options || {}; + this.provider = this.serverless.getProvider('aws'); + + this.hooks = { + 'before:deploy:deploy': async () => this.validateConfiguration(), + 'after:deploy:deploy': async () => this.postDeploy(), + }; + } + + resolvePruneConfig() { + const setting = this.serverless.service.provider.pruneFunctionVersions; + if (setting == null || setting === false) return null; + if (setting === true) return { number: DEFAULT_PRUNE_VERSIONS }; + if (typeof setting === 'object' && setting.number != null) { + const number = parseInt(setting.number, 10); + if (!isNaN(number) && number >= 0) return { number }; + } + throw new ServerlessError( + 'provider.pruneFunctionVersions must be true or an object with a non-negative number property', + 'INVALID_PRUNE_FUNCTION_VERSIONS_CONFIG' + ); + } + + getNumber() { + const config = this.resolvePruneConfig(); + return config ? config.number : undefined; + } + + shouldVersionFunction(functionKey) { + const functionObject = this.serverless.service.getFunction(functionKey); + if (functionObject.versionFunction != null) { + return functionObject.versionFunction; + } + return this.serverless.service.provider.versionFunctions !== false; + } + + validateConfiguration() { + const config = this.resolvePruneConfig(); + if (!config) return; + + if (this.serverless.service.provider.versionFunctions === false) { + throw new ServerlessError( + 'provider.pruneFunctionVersions cannot be used when provider.versionFunctions is false', + 'PRUNE_INCOMPATIBLE_WITH_VERSION_FUNCTIONS' + ); + } + } + + async postDeploy() { + const config = this.resolvePruneConfig(); + if (!config) return; + + this.validateConfiguration(); + + await Promise.all([this.pruneFunctions(), this.pruneLayers()]); + } + + async getLambdaClient() { + this.lambdaClientPromise ||= this.provider + .getAwsSdkV3Config() + .then((lambdaConfig) => new LambdaClient(lambdaConfig)); + return this.lambdaClientPromise; + } + + async pruneLayers() { + const layerNames = this.serverless.service + .getAllLayers() + .map((key) => this.serverless.service.getLayer(key).name || key); + + let prunedCount = 0; + for (const layerName of layerNames) { + const versions = await this.listVersionsForLayer(layerName); + if (!versions.length) continue; + + const deletionCandidates = this.selectPruneVersionsForLayer(versions); + prunedCount += await this.deleteVersionsForLayer(layerName, deletionCandidates); + } + + if (prunedCount > 0) { + log.notice.success(`Pruned ${prunedCount} layer version${prunedCount === 1 ? '' : 's'}`); + } + } + + async pruneFunctions() { + let prunedCount = 0; + + for (const functionKey of this.serverless.service.getAllFunctions()) { + if (!this.shouldVersionFunction(functionKey)) continue; + + const functionName = this.serverless.service.getFunction(functionKey).name; + const [versions, aliases] = await Promise.all([ + this.listVersionsForFunction(functionName), + this.listAliasesForFunction(functionName), + ]); + if (!versions.length) continue; + + const deletionCandidates = this.selectPruneVersionsForFunction(versions, aliases); + prunedCount += await this.deleteVersionsForFunction(functionName, deletionCandidates); + } + + if (prunedCount > 0) { + log.notice.success(`Pruned ${prunedCount} function version${prunedCount === 1 ? '' : 's'}`); + } + } + + async deleteVersionsForLayer(layerName, versions) { + const lambda = await this.getLambdaClient(); + let deletedCount = 0; + + for (const version of versions) { + log.info(`Deleting layer version ${layerName}:${version}.`); + await lambda.send( + new DeleteLayerVersionCommand({ + LayerName: layerName, + VersionNumber: version, + }) + ); + deletedCount++; + } + + return deletedCount; + } + + async deleteVersionsForFunction(functionName, versions) { + const lambda = await this.getLambdaClient(); + let deletedCount = 0; + + for (const version of versions) { + log.info(`Deleting function version ${functionName}:${version}.`); + try { + await lambda.send( + new DeleteFunctionCommand({ + FunctionName: functionName, + Qualifier: version, + }) + ); + deletedCount++; + } catch (error) { + const statusCode = getAwsErrorStatusCode(error); + const message = getAwsErrorMessage(error) || ''; + if ( + statusCode === 400 && + message.startsWith('Lambda was unable to delete') && + message.includes('because it is a replicated function.') + ) { + log.warning( + `Unable to delete replicated Lambda@Edge function version ${functionName}:${version}.` + ); + } else { + throw error; + } + } + } + + return deletedCount; + } + + async listAliasesForFunction(functionName) { + try { + return await this.paginateLambda( + ListAliasesCommand, + { FunctionName: functionName }, + 'Aliases' + ); + } catch (error) { + if (isLambdaResourceNotFoundError(error)) return []; + throw error; + } + } + + async listVersionsForFunction(functionName) { + try { + return await this.paginateLambda( + ListVersionsByFunctionCommand, + { FunctionName: functionName }, + 'Versions' + ); + } catch (error) { + if (isLambdaResourceNotFoundError(error)) return []; + throw error; + } + } + + async listVersionsForLayer(layerName) { + try { + return await this.paginateLambda( + ListLayerVersionsCommand, + { LayerName: layerName }, + 'LayerVersions' + ); + } catch (error) { + if (isLambdaResourceNotFoundError(error)) return []; + throw error; + } + } + + async paginateLambda(Command, params, resultKey) { + const lambda = await this.getLambdaClient(); + const results = []; + let input = params; + + do { + const response = await lambda.send(new Command(input)); + results.push(...(response[resultKey] || [])); + input = response.NextMarker ? { ...params, Marker: response.NextMarker } : null; + } while (input); + + return results; + } + + selectPruneVersionsForFunction(versions, aliases) { + const aliasedVersions = new Set(aliases.map((alias) => String(alias.FunctionVersion))); + + return versions + .map((versionEntry) => versionEntry.Version) + .filter((version) => version !== '$LATEST') + .filter((version) => !aliasedVersions.has(String(version))) + .sort((a, b) => + parseInt(a, 10) === parseInt(b, 10) ? 0 : parseInt(a, 10) > parseInt(b, 10) ? -1 : 1 + ) + .slice(this.getNumber()); + } + + selectPruneVersionsForLayer(versions) { + return versions + .map((versionEntry) => versionEntry.Version) + .sort((a, b) => + parseInt(a, 10) === parseInt(b, 10) ? 0 : parseInt(a, 10) > parseInt(b, 10) ? -1 : 1 + ) + .slice(this.getNumber()); + } +} + +module.exports = AwsPrune; diff --git a/lib/plugins/index.js b/lib/plugins/index.js index 181489f5b..8d7b2e8fe 100644 --- a/lib/plugins/index.js +++ b/lib/plugins/index.js @@ -28,6 +28,7 @@ module.exports = [ require('./aws/remove/index.js'), require('./aws/rollback.js'), require('./aws/rollback-function.js'), + require('./aws/prune.js'), require('./aws/package/compile/layers.js'), require('./aws/package/compile/functions.js'), require('./aws/package/compile/events/schedule.js'), diff --git a/test/unit/lib/plugins/aws/prune.test.js b/test/unit/lib/plugins/aws/prune.test.js new file mode 100644 index 000000000..14f16b9f0 --- /dev/null +++ b/test/unit/lib/plugins/aws/prune.test.js @@ -0,0 +1,224 @@ +'use strict'; + +const expect = require('chai').expect; +const sinon = require('sinon'); +const AwsPrune = require('../../../../../lib/plugins/aws/prune'); +const AwsProvider = require('../../../../../lib/plugins/aws/provider'); +const Serverless = require('../../../../../lib/serverless'); +const ServerlessError = require('../../../../../lib/serverless-error'); +const CLI = require('../../../../../lib/classes/cli'); + +describe('AwsPrune', () => { + let serverless; + let awsPrune; + + beforeEach(() => { + serverless = new Serverless({ commands: [], options: {} }); + serverless.cli = new CLI(serverless); + serverless.service.provider = { name: 'aws', versionFunctions: true }; + serverless.service.functions = { + FunctionA: { name: 'service-FunctionA' }, + FunctionB: { name: 'service-FunctionB' }, + }; + serverless.service.layers = { + LayerA: { name: 'layer-LayerA' }, + }; + serverless.service.getAllLayers = () => Object.keys(serverless.service.layers); + serverless.service.getLayer = (key) => serverless.service.layers[key]; + const options = { stage: 'dev', region: 'us-east-1' }; + serverless.setProvider('aws', new AwsProvider(serverless, options)); + awsPrune = new AwsPrune(serverless, options); + }); + + describe('#constructor()', () => { + it('should define deploy hooks', () => { + expect(awsPrune.hooks['before:deploy:deploy']).to.be.a('function'); + expect(awsPrune.hooks['after:deploy:deploy']).to.be.a('function'); + }); + }); + + describe('#resolvePruneConfig()', () => { + it('should return null when pruning is not configured', () => { + expect(awsPrune.resolvePruneConfig()).to.equal(null); + }); + + it('should return the default number when enabled with true', () => { + serverless.service.provider.pruneFunctionVersions = true; + expect(awsPrune.resolvePruneConfig()).to.deep.equal({ number: 10 }); + }); + + it('should return an explicit number from object configuration', () => { + serverless.service.provider.pruneFunctionVersions = { number: 20 }; + expect(awsPrune.resolvePruneConfig()).to.deep.equal({ number: 20 }); + }); + + it('should return null when explicitly disabled', () => { + serverless.service.provider.pruneFunctionVersions = false; + expect(awsPrune.resolvePruneConfig()).to.equal(null); + }); + + it('should throw when configuration is invalid', () => { + serverless.service.provider.pruneFunctionVersions = { number: 'invalid' }; + expect(() => awsPrune.resolvePruneConfig()).to.throw(ServerlessError); + }); + }); + + describe('#shouldVersionFunction()', () => { + it('should skip functions with versionFunction set to false', () => { + serverless.service.functions.FunctionA.versionFunction = false; + expect(awsPrune.shouldVersionFunction('FunctionA')).to.equal(false); + expect(awsPrune.shouldVersionFunction('FunctionB')).to.equal(true); + }); + }); + + describe('#validateConfiguration()', () => { + it('should throw when versionFunctions is false', () => { + serverless.service.provider.pruneFunctionVersions = true; + serverless.service.provider.versionFunctions = false; + + expect(() => awsPrune.validateConfiguration()).to.throw(ServerlessError); + }); + + it('should not throw when pruning is disabled', () => { + serverless.service.provider.versionFunctions = false; + expect(() => awsPrune.validateConfiguration()).to.not.throw(); + }); + }); + + describe('#selectPruneVersionsForFunction()', () => { + beforeEach(() => { + serverless.service.provider.pruneFunctionVersions = { number: 2 }; + }); + + it('should keep the requested number of newest versions', () => { + const versions = [{ Version: '1' }, { Version: '2' }, { Version: '3' }, { Version: '4' }]; + const result = awsPrune.selectPruneVersionsForFunction(versions, []); + expect(result).to.deep.equal(['2', '1']); + }); + + it('should not delete $LATEST or aliased versions', () => { + const versions = [ + { Version: '$LATEST' }, + { Version: '1' }, + { Version: '2' }, + { Version: '3' }, + { Version: '4' }, + { Version: '5' }, + ]; + const aliases = [ + { FunctionVersion: '1' }, + { FunctionVersion: '3' }, + { FunctionVersion: '4' }, + ]; + const result = awsPrune.selectPruneVersionsForFunction(versions, aliases); + expect(result).to.not.include('$LATEST'); + expect(result).to.not.include('1'); + expect(result).to.not.include('3'); + expect(result).to.not.include('4'); + }); + + it('should not delete aliased versions when alias version is a number', () => { + const versions = [{ Version: '1' }, { Version: '2' }, { Version: '3' }, { Version: '4' }]; + const aliases = [{ FunctionVersion: 2 }]; + const result = awsPrune.selectPruneVersionsForFunction(versions, aliases); + expect(result).to.not.include('2'); + }); + }); + + describe('#selectPruneVersionsForLayer()', () => { + beforeEach(() => { + serverless.service.provider.pruneFunctionVersions = { number: 2 }; + }); + + it('should keep the requested number of newest layer versions', () => { + const versions = [{ Version: 1 }, { Version: 2 }, { Version: 3 }, { Version: 4 }]; + const result = awsPrune.selectPruneVersionsForLayer(versions); + expect(result).to.deep.equal([2, 1]); + }); + }); + + describe('#pruneFunctions()', () => { + let listVersionsStub; + let listAliasesStub; + let deleteStub; + + beforeEach(() => { + serverless.service.provider.pruneFunctionVersions = { number: 2 }; + listVersionsStub = sinon.stub(awsPrune, 'listVersionsForFunction'); + listAliasesStub = sinon.stub(awsPrune, 'listAliasesForFunction'); + deleteStub = sinon.stub(awsPrune, 'deleteVersionsForFunction').resolves(); + }); + + afterEach(() => { + listVersionsStub.restore(); + listAliasesStub.restore(); + deleteStub.restore(); + }); + + it('should delete old versions of functions', async () => { + listVersionsStub.resolves([ + { Version: '1' }, + { Version: '2' }, + { Version: '3' }, + { Version: '4' }, + { Version: '5' }, + ]); + listAliasesStub.resolves([]); + + await awsPrune.pruneFunctions(); + + expect(deleteStub.callCount).to.equal(2); + expect(deleteStub.getCall(0).args).to.deep.equal(['service-FunctionA', ['3', '2', '1']]); + }); + + it('should skip functions with versionFunction set to false', async () => { + serverless.service.functions.FunctionA.versionFunction = false; + listVersionsStub.resolves([{ Version: '1' }, { Version: '2' }, { Version: '3' }]); + listAliasesStub.resolves([]); + + await awsPrune.pruneFunctions(); + + expect(deleteStub.calledOnce).to.equal(true); + expect(deleteStub.firstCall.args[0]).to.equal('service-FunctionB'); + }); + }); + + describe('#postDeploy()', () => { + it('should prune functions and layers when enabled', async () => { + serverless.service.provider.pruneFunctionVersions = true; + const pruneFunctionsStub = sinon.stub(awsPrune, 'pruneFunctions').resolves(); + const pruneLayersStub = sinon.stub(awsPrune, 'pruneLayers').resolves(); + + await awsPrune.postDeploy(); + + expect(pruneFunctionsStub.calledOnce).to.equal(true); + expect(pruneLayersStub.calledOnce).to.equal(true); + pruneFunctionsStub.restore(); + pruneLayersStub.restore(); + }); + + it('should prune when number is 0', async () => { + serverless.service.provider.pruneFunctionVersions = { number: 0 }; + const pruneFunctionsStub = sinon.stub(awsPrune, 'pruneFunctions').resolves(); + const pruneLayersStub = sinon.stub(awsPrune, 'pruneLayers').resolves(); + + await awsPrune.postDeploy(); + + expect(pruneFunctionsStub.calledOnce).to.equal(true); + pruneFunctionsStub.restore(); + pruneLayersStub.restore(); + }); + + it('should not prune when disabled', async () => { + const pruneFunctionsStub = sinon.stub(awsPrune, 'pruneFunctions').resolves(); + const pruneLayersStub = sinon.stub(awsPrune, 'pruneLayers').resolves(); + + await awsPrune.postDeploy(); + + expect(pruneFunctionsStub.called).to.equal(false); + expect(pruneLayersStub.called).to.equal(false); + pruneFunctionsStub.restore(); + pruneLayersStub.restore(); + }); + }); +}); diff --git a/types/index.d.ts b/types/index.d.ts index 078714b30..96a09d78a 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -81,6 +81,11 @@ export type AwsLambdaRuntimeManagement = export type AwsLambdaTimeout = number; export type AwsLambdaTracing = ('Active' | 'PassThrough') | boolean; export type AwsLambdaVersioning = boolean; +export type AwsPruneFunctionVersions = + | boolean + | { + number: number; + }; export type AwsHttpApiPayload = '1.0' | '2.0'; export type AwsApiGatewayApiKeys = ( | string @@ -1341,6 +1346,7 @@ export interface AWS { vpc?: AwsLambdaVpcConfig; vpcEndpointIds?: AwsCfArrayInstruction; versionFunctions?: AwsLambdaVersioning; + pruneFunctionVersions?: AwsPruneFunctionVersions; websocket?: { useProviderTags?: boolean; };