From 3d8de1d36155ef38a22939e7f34ce9ef8ef9d583 Mon Sep 17 00:00:00 2001 From: observerr411 Date: Mon, 29 Jun 2026 11:29:57 +0100 Subject: [PATCH] fix(infra): harden RDS backups, Redis Multi-AZ, bootstrap script, and ECS deploy strategy Closes #956 Closes #957 Closes #958 Closes #959 --- docs/deployment.md | 81 ++++++++++++ infrastructure/README.md | 8 +- infrastructure/terraform/bootstrap.sh | 118 ++++++++++++------ infrastructure/terraform/modules/ecs/main.tf | 10 ++ infrastructure/terraform/modules/rds/main.tf | 11 +- .../terraform/modules/rds/rds_backup_test.go | 54 ++++++++ .../terraform/modules/redis/main.tf | 43 ++++--- 7 files changed, 268 insertions(+), 57 deletions(-) create mode 100644 docs/deployment.md mode change 100644 => 100755 infrastructure/terraform/bootstrap.sh create mode 100644 infrastructure/terraform/modules/rds/rds_backup_test.go diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..86a56db7 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,81 @@ +# Deployment Guide + +## ECS Rolling Deploy Strategy + +PredictIQ uses ECS Fargate with a **zero-downtime rolling deploy** strategy. + +### Configuration + +| Setting | Value | Rationale | +|---|---|---| +| `deployment_minimum_healthy_percent` | 100 | Never reduce capacity below 100 % during a deploy | +| `deployment_maximum_percent` | 200 | Allow new tasks to start before old tasks are drained | +| `deployment_circuit_breaker.enable` | true | Automatically detect a failed deploy | +| `deployment_circuit_breaker.rollback` | true | Automatically roll back to the previous task definition on failure | + +### Deploy Sequence + +1. **Run database migrations** before deploying new tasks. + Migrations must be backward-compatible with the currently running task version so both versions can co-exist during the rollover window. + +2. **ECS starts new tasks** (up to `maximum_percent` total capacity) using the new task definition. + +3. **ALB health checks** confirm the new tasks are healthy (`/health` returns 200). + +4. **ECS drains old tasks** — the load balancer stops routing traffic to them and waits for in-flight requests to complete before terminating. + +5. **Circuit breaker monitors** the deploy. If the new tasks fail health checks repeatedly, ECS automatically rolls back to the previous stable task definition and raises a `SERVICE_DEPLOYMENT_FAILED` CloudWatch event. + +### Deployment Steps (Manual) + +```bash +# 1. Build and push the new image +docker build -t predictiq-api:$VERSION . +docker push $ECR_REPO:$VERSION + +# 2. Run migrations against the target environment +DATABASE_URL=$PROD_DATABASE_URL ./services/api/scripts/run_migrations.sh + +# 3. Update the ECS service (GitHub Actions does this automatically on merge to main) +aws ecs update-service \ + --cluster predictiq-prod \ + --service predictiq-prod-api \ + --task-definition predictiq-prod-api:$NEW_REVISION \ + --region us-east-1 + +# 4. Wait for the deploy to stabilise +aws ecs wait services-stable \ + --cluster predictiq-prod \ + --services predictiq-prod-api \ + --region us-east-1 +``` + +### Rollback (Manual) + +If automatic rollback did not trigger: + +```bash +# Find the last known-good task definition revision +aws ecs describe-services \ + --cluster predictiq-prod \ + --services predictiq-prod-api \ + --query 'services[0].deployments' + +# Force rollback to a specific revision +aws ecs update-service \ + --cluster predictiq-prod \ + --service predictiq-prod-api \ + --task-definition predictiq-prod-api:$PREVIOUS_REVISION \ + --region us-east-1 +``` + +### AWS CodeDeploy Blue-Green — Assessment + +A full blue-green deployment (via AWS CodeDeploy) would eliminate even the brief overlap window of rolling deploys by switching traffic at the ALB listener level in one atomic step. + +**Warranted when:** +- Migrations are not backward-compatible and cannot be made so. +- The team needs a well-defined, instant traffic cutover with a single-click rollback. +- Compliance requirements mandate zero request failures during deploy. + +**Current recommendation:** The rolling strategy with `minimum_healthy_percent = 100` already delivers zero-downtime deploys for backward-compatible migrations. CodeDeploy blue-green adds significant operational complexity (separate target groups, CodeDeploy application/deployment group resources, lifecycle hook Lambdas). Given that PredictIQ migrations are currently written to be additive and backward-compatible, the rolling strategy is sufficient. Revisit this decision if a migration arises that requires a hard cutover. diff --git a/infrastructure/README.md b/infrastructure/README.md index 49e4ff44..82b5d49c 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -157,9 +157,11 @@ See `environments/README.md` for detailed environment management instructions. - Encryption at rest ### Redis Module -- ElastiCache cluster -- Automatic failover -- Parameter group configuration +- ElastiCache **replication group** (Multi-AZ) +- `automatic_failover_enabled = true` and `multi_az_enabled = true` by default +- Minimum 2 cache clusters to support failover +- **Failover RTO**: AWS promotes a read replica to primary in approximately **60–120 seconds**. During this window, write operations will fail; read-only operations served by replicas remain available. Applications should implement retry logic with exponential backoff (recommended: 3 retries, starting at 200 ms) to handle the failover window gracefully. +- At-rest and in-transit encryption enabled - Subnet group for VPC placement ### ECS Module diff --git a/infrastructure/terraform/bootstrap.sh b/infrastructure/terraform/bootstrap.sh old mode 100644 new mode 100755 index df6e6167..e5cb6e6d --- a/infrastructure/terraform/bootstrap.sh +++ b/infrastructure/terraform/bootstrap.sh @@ -1,8 +1,9 @@ #!/bin/bash -set -e +set -euo pipefail # Bootstrap script to create S3 bucket and DynamoDB table for Terraform state management # Usage: ./bootstrap.sh +# Idempotent: safe to run multiple times. AWS_REGION=${1:-us-east-1} ENVIRONMENT=${2:-dev} @@ -11,66 +12,111 @@ LOCK_TABLE="terraform-locks-${ENVIRONMENT}" echo "Bootstrapping Terraform state backend for environment: $ENVIRONMENT in region: $AWS_REGION" -# Create S3 bucket -echo "Creating S3 bucket: $BUCKET_NAME" -aws s3api create-bucket \ - --bucket "$BUCKET_NAME" \ - --region "$AWS_REGION" \ - $([ "$AWS_REGION" != "us-east-1" ] && echo "--create-bucket-configuration LocationConstraint=$AWS_REGION") \ - 2>/dev/null || echo "Bucket already exists or error occurred" +# --------------------------------------------------------------------------- +# S3 bucket +# --------------------------------------------------------------------------- +echo "Checking S3 bucket: $BUCKET_NAME" +if aws s3api head-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" 2>/dev/null; then + echo " → Bucket already exists, skipping creation." +else + echo " → Creating S3 bucket: $BUCKET_NAME" + if [ "$AWS_REGION" = "us-east-1" ]; then + aws s3api create-bucket \ + --bucket "$BUCKET_NAME" \ + --region "$AWS_REGION" + else + aws s3api create-bucket \ + --bucket "$BUCKET_NAME" \ + --region "$AWS_REGION" \ + --create-bucket-configuration "LocationConstraint=$AWS_REGION" + fi + + # Verify bucket was created + aws s3api head-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" + echo " → Bucket created and verified." +fi -# Enable versioning echo "Enabling versioning on S3 bucket" aws s3api put-bucket-versioning \ --bucket "$BUCKET_NAME" \ --versioning-configuration Status=Enabled \ --region "$AWS_REGION" +VERSIONING=$(aws s3api get-bucket-versioning --bucket "$BUCKET_NAME" --region "$AWS_REGION" --query 'Status' --output text) +[ "$VERSIONING" = "Enabled" ] || { echo "ERROR: Versioning not enabled on $BUCKET_NAME"; exit 1; } -# Enable encryption echo "Enabling server-side encryption on S3 bucket" aws s3api put-bucket-encryption \ --bucket "$BUCKET_NAME" \ --server-side-encryption-configuration '{ - "Rules": [ - { - "ApplyServerSideEncryptionByDefault": { - "SSEAlgorithm": "AES256" - } - } - ] + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}] }' \ --region "$AWS_REGION" +aws s3api get-bucket-encryption --bucket "$BUCKET_NAME" --region "$AWS_REGION" > /dev/null +echo " → Encryption verified." -# Block public access echo "Blocking public access to S3 bucket" aws s3api put-public-access-block \ --bucket "$BUCKET_NAME" \ --public-access-block-configuration \ "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" \ --region "$AWS_REGION" +BLOCK=$(aws s3api get-public-access-block --bucket "$BUCKET_NAME" --region "$AWS_REGION" \ + --query 'PublicAccessBlockConfiguration.BlockPublicAcls' --output text) +[ "$BLOCK" = "True" ] || { echo "ERROR: Public access block not enabled on $BUCKET_NAME"; exit 1; } -# Create DynamoDB table for state locking -echo "Creating DynamoDB table: $LOCK_TABLE" -aws dynamodb create-table \ - --table-name "$LOCK_TABLE" \ - --attribute-definitions AttributeName=LockID,AttributeType=S \ - --key-schema AttributeName=LockID,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST \ - --region "$AWS_REGION" \ - 2>/dev/null || echo "Table already exists or error occurred" +# --------------------------------------------------------------------------- +# DynamoDB lock table +# --------------------------------------------------------------------------- +echo "Checking DynamoDB table: $LOCK_TABLE" +if aws dynamodb describe-table --table-name "$LOCK_TABLE" --region "$AWS_REGION" > /dev/null 2>&1; then + echo " → Table already exists, skipping creation." +else + echo " → Creating DynamoDB table: $LOCK_TABLE" + aws dynamodb create-table \ + --table-name "$LOCK_TABLE" \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region "$AWS_REGION" + + echo " → Waiting for table to become active..." + aws dynamodb wait table-exists --table-name "$LOCK_TABLE" --region "$AWS_REGION" + + # Verify + STATUS=$(aws dynamodb describe-table --table-name "$LOCK_TABLE" --region "$AWS_REGION" \ + --query 'Table.TableStatus' --output text) + [ "$STATUS" = "ACTIVE" ] || { echo "ERROR: DynamoDB table $LOCK_TABLE is not ACTIVE (status: $STATUS)"; exit 1; } + echo " → Table created and verified." +fi -# Enable point-in-time recovery echo "Enabling point-in-time recovery on DynamoDB table" aws dynamodb update-continuous-backups \ --table-name "$LOCK_TABLE" \ --point-in-time-recovery-specification PointInTimeRecoveryEnabled=true \ - --region "$AWS_REGION" \ - 2>/dev/null || echo "PITR already enabled or error occurred" + --region "$AWS_REGION" > /dev/null +PITR=$(aws dynamodb describe-continuous-backups --table-name "$LOCK_TABLE" --region "$AWS_REGION" \ + --query 'ContinuousBackupsDescription.PointInTimeRecoveryDescription.PointInTimeRecoveryStatus' --output text) +[ "$PITR" = "ENABLED" ] || { echo "ERROR: PITR not enabled on $LOCK_TABLE"; exit 1; } -echo "Bootstrap complete!" -echo "S3 Bucket: $BUCKET_NAME" -echo "DynamoDB Table: $LOCK_TABLE" +# --------------------------------------------------------------------------- +# Success +# --------------------------------------------------------------------------- +echo "" +echo "✅ Bootstrap complete!" +echo " S3 Bucket: $BUCKET_NAME" +echo " DynamoDB Table: $LOCK_TABLE" +echo " Region: $AWS_REGION" +echo "" +echo "Paste the following backend configuration into your main.tf or backend-config.hcl:" +echo "" +echo " terraform {" +echo " backend \"s3\" {" +echo " bucket = \"$BUCKET_NAME\"" +echo " key = \"${ENVIRONMENT}/terraform.tfstate\"" +echo " region = \"$AWS_REGION\"" +echo " dynamodb_table = \"$LOCK_TABLE\"" +echo " encrypt = true" +echo " }" +echo " }" echo "" -echo "Next steps:" -echo "1. Update infrastructure/terraform/backend-config.hcl with the bucket and table names" -echo "2. Run: terraform init -backend-config=backend-config.hcl" +echo "Then run: terraform init -backend-config=backend-config.hcl" diff --git a/infrastructure/terraform/modules/ecs/main.tf b/infrastructure/terraform/modules/ecs/main.tf index da4db3fb..d0ed7331 100644 --- a/infrastructure/terraform/modules/ecs/main.tf +++ b/infrastructure/terraform/modules/ecs/main.tf @@ -253,6 +253,16 @@ resource "aws_ecs_service" "api" { desired_count = var.api_desired_count launch_type = "FARGATE" + # Zero-downtime rolling deploy: always keep 100 % capacity during updates, + # allow up to 200 % so new tasks start before old ones are drained. + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + + deployment_circuit_breaker { + enable = true + rollback = true + } + network_configuration { subnets = var.private_subnet_ids security_groups = [aws_security_group.ecs_tasks.id] diff --git a/infrastructure/terraform/modules/rds/main.tf b/infrastructure/terraform/modules/rds/main.tf index d94e20df..a7d4d4fd 100644 --- a/infrastructure/terraform/modules/rds/main.tf +++ b/infrastructure/terraform/modules/rds/main.tf @@ -33,7 +33,13 @@ variable "allocated_storage" { } variable "backup_retention" { - type = number + type = number + default = 7 +} + +variable "deletion_protection" { + type = bool + default = false } locals { @@ -99,7 +105,8 @@ resource "aws_db_instance" "main" { backup_retention_period = var.backup_retention backup_window = "03:00-04:00" maintenance_window = "mon:04:00-mon:05:00" - + deletion_protection = var.deletion_protection + multi_az = var.environment == "prod" ? true : false publicly_accessible = false skip_final_snapshot = var.environment != "prod" diff --git a/infrastructure/terraform/modules/rds/rds_backup_test.go b/infrastructure/terraform/modules/rds/rds_backup_test.go new file mode 100644 index 00000000..36c4ea25 --- /dev/null +++ b/infrastructure/terraform/modules/rds/rds_backup_test.go @@ -0,0 +1,54 @@ +package rds_test + +import ( + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +// TestRDSBackupConfiguration verifies backup_retention_period, backup_window, +// and deletion_protection are explicitly set in the RDS module. +func TestRDSBackupConfiguration(t *testing.T) { + t.Parallel() + + opts := &terraform.Options{ + TerraformDir: ".", + Vars: map[string]interface{}{ + "environment": "prod", + "vpc_id": "vpc-00000000", + "private_subnet_ids": []string{"subnet-00000001", "subnet-00000002"}, + "db_name": "predictiq", + "db_username": "admin", + "db_password": "testpassword123", + "db_instance_class": "db.t3.micro", + "allocated_storage": 20, + "backup_retention": 7, + "deletion_protection": true, + }, + // Plan only — no real AWS resources are created + PlanFilePath: "/tmp/rds-test.tfplan", + } + + plan := terraform.InitAndPlanAndShowWithStruct(t, opts) + + rds := plan.ResourcePlannedValuesMap["aws_db_instance.main"] + assert.NotNil(t, rds, "aws_db_instance.main must be planned") + + values := rds.AttributeValues + + // backup_retention_period must be >= 7 + retention, ok := values["backup_retention_period"].(float64) + assert.True(t, ok, "backup_retention_period must be a number") + assert.GreaterOrEqual(t, int(retention), 7, "backup_retention_period must be at least 7 days") + + // backup_window must be set + backupWindow, ok := values["backup_window"].(string) + assert.True(t, ok, "backup_window must be a string") + assert.NotEmpty(t, backupWindow, "backup_window must not be empty") + + // deletion_protection must be true for prod + deletionProtection, ok := values["deletion_protection"].(bool) + assert.True(t, ok, "deletion_protection must be a boolean") + assert.True(t, deletionProtection, "deletion_protection must be true for production") +} diff --git a/infrastructure/terraform/modules/redis/main.tf b/infrastructure/terraform/modules/redis/main.tf index b8518c71..a41911d6 100644 --- a/infrastructure/terraform/modules/redis/main.tf +++ b/infrastructure/terraform/modules/redis/main.tf @@ -14,20 +14,27 @@ variable "node_type" { type = string } -variable "num_cache_nodes" { - type = number +variable "num_cache_clusters" { + type = number + default = 2 } variable "engine_version" { type = string } +variable "redis_multi_az_enabled" { + type = bool + default = true + description = "Enable Multi-AZ automatic failover for the Redis replication group." +} + locals { common_tags = { - Project = "predictiq" + Project = "predictiq" Environment = var.environment - Owner = "infrastructure-team" - ManagedBy = "terraform" + Owner = "infrastructure-team" + ManagedBy = "terraform" } } @@ -69,23 +76,27 @@ resource "aws_security_group" "redis" { ) } -resource "aws_elasticache_cluster" "main" { - cluster_id = "predictiq-${var.environment}" +resource "aws_elasticache_replication_group" "main" { + replication_group_id = "predictiq-${var.environment}" + description = "PredictIQ Redis replication group (${var.environment})" + engine = "redis" engine_version = var.engine_version node_type = var.node_type - num_cache_nodes = var.num_cache_nodes + num_cache_clusters = var.num_cache_clusters >= 2 ? var.num_cache_clusters : 2 parameter_group_name = "default.redis7" port = 6379 - subnet_group_name = aws_elasticache_subnet_group.main.name - security_group_ids = [aws_security_group.redis.id] - - automatic_failover_enabled = var.num_cache_nodes > 1 ? true : false + + subnet_group_name = aws_elasticache_subnet_group.main.name + security_group_ids = [aws_security_group.redis.id] + + automatic_failover_enabled = var.redis_multi_az_enabled + multi_az_enabled = var.redis_multi_az_enabled + at_rest_encryption_enabled = true transit_encryption_enabled = true - + maintenance_window = "mon:03:00-mon:04:00" - notification_topic_arn = null tags = merge( local.common_tags, @@ -96,9 +107,9 @@ resource "aws_elasticache_cluster" "main" { } output "endpoint" { - value = aws_elasticache_cluster.main.cache_nodes[0].address + value = aws_elasticache_replication_group.main.primary_endpoint_address } output "redis_url" { - value = "redis://${aws_elasticache_cluster.main.cache_nodes[0].address}:6379" + value = "redis://${aws_elasticache_replication_group.main.primary_endpoint_address}:6379" }