From 1a9b59454ac739283ca6c6267692ec40b6490d36 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 11:40:36 -0800 Subject: [PATCH 01/15] Update aws-semaphore-agent-stack.js --- lib/aws-semaphore-agent-stack.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index d8b90b2..6eb52cd 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -91,6 +91,18 @@ class AwsSemaphoreAgentStack extends Stack { let policy = new Policy(this, 'instanceProfilePolicy', { policyName: `${this.stackName}-instance-profile-policy`, statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:CreateReplaceRootVolumeTask" + ], + resources: [`arn:aws:ec2:*:${this.account}:instance/*`], + conditions: [ + StringLike: { + "aws:userid": "*:${ec2:InstanceID}" + } + ] + }), new PolicyStatement({ effect: Effect.ALLOW, actions: [ From 9eb2a5fa77917f93eb568990d9b02cb1c4eb2142 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 11:43:38 -0800 Subject: [PATCH 02/15] Update terminate-instance.sh --- packer/macos/files/terminate-instance.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index b1e2ce1..18cd696 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -11,6 +11,7 @@ fi token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" --fail --silent --show-error --location "http://169.254.169.254/latest/api/token") instance_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/instance-id") +ami_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/ami-id") region=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/placement/region") # We unset all AWS related variables to make sure the instance profile is always used. @@ -27,8 +28,9 @@ if [[ $SEMAPHORE_AGENT_SHUTDOWN_REASON == "IDLE" ]]; then --instance-id "$instance_id" \ --should-decrement-desired-capacity else - aws autoscaling terminate-instance-in-auto-scaling-group \ + aws ec2 create-replace-root-volume-task \ --region "$region" \ --instance-id "$instance_id" \ - --no-should-decrement-desired-capacity + --image-id "$ami_id" \ + –-delete-replaced-root-volume fi From 876697d1fa7fa87a3e5b091f2db361cc8d01911f Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 11:51:22 -0800 Subject: [PATCH 03/15] Update terminate-instance.sh --- packer/macos/files/terminate-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index 18cd696..615a13d 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -28,7 +28,7 @@ if [[ $SEMAPHORE_AGENT_SHUTDOWN_REASON == "IDLE" ]]; then --instance-id "$instance_id" \ --should-decrement-desired-capacity else - aws ec2 create-replace-root-volume-task \ + aws ec2 create-replace-root-volume-task \ --region "$region" \ --instance-id "$instance_id" \ --image-id "$ami_id" \ From f27c1a8dd42fa869c189899ee14cb396eed2a2e7 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:40:50 -0800 Subject: [PATCH 04/15] Update terminate-instance.sh --- packer/macos/files/terminate-instance.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index 615a13d..8eedda8 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -11,7 +11,6 @@ fi token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" --fail --silent --show-error --location "http://169.254.169.254/latest/api/token") instance_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/instance-id") -ami_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/ami-id") region=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/placement/region") # We unset all AWS related variables to make sure the instance profile is always used. @@ -28,6 +27,17 @@ if [[ $SEMAPHORE_AGENT_SHUTDOWN_REASON == "IDLE" ]]; then --instance-id "$instance_id" \ --should-decrement-desired-capacity else + # Enter standby LifecycleState because the EC2 health check will fail while we're replacing the root volume + # We also decrement desired capacity so the ASG doesn't create a new replacement instance in the meantime + # The instance will exit standby automatically in start-agent.sh after reboot + asg_name=$(aws autoscaling describe-auto-scaling-instances --region "$region" --instance-ids "$instance_id" --output text --query "AutoScalingInstances[0].AutoScalingGroupName") + aws autoscaling enter-standby \ + --region "$region" \ + --instance-ids "$instance_id" \ + --auto-scaling-group-name "$asg_name" \ + --should-decrement-desired-capacity + # https://aws.amazon.com/blogs/compute/new-reset-amazon-ec2-mac-instances-to-a-known-state-using-replace-root-volume-capability/ + ami_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/ami-id") aws ec2 create-replace-root-volume-task \ --region "$region" \ --instance-id "$instance_id" \ From 769f37e881f76fb28063d0fe37b7716620865a55 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:45:56 -0800 Subject: [PATCH 05/15] Update start-agent.sh --- packer/macos/files/start-agent.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packer/macos/files/start-agent.sh b/packer/macos/files/start-agent.sh index 054aabb..a3b77a7 100755 --- a/packer/macos/files/start-agent.sh +++ b/packer/macos/files/start-agent.sh @@ -204,6 +204,20 @@ region=$(curl \ --show-error \ --location "http://169.254.169.254/latest/meta-data/placement/region" ) +instance_id=$(curl \ + -H "X-aws-ec2-metadata-token: $token" \ + --fail \ + --silent \ + --show-error \ + --location "http://169.254.169.254/latest/meta-data/placement/instance-id" +) + +# If the instance is in "standby" mode (after root volume replacement) we want to exit +asg_name=$(aws autoscaling describe-auto-scaling-instances --region "$region" --instance-ids "$instance_id" --output text --query 'AutoScalingInstances[?LifecycleState==`Standby`].AutoScalingGroupName | [0]') +if [[ -n "$asg_name" ]]; then + echo "Exiting standby mode for '$instance_id' in '$asg_name'..." + aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" +fi # The parameters required for the agent configuration are stored in an SSM parameter. # We need to fetch them before proceeding with anything else. From f692ea2510e66efa6f47efeef174277fd8525079 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:47:26 -0800 Subject: [PATCH 06/15] Update aws-semaphore-agent-stack.js --- lib/aws-semaphore-agent-stack.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index 6eb52cd..338911d 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -91,6 +91,13 @@ class AwsSemaphoreAgentStack extends Stack { let policy = new Policy(this, 'instanceProfilePolicy', { policyName: `${this.stackName}-instance-profile-policy`, statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:DescribeAutoScalingInstances" + ], + resources: [`*`], + }), new PolicyStatement({ effect: Effect.ALLOW, actions: [ From 390d531a49f9e57ca51e3006fcd8a15b4afd48fe Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:48:32 -0800 Subject: [PATCH 07/15] Update aws-semaphore-agent-stack.js --- lib/aws-semaphore-agent-stack.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index 338911d..606c059 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -113,6 +113,8 @@ class AwsSemaphoreAgentStack extends Stack { new PolicyStatement({ effect: Effect.ALLOW, actions: [ + "autoscaling:EnterStandby", + "autoscaling:ExitStandby", "autoscaling:SetInstanceHealth", "autoscaling:TerminateInstanceInAutoScalingGroup" ], From a74fccbaadf9bcc2bd0ccc74e712e92179a2fa57 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:57:16 -0800 Subject: [PATCH 08/15] Update aws-semaphore-agent-stack.js --- lib/aws-semaphore-agent-stack.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index 606c059..acfc915 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -344,6 +344,7 @@ class AwsSemaphoreAgentStack extends Stack { blockDevices: this.createBlockDevices(), securityGroup: securityGroup, instanceInitiatedShutdownBehavior: InstanceInitiatedShutdownBehavior.TERMINATE, + instanceMetadataTags: true, keyName: keyName != "" ? keyName : undefined }); From 8ed30bd41f3b57fb3ebdbe5c0b3abb6591738a10 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:58:45 -0800 Subject: [PATCH 09/15] Update start-agent.sh --- packer/macos/files/start-agent.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packer/macos/files/start-agent.sh b/packer/macos/files/start-agent.sh index a3b77a7..9b98150 100755 --- a/packer/macos/files/start-agent.sh +++ b/packer/macos/files/start-agent.sh @@ -211,13 +211,16 @@ instance_id=$(curl \ --show-error \ --location "http://169.254.169.254/latest/meta-data/placement/instance-id" ) +asg_name=$(curl \ + -H "X-aws-ec2-metadata-token: $token" \ + --fail \ + --silent \ + --show-error \ + --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName" +) # If the instance is in "standby" mode (after root volume replacement) we want to exit -asg_name=$(aws autoscaling describe-auto-scaling-instances --region "$region" --instance-ids "$instance_id" --output text --query 'AutoScalingInstances[?LifecycleState==`Standby`].AutoScalingGroupName | [0]') -if [[ -n "$asg_name" ]]; then - echo "Exiting standby mode for '$instance_id' in '$asg_name'..." - aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" -fi +aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" || true # The parameters required for the agent configuration are stored in an SSM parameter. # We need to fetch them before proceeding with anything else. From 979318e18afa33f928d55bfabc4d2a6db98d0332 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:59:25 -0800 Subject: [PATCH 10/15] Update terminate-instance.sh --- packer/macos/files/terminate-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index 8eedda8..f588b5d 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -30,7 +30,7 @@ else # Enter standby LifecycleState because the EC2 health check will fail while we're replacing the root volume # We also decrement desired capacity so the ASG doesn't create a new replacement instance in the meantime # The instance will exit standby automatically in start-agent.sh after reboot - asg_name=$(aws autoscaling describe-auto-scaling-instances --region "$region" --instance-ids "$instance_id" --output text --query "AutoScalingInstances[0].AutoScalingGroupName") + asg_name=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName") aws autoscaling enter-standby \ --region "$region" \ --instance-ids "$instance_id" \ From 8efc507db8062cca720d9634e053e4968aee6ee0 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:59:39 -0800 Subject: [PATCH 11/15] Update aws-semaphore-agent-stack.js --- lib/aws-semaphore-agent-stack.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index acfc915..0b0e42c 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -91,13 +91,6 @@ class AwsSemaphoreAgentStack extends Stack { let policy = new Policy(this, 'instanceProfilePolicy', { policyName: `${this.stackName}-instance-profile-policy`, statements: [ - new PolicyStatement({ - effect: Effect.ALLOW, - actions: [ - "ec2:DescribeAutoScalingInstances" - ], - resources: [`*`], - }), new PolicyStatement({ effect: Effect.ALLOW, actions: [ From bdf07332be73dd6e2f6f4abe8f77dacf610d1a0b Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:07:15 -0800 Subject: [PATCH 12/15] Update start-agent.sh --- packer/macos/files/start-agent.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packer/macos/files/start-agent.sh b/packer/macos/files/start-agent.sh index 9b98150..ecccf6b 100755 --- a/packer/macos/files/start-agent.sh +++ b/packer/macos/files/start-agent.sh @@ -209,7 +209,7 @@ instance_id=$(curl \ --fail \ --silent \ --show-error \ - --location "http://169.254.169.254/latest/meta-data/placement/instance-id" + --location "http://169.254.169.254/latest/meta-data/instance-id" ) asg_name=$(curl \ -H "X-aws-ec2-metadata-token: $token" \ @@ -219,7 +219,8 @@ asg_name=$(curl \ --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName" ) -# If the instance is in "standby" mode (after root volume replacement) we want to exit +# If the instance is in the "Standby" LifecycleState (after root volume replacement) we want to exit so the health-check runs +# If it's the first boot the instance may already be in "InService" LifecycleState so we ignore any errors from this command aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" || true # The parameters required for the agent configuration are stored in an SSM parameter. From 6f537290ad2629fb4947dcbb68abf694c80ad2b2 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Thu, 2 Jan 2025 11:21:32 -0300 Subject: [PATCH 13/15] =?UTF-8?q?fix=20typo=20in=20=E2=80=93-=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packer/macos/files/terminate-instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index f588b5d..2548a63 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -42,5 +42,5 @@ else --region "$region" \ --instance-id "$instance_id" \ --image-id "$ami_id" \ - –-delete-replaced-root-volume + --delete-replaced-root-volume fi From 111a0bfc0cc78d59ee17b1d200f511ea53c0ecdd Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 3 Jan 2025 08:04:19 -0300 Subject: [PATCH 14/15] fix condition and use macos 13.7 --- lib/aws-semaphore-agent-stack.js | 11 +++++++---- packer/macos/macos.pkr.hcl | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index 0b0e42c..23f305a 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -96,12 +96,15 @@ class AwsSemaphoreAgentStack extends Stack { actions: [ "ec2:CreateReplaceRootVolumeTask" ], - resources: [`arn:aws:ec2:*:${this.account}:instance/*`], - conditions: [ - StringLike: { + resources: [ + `arn:aws:ec2:*:${this.account}:instance/*`, + `arn:aws:ec2:${this.region}::image/*` + ], + conditions: { + "StringLike": { "aws:userid": "*:${ec2:InstanceID}" } - ] + } }), new PolicyStatement({ effect: Effect.ALLOW, diff --git a/packer/macos/macos.pkr.hcl b/packer/macos/macos.pkr.hcl index 112fd1b..f0c9bf3 100644 --- a/packer/macos/macos.pkr.hcl +++ b/packer/macos/macos.pkr.hcl @@ -76,7 +76,7 @@ source "amazon-ebs" "macos" { owners = ["amazon"] filters = { - name = "amzn-ec2-macos-12.6*" + name = "amzn-ec2-macos-13.7*" architecture = "${var.arch}*" root-device-type = "ebs" virtualization-type = "hvm" From 1da19d9517be3a905aefcf1620082d88de714680 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:57:37 -0700 Subject: [PATCH 15/15] fix: ec2:CreateReplaceRootVolumeTask --- lib/aws-semaphore-agent-stack.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index 23f305a..31e6634 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -97,8 +97,7 @@ class AwsSemaphoreAgentStack extends Stack { "ec2:CreateReplaceRootVolumeTask" ], resources: [ - `arn:aws:ec2:*:${this.account}:instance/*`, - `arn:aws:ec2:${this.region}::image/*` + `arn:aws:ec2:${this.region}:${this.account}:instance/*`, ], conditions: { "StringLike": { @@ -106,6 +105,17 @@ class AwsSemaphoreAgentStack extends Stack { } } }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:CreateReplaceRootVolumeTask" + ], + resources: [ + `arn:aws:ec2:${this.region}:${this.account}:replace-root-volume-task/*`, + `arn:aws:ec2:${this.region}:${this.account}:volume/*`, + `arn:aws:ec2:${this.region}:${this.account}:image/*`, + ] + }), new PolicyStatement({ effect: Effect.ALLOW, actions: [