Merge pull request #59 from SoManyHs/arm-support

chore: support ARM architecture

Author: SoManyHs

.gitignore

@@ -19,3 +19,7 @@ out/
 .idea
 .idea/*
 .DS_Store
+
+infra/node_modules
+infra/*.js
+infra/*.d.ts

Dockerfile

@@ -1,11 +1,14 @@
 # Use amazonlinux as the base image so that:
 # - we have certificates to make calls to the AWS APIs (/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem)
 # - it provides 'sh' excutable that is required by aws-sdk-go credential_process
+# NOTE: the amazonlinux:2 base image is multi-arch, so docker should be
+# able to detect the correct one to use when the image is run
 FROM amazonlinux:2
 
 COPY ["LICENSE", "NOTICE", "THIRD-PARTY", "/"]
 
-ADD bin/linux-amd64/local-container-endpoints /
+ARG ARCH_DIR
+ADD bin/$ARCH_DIR/local-container-endpoints /
 
 EXPOSE 80
 

Makefile

@@ -15,26 +15,74 @@ ROOT := $(shell pwd)
 
 all: local-build
 
+GO_VERSION := 1.15
 SCRIPT_PATH := $(ROOT)/scripts/:${PATH}
 SOURCES := $(shell find . -name '*.go')
-LOCAL_BINARY := bin/local-container-endpoints
-LINUX_BINARY := bin/linux-amd64/local-container-endpoints
+BINARY_NAME := local-container-endpoints
+IMAGE_REPO_NAME := amazon/amazon-ecs-local-container-endpoints
+LOCAL_BINARY := bin/local/${BINARY_NAME}
+
+# AMD_DIR and ARM_DIR correspond to ARCH_SUFFIX env var set in each CodeBuild
+# project which is used in the image tags.
+AMD_DIR := amd64
+ARM_DIR := arm64
+
+AMD_BINARY := bin/${AMD_DIR}/${BINARY_NAME}
+ARM_BINARY := bin/${ARM_DIR}/${BINARY_NAME}
 VERSION := $(shell cat VERSION)
 AGENT_VERSION_COMPATIBILITY := $(shell cat AGENT_VERSION_COMPATIBILITY)
 TAG := $(VERSION)-agent$(AGENT_VERSION_COMPATIBILITY)-compatible
 
+.PHONY: generate
+generate: $(SOURCES)
+	PATH=$(SCRIPT_PATH) go generate ./...
+
 .PHONY: local-build
 local-build: $(LOCAL_BINARY)
 
+.PHONY: build-local-image
+build-local-image:
+	docker run -v $(shell pwd):/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
+		--workdir=/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
+		--env GOPATH=/usr/src/app \
+		--env ECS_RELEASE=cleanbuild \
+		golang:$(GO_VERSION) make ${LOCAL_BINARY}
+	docker build --build-arg ARCH_DIR=local -t $(IMAGE_REPO_NAME):latest-local .
+
+# Build binaries for each architecture into their own subdirectories
 $(LOCAL_BINARY): $(SOURCES)
 	PATH=${PATH} golint ./local-container-endpoints/...
-	./scripts/build_binary.sh ./bin/
+	./scripts/build_binary.sh ./bin/local
 	@echo "Built local-container-endpoints"
 
-.PHONY: generate
-generate: $(SOURCES)
-	PATH=$(SCRIPT_PATH) go generate ./...
+$(AMD_BINARY): $(SOURCES)
+	@mkdir -p ./bin/$(AMD_DIR)
+	TARGET_GOOS=linux GOARCH=amd64 ./scripts/build_binary.sh ./bin/$(AMD_DIR)
+	@echo "Built local-container-endpoints for linux-amd64"
 
+$(ARM_BINARY): $(SOURCES)
+	@mkdir -p ./bin/$(ARM_DIR)
+	TARGET_GOOS=linux GOARCH=arm64 ./scripts/build_binary.sh ./bin/$(ARM_DIR)
+	@echo "Built local-container-endpoints for linux-arm64"
+
+# Relies on ARCH_SUFFIX environment variable which is set in the build
+# environment (e.g. CodeBuild project). Value will either be amd64 or arm64.
+.PHONY: build-image
+build-image:
+	docker run -v $(shell pwd):/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
+		--workdir=/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
+		--env GOPATH=/usr/src/app \
+		--env ECS_RELEASE=cleanbuild \
+		golang:$(GO_VERSION) make bin/${ARCH_SUFFIX}/${BINARY_NAME}
+	docker build --build-arg ARCH_DIR=$(ARCH_SUFFIX) -t $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX) .
+	docker tag $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX) $(IMAGE_REPO_NAME):$(TAG)-$(ARCH_SUFFIX)
+	docker tag $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX) $(IMAGE_REPO_NAME):$(VERSION)-$(ARCH_SUFFIX)
+
+.PHONY: publish-dockerhub
+publish-dockerhub:
+	docker push $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX)
+	docker push $(IMAGE_REPO_NAME):$(TAG)-$(ARCH_SUFFIX)
+	docker push $(IMAGE_REPO_NAME):$(VERSION)-$(ARCH_SUFFIX)
 
 .PHONY: test
 test:
@@ -44,34 +92,37 @@ test:
 functional-test:
 	go test -mod=vendor -timeout=120s -v -tags functional -cover ./local-container-endpoints/handlers/functional_tests/...
 
-$(LINUX_BINARY): $(SOURCES)
-	@mkdir -p ./bin/linux-amd64
-	TARGET_GOOS=linux GOARCH=amd64 ./scripts/build_binary.sh ./bin/linux-amd64
-	@echo "Built local-container-endpoints for linux"
-
-.PHONY: release
-release:
-	docker run -v $(shell pwd):/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
-		--workdir=/usr/src/app/src/github.com/awslabs/amazon-ecs-local-container-endpoints \
-		--env GOPATH=/usr/src/app \
-		--env ECS_RELEASE=cleanbuild \
-		golang:1.12 make $(LINUX_BINARY)
-	docker build -t amazon/amazon-ecs-local-container-endpoints:latest .
-	docker tag amazon/amazon-ecs-local-container-endpoints:latest amazon/amazon-ecs-local-container-endpoints:$(TAG)
-	docker tag amazon/amazon-ecs-local-container-endpoints:latest amazon/amazon-ecs-local-container-endpoints:$(VERSION)
-
 .PHONY: integ
-integ: release
+integ: build-local-image
 	docker build -t amazon-ecs-local-container-endpoints-integ-test:latest -f ./integ/Dockerfile .
 	docker-compose --file ./integ/docker-compose.yml up --abort-on-container-exit
 
-
-.PHONY: publish
-publish: release
-	docker push amazon/amazon-ecs-local-container-endpoints:latest
-	docker push amazon/amazon-ecs-local-container-endpoints:$(TAG)
-	docker push amazon/amazon-ecs-local-container-endpoints:$(VERSION)
+.PHONY: verify
+verify:
+	docker pull $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX)
+	docker run -d -p 8000:80 -v /var/run:/var/run -v $(HOME)/.aws/:/home/.aws/ -e "ECS_LOCAL_METADATA_PORT=80" -e "HOME=/home" -e "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI=${AWS_CONTAINER_CREDENTIALS_RELATIVE_URI}" --name endpoints $(IMAGE_REPO_NAME):latest-$(ARCH_SUFFIX)
+	curl -s localhost:8000/creds
+	curl -s localhost:8000/v2/stats
+	curl -s localhost:8000/v2/metadata
+	curl -s localhost:8000/v3
+	curl -s localhost:8000/v4
+	docker stop endpoints && docker rm endpoints
+	docker pull $(IMAGE_REPO_NAME):$(TAG)-$(ARCH_SUFFIX)
+	docker run -d -p 8000:80 -v /var/run:/var/run -v $(HOME)/.aws/:/home/.aws/ -e "ECS_LOCAL_METADATA_PORT=80" -e "HOME=/home" -e "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI=${AWS_CONTAINER_CREDENTIALS_RELATIVE_URI}" --name endpoints $(IMAGE_REPO_NAME):$(TAG)-$(ARCH_SUFFIX)
+	curl -s localhost:8000/creds
+	curl -s localhost:8000/v2/stats
+	curl -s localhost:8000/v3
+	curl -s localhost:8000/v4
+	docker stop endpoints && docker rm endpoints
+	docker pull $(IMAGE_REPO_NAME):$(VERSION)-$(ARCH_SUFFIX)
+	docker run -d -p 8000:80 -v /var/run:/var/run -v $(HOME)/.aws/:/home/.aws/ -e "ECS_LOCAL_METADATA_PORT=80" -e "HOME=/home" -e "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI=${AWS_CONTAINER_CREDENTIALS_RELATIVE_URI}" --name endpoints $(IMAGE_REPO_NAME):$(VERSION)-$(ARCH_SUFFIX)
+	curl -s localhost:8000/creds
+	curl -s localhost:8000/v2/stats
+	curl -s localhost:8000/v2/metadata
+	curl -s localhost:8000/v3
+	curl -s localhost:8000/v4
+	docker stop endpoints && docker rm endpoints
 
 .PHONY: clean
 clean:
-	rm bin/local-container-endpoints
+	rm bin/local/local-container-endpoints

buildspec.yml

@@ -0,0 +1,26 @@
+version: 0.2
+
+env:
+  secrets-manager:
+    USERNAME: "com.amazonaws.ec2.madison.dockerhub.amazon-ecs-local-container-endpoints.credentials:username"
+    PASSWORD: "com.amazonaws.ec2.madison.dockerhub.amazon-ecs-local-container-endpoints.credentials:password"
+
+phases:
+  install:
+    commands:
+      - echo '#!/bin/bash' > /usr/local/bin/ok; echo 'if [[ "$CODEBUILD_BUILD_SUCCEEDING" == "0" ]]; then exit 1; else exit 0; fi' >> /usr/local/bin/ok; chmod +x /usr/local/bin/ok
+  pre_build:
+    commands:
+      - echo "Logging into DockerHub..."
+      - docker login -u ${USERNAME} --password ${PASSWORD}
+  build:
+    # build and tag docker image. This will read ARCH_SUFFIX env var set in the
+    # Codebuild project.
+    commands:
+      - echo Build started on `date`
+      - echo Building Docker image...
+      - make build-image
+      - make publish-dockerhub
+  post_build:
+    commands:
+      - ok && echo Build completed on `date`

buildspec_verify.yml

@@ -0,0 +1,22 @@
+version: 0.2
+
+env:
+  secrets-manager:
+    USERNAME: "com.amazonaws.ec2.madison.dockerhub.amazon-ecs-local-container-endpoints.credentials:username"
+    PASSWORD: "com.amazonaws.ec2.madison.dockerhub.amazon-ecs-local-container-endpoints.credentials:password"
+
+phases:
+  install:
+    commands:
+      - echo '#!/bin/bash' > /usr/local/bin/ok; echo 'if [[ "$CODEBUILD_BUILD_SUCCEEDING" == "0" ]]; then exit 1; else exit 0; fi' >> /usr/local/bin/ok; chmod +x /usr/local/bin/ok
+  pre_build:
+    commands:
+      - echo "Logging into DockerHub..."
+      - docker login -u ${USERNAME} --password ${PASSWORD}
+  build:
+    commands:
+      - echo "Verifying Docker images..."
+      - make verify
+  post_build:
+    commands:
+      - ok && echo Build completed on `date`

docs/configuration.md

@@ -15,7 +15,7 @@ For example, on an Ubuntu machine, you can mount your machine's certificates fil
 
 ### Custom IAM and STS Endpoints
 
-Local Endpoionts can be configured to use custom IAM and STS endpoints. Simply define the `IAM_ENDPOINT` and `STS_ENDPOINT` environment variables in the Local Endpoints container.
+Local Endpoints can be configured to use custom IAM and STS endpoints. Simply define the `IAM_ENDPOINT` and `STS_ENDPOINT` environment variables in the Local Endpoints container.
 
 This may be useful in scenarios where your application container is configured to obtain credentials from ECS (see [Vend Credentials to Containers](features.md#vend-credentials-to-containers)), but you do not want to provide Local Endpoints with AWS credentials. Providing an IAM and STS simulator and configuring the Local Endpoints container with custom IAM and STS endpoints enables testing without an AWS account.
 

docs/setup-networking.md

@@ -1,9 +1,10 @@
 ## Setting Up Networking
 
-ECS Local Container Endpoints supports 3 endpoints:
+ECS Local Container Endpoints supports 4 endpoints:
 * The [ECS Task IAM Roles endpoint](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html)
 * The [Task Metadata V2 Endpoint](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v2.html)
 * The [Task Metadata V3 Endpoint](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v3.html)
+* The [Task Metadata V4 Endpoint](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html)
 
 The Task Metadata V2 and Credentials endpoints require the Local Endpoints container to be able to receive requests made to the special IP Address, `169.254.170.2`.
 
@@ -36,5 +37,5 @@ docker run -d -p 51679:51679 \
 -v $HOME/.aws/:/home/.aws/ \
 -e "ECS_LOCAL_METADATA_PORT=51679" \
 --name ecs-local-endpoints \
-amazon/amazon-ecs-local-container-endpoints:latest
-```
+amazon/amazon-ecs-local-container-endpoints:latest-amd64
+```

infra/README.md

@@ -0,0 +1,33 @@
+# Continuous delivery pipelines
+
+This package uses the [AWS Cloud Development Kit (AWS)](https://github.com/awslabs/aws-cdk) to model AWS CodePipeline pipelines and to provision them with AWS CloudFormation.
+
+* pipeline.ts: Builds and publishes the base Docker image for amazon/amazon-ecs-local-container-endpoints.
+
+This creates as CodePipeline pipeline which consists of a souce stage that uses
+a GitHub webhook, and build stages that uses AWS CodeBuild to build, publish
+and verify Docker images for both amd64 and arm64 architectures to DockerHub.
+
+## GitHub Access Token
+The official pipeilne uses a team account (ecs-local-container-endpoints+release@amazon.com).
+
+Create a GitHub [personal access token](https://github.com/settings/tokens) with access to your fork of the repo, including "admin:repo_hook" and "repo" permissions.  Then store the token in Secrets Manager:
+
+```
+aws secretsmanager create-secret --region us-west-2 --name EcsDevXGitHubToken --secret-string <my-github-personal-access-token>
+```
+
+## Deploy
+
+To deploy this pipeline, install the AWS CDK CLI: `npm i -g aws-cdk`
+
+Install and build everything: `npm install && npm run build`
+
+Then deploy the pipeline stacks:
+
+```
+cdk deploy --app 'node pipeline.js'
+
+```
+
+See the pipelines in the CodePipeline console.