Update linux arm building job and rename script to upload darwin arm binaries. Closes #221 (#222)

Author: pskrbasu

.github/workflows/buildimage.yml

@@ -265,93 +265,84 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
 
-    - name: Build and upload binary
-      uses: uraimo/run-on-arch-action@v2
+    - name: Configure AWS Credentials
+      run: |
+        mkdir ~/.aws
+        echo -e "[default]\naws_access_key_id = ${{ secrets.AWS_ACCESS_KEY_ID }}\naws_secret_access_key = ${{ secrets.AWS_SECRET_ACCESS_KEY }}\nregion = ap-south-1" > ~/.aws/credentials
+
+    - name: SSH into the system and build binary
       env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        arch: aarch64
-        distro: ubuntu_latest
-        # Create an artifacts directory
-        setup: |
-          mkdir -p "${PWD}/artifacts"
-        # Mount the artifacts directory as /artifacts in the container
-        dockerRunArgs: |
-          --volume "${PWD}/artifacts:/artifacts"
-        env: |
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          echo "update apt and install necessary packages"
-          apt-get update -y
-          apt-get -y install sudo git curl tar gzip file postgresql-server-dev-14 build-essential make
-
-          echo "install go"
-          curl -LO https://go.dev/dl/go1.18beta1.linux-arm64.tar.gz
-          tar -C /usr/local -xzf go1.18beta1.linux-arm64.tar.gz
-          export GOROOT=/usr/local/go
-          export GOPATH=$HOME/go
-          export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
-          go version
-          
-          echo "find stuff and set env"
-          which pg_config
-          pg_config --version
-          export PATH=$(pg_config --bindir):$PATH
-          export PGXS=$(pg_config --pgxs)
-          export SERVER_LIB=$(pg_config --includedir)/server
-          export INTERNAL_LIB=$(pg_config --includedir)/internal
-          export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g"
-          export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g"
-          export LDFLAGS=$(pg_config --ldflags)
-          export PG_LDFLAGS=$(pg_config --ldflags)
-
-          echo "clone repo"
-          git clone https://github.com/turbot/steampipe-postgres-fdw.git
-          git config --global --add safe.directory /home/runner/work/steampipe-postgres-fdw/steampipe-postgres-fdw
-          git init
-          git fetch
-          pwd
-          git checkout ${{ github.event.inputs.tag }}
-          cd steampipe-postgres-fdw/fdw
-
-          echo "make clean"
-          go version
-          make clean
-
-          echo "make go"
-          make go
-
-          echo "make"
-          make
-
-          echo "check binary and copy into artifacts directory"
-          file steampipe_postgres_fdw.so
-          gzip steampipe_postgres_fdw.so
-          mv steampipe_postgres_fdw.so.gz steampipe_postgres_fdw.so.linux_arm64.gz
-          ls
-          cp steampipe_postgres_fdw.so.linux_arm64.gz /artifacts
-          ls -al /artifacts
-    
-    - name: Show the artifact
-      # Items placed in /artifacts in the container will be in
-      # ${PWD}/artifacts on the host.
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: 'ap-south-1'
+        PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY  }}
       run: |
-        ls -al "${PWD}/artifacts"
+
+        # get public IP of the linux machine
+        ip=$(dig +short myip.opendns.com @resolver1.opendns.com)
+        echo $ip
+
+        # set PAGER to empty to not get cli output automatically being sent to vim
+        export PAGER=
+        
+        # whitelist the ip, add the ip to the security group to allow access into the instance
+        aws ec2 authorize-security-group-ingress --group-name pskr-sg --protocol tcp --port 22 --cidr $ip/32 --region ap-south-1
+
+        #start instance
+        aws ec2 start-instances --instance-ids i-059a0baa5fb90a263 --region ap-south-1 --output text
+        echo "starting instance..."
+        sleep 60
+
+        # get the public ip and status of the instance
+        public_ip=$(aws ec2 describe-instances --instance-ids i-059a0baa5fb90a263 --query 'Reservations[*].Instances[*].PublicDnsName' --region ap-south-1 --output text)
+        status=$(aws ec2 describe-instance-status --instance-ids i-059a0baa5fb90a263 --query 'InstanceStatuses[*].InstanceState.Name' --region ap-south-1 --output text)
+        echo $public_ip
+        echo $status
+
+        # check if the instance is available for use, if not, then wait for it
+        while [ $status != "running" ] ; do
+          echo "instance not yet available"
+          sleep 5
+          status=$(aws ec2 describe-instance-status --instance-ids i-059a0baa5fb90a263 --query 'InstanceStatuses[*].InstanceState.Name' --region ap-south-1 --output text)
+        done
+        
+        # set up the private key
+        echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
+        
+        # ssh into the instance and run the set of commands to build
+        ssh -o StrictHostKeyChecking=accept-new -i private_key ubuntu@$public_ip 'cat test.txt; uname -m; cd steampipe-postgres-fdw; git checkout main; git pull; git checkout $VERSION; rm -rf build-Linux; ./build.sh; exit 0'
+
+        # use scp to fetch the built binary out of the instance
+        scp -i private_key ubuntu@$public_ip:/home/ubuntu/steampipe-postgres-fdw/build-Linux/steampipe_postgres_fdw.so .
+        
+        # stop instance
+        aws ec2 stop-instances --instance-ids i-059a0baa5fb90a263 --region ap-south-1
+
+        # remove the ip from the security group
+        aws ec2 revoke-security-group-ingress --group-name pskr-sg --protocol tcp --port 22 --cidr $ip/32 --region ap-south-1
+
+        # check binary exists
+        file steampipe_postgres_fdw.so
+
+    - name: gzip the steampipe_postgres_fdw.so
+      run: |-
+        gzip steampipe_postgres_fdw.so
+        mv steampipe_postgres_fdw.so.gz steampipe_postgres_fdw.so.linux_arm64.gz
 
     - name: Save Linux Build Artifact - ARM64
       uses: actions/upload-artifact@v3
       with:
         name: steampipe_postgres_fdw.so.linux_arm64
-        path: /home/runner/work/steampipe-postgres-fdw/steampipe-postgres-fdw/artifacts/steampipe_postgres_fdw.so.linux_arm64.gz
+        path: steampipe_postgres_fdw.so.linux_arm64.gz
         if-no-files-found: error
 
   build-draft-release:
     name: Build Draft Release
     runs-on: ubuntu-latest
     needs:
       - build-linux
-      - build-osx
       - build-linux-arm
+      - build-osx
     steps:
 
       - name: Get latest version tag

build_arm_and_upload.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# This script is used to build the ARM binaries and upload to the github draft release.
+# Just run this script from your Mac M1 or Linux ARM machine, if you want to build and upload the ARM binaries.
+
+cd fdw
+make clean
+make go
+make
+make release
+cd -

build_release.sh

@@ -50,4 +50,4 @@ inst:
 	rm ./*.o
 
 release:
-	./upload_asset.sh
+	./../upload_arm_asset.sh

fdw/Makefile

@@ -1,5 +1,23 @@
 #!/usr/bin/env bash
 
+ARCH=$(uname -m)
+# exit if the architecture is not arm64(darwin) or aarch64(linux)
+if [[ "$ARCH" != "arm64" ]] && [[ "$ARCH" != "aarch64" ]]; then
+  echo "Not an ARM64 system"
+  exit
+fi
+
+# Must have these commands for the script to run
+declare -a required_commands=("gh" "gzip" "postgres")
+
+for required_command in "${required_commands[@]}"
+do
+  if [[ $(command -v $required_command | head -c1 | wc -c) -eq 0 ]]; then
+    echo "$required_command is required for this script to run."
+    exit -1
+  fi
+done
+
 # get the tag_names of draft releases
 TAG=$(gh api -X GET /repos/{owner}/{repo}/releases -F owner=turbot -F repo=steampipe --jq '.[] | select(.draft == true) | .tag_name')