From ded5f9a8cd8fcb9086304e6e186fe653eca1217a Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 14 Apr 2026 12:40:18 +0100 Subject: [PATCH 1/4] Remove aarch64-runner dir for self-hosted GHA setup --- aarch64-runner/README.md | 76 ---------------------------------------- aarch64-runner/setup.sh | 75 --------------------------------------- 2 files changed, 151 deletions(-) delete mode 100644 aarch64-runner/README.md delete mode 100755 aarch64-runner/setup.sh diff --git a/aarch64-runner/README.md b/aarch64-runner/README.md deleted file mode 100644 index d2348f8e..00000000 --- a/aarch64-runner/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Self-hosted runners - -For building `aarch64` images, we use self-hosted GitHub runners. -The runner is hosted on the apple silicon machine in PSI. - -Configure your runner: - -1. Run under `root`: - - Run this with caution. It can also be run manually step by step. See [setup.sh](setup.sh) for details. - - ```bash - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/main/HEAD/aarch64-runner/setup.sh)" - ``` - - This will perform the initial runner setup and create a user `runner-user`. - -2. Run under `root`, Start docker service, we use [`colima`](https://github.com/abiosoft/colima) as the container runtime: - - ```bash - colima start - ``` - - This command needs to be run every time after reboot. *(Optional: make it auto start on boot)* - -3. Setup new GitHub Runner under `runner-user` using [GitHub Instructions](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners). - **Do not `./run.sh` yet**. - **In the first step, use folder `actions-runner-aiidalab` to distinguish from the other runners.** - -4. Run under `runner-user`, install the runner as a service: - - ```bash - cd /Users/runner-user/actions-runner-aiidalab/ && ./svc.sh install - ``` - This will create the plist file for the runner service, it is not able to run it with the non-gui user. - As shown in the [issue](https://github.com/actions/runner/issues/1056#issuecomment-1237426462), real services start on boot, not on login so on macOS this means the service needs to be a `LaunchDaemon` and not a `LaunchAgent`. - - In case the python path is not correct, change the `runsvc.sh` file to the correct path. - Since we use `colima` as the container runtime, the docker sock is located at `unix://$HOME/.colima/default/docker.sock`. - Change the `runsvc.sh` file to (notice we add two export lines so the runner can find the correct python and docker sock): - - ```bash - #!/bin/bash - - # convert SIGTERM signal to SIGINT - # for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html - trap 'kill -INT $PID' TERM INT - - if [ -f ".path" ]; then - # configure - export PATH=`cat .path` - eval "$(/opt/homebrew/bin/brew shellenv)" - export PATH="/opt/homebrew/bin:$PATH" - export DOCKER_HOST="unix://$HOME/.colima/default/docker.sock - echo ".path=${PATH}" - fi - - nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16} - - # insert anything to setup env when running as a service - # run the host process which keep the listener alive - ./externals/$nodever/bin/node ./bin/RunnerService.js & - PID=$! - wait $PID - trap - TERM INT - wait $PID - ``` - - Then, move the plist file to the correct location and load the service: - ```bash - sudo mv /Users/runner-user/Library/LaunchAgents/actions.runner.*.plist /Library/LaunchDaemons/ - sudo chown root:wheel /Library/LaunchDaemons/actions.runner.*.plist - sudo /bin/launchctl load /Library/LaunchDaemons/actions.runner.aiidalab.Jusong-MacBook-Air.plist - ``` - -5. Reboot the VM to apply all updates and run GitHub runner. diff --git a/aarch64-runner/setup.sh b/aarch64-runner/setup.sh deleted file mode 100755 index e7b3eb2c..00000000 --- a/aarch64-runner/setup.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -set -ex - -GITHUB_RUNNER_USER="runner-user" - -if [ $UID -ne 0 ] ; then echo "Please run $0 as root." && exit 1; fi - -getHiddenUserUid() -{ - local __UIDS=$(dscl . -list /Users UniqueID | awk '{print $2}' | sort -ugr) - - #echo $__UIDS - local __NewUID - for __NewUID in $__UIDS - do - if [[ $__NewUID -lt 499 ]] ; then - break; - fi - done - - echo $((__NewUID+1)) -} - -getInteractiveUserUid() -{ - # Find out the next available user ID - local __MAXID=$(dscl . -list /Users UniqueID | awk '{print $2}' | sort -ug | tail -1) - echo $((__MAXID+1)) -} - - -echo "Setting up runner-user, who will run GitHub Actions runner" - -# Create the user account by running dscl (normally you would have to do each of these commands one -# by one in an obnoxious and time consuming way. - -FULLNAME="Runner User" -USERID=$(getInteractiveUserUid) -GROUPID=20 - -read -s -p "Enter a password for this user: " PASSWORD -echo -read -s -p "Validate a password: " PASSWORD_VALIDATE -echo - -if [[ $PASSWORD != $PASSWORD_VALIDATE ]] ; then - echo "Passwords do not match!" - exit 1; -fi - -sysadminctl -addUser ${GITHUB_RUNNER_USER} -fullName "${FULLNAME}" -UID ${USERID} -GID ${GROUPID} -password "${PASSWORD}" -home /Users/${GITHUB_RUNNER_USER} -admin - -mkdir -p /Users/${GITHUB_RUNNER_USER}/.ssh/ -cp "/Users/${SUDO_USER}/.ssh/authorized_keys" "/Users/${GITHUB_RUNNER_USER}/.ssh/authorized_keys" || true -chown -R $USERID:$GROUPID /Users/${GITHUB_RUNNER_USER}/.ssh - -# Install homebrew (as runner-user) -# You may need to run this by hand, but only the first time setup the self-hosted runner it is required. -echo "Setting up homebrew" -sudo -i -u ${GITHUB_RUNNER_USER} bash << EOF -curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh -echo "Setting up python3" -brew install python -# For Apple Silicon machines, the path are slightly different. -# After running brew install python, must ensure your ~/.zprofile uses the correct Homebrew paths: -echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/${GITHUB_RUNNER_USER}/.zprofile -echo 'export PATH="/opt/homebrew/opt/python/libexec/bin:$PATH"' >> /Users/${GITHUB_RUNNER_USER}/.zprofile -echo 'export DOCKER_HOST="unix://$HOME/.colima/default/docker.sock"' >> /Users/${GITHUB_RUNNER_USER}/.zprofile -echo "Setting up docker " -brew install docker -brew install docker-compose -brew install docker-buildx -brew install colima -brew install jq -EOF From 37b22ad58ef9b2f3a4fca4a30cbb6317f35b549a Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 14 Apr 2026 12:44:14 +0100 Subject: [PATCH 2/4] CI: rename amd64->x86 and arm64 to arm Much easier to distinguish at a glance! --- .github/workflows/main.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee7ecc9a..08e4b7e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,13 +10,11 @@ on: - .pre-commit-config.yaml # We use doit only for local builds - dodo.py - - aarch64-runner/** push: branches: - main tags: - "v*" - workflow_dispatch: # https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: @@ -29,20 +27,20 @@ env: jobs: - build-amd64: + build-x86: uses: ./.github/workflows/build.yml with: runsOn: ubuntu-22.04 platforms: linux/amd64 - build-arm64: + build-arm: uses: ./.github/workflows/build.yml with: runsOn: ubuntu-22.04-arm platforms: linux/arm64 - test-amd64: - needs: build-amd64 + test-x86: + needs: build-x86 strategy: fail-fast: false matrix: @@ -50,12 +48,12 @@ jobs: uses: ./.github/workflows/test.yml with: runsOn: ubuntu-22.04 - images: ${{ needs.build-amd64.outputs.images }} + images: ${{ needs.build-x86.outputs.images }} target: ${{ matrix.target }} integration: false - test-arm64: - needs: build-arm64 + test-arm: + needs: build-arm strategy: fail-fast: false matrix: @@ -63,12 +61,12 @@ jobs: uses: ./.github/workflows/test.yml with: runsOn: ubuntu-22.04-arm - images: ${{ needs.build-arm64.outputs.images }} + images: ${{ needs.build-arm.outputs.images }} target: ${{ matrix.target }} integration: false build: - needs: [test-amd64, test-arm64] + needs: [test-x86, test-arm] uses: ./.github/workflows/build.yml with: runsOn: ubuntu-22.04 @@ -89,7 +87,7 @@ jobs: integration: true publish-ghcr: - needs: [build, test-amd64] + needs: [build] uses: ./.github/workflows/publish.yml with: runsOn: ubuntu-22.04 @@ -101,7 +99,7 @@ jobs: if: >- github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref_type == 'tag' || github.ref_name == 'main') - needs: [build, test-amd64, test-arm64, publish-ghcr] + needs: [build, test-x86, test-arm, publish-ghcr] uses: ./.github/workflows/publish.yml with: runsOn: ubuntu-22.04 From 03460681f5497fa45851432d53c06c73bb321b9a Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 14 Apr 2026 13:30:55 +0100 Subject: [PATCH 3/4] Fix integration tests --- tests/test_aiidalab_apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_aiidalab_apps.py b/tests/test_aiidalab_apps.py index 305d52f7..1b20258d 100644 --- a/tests/test_aiidalab_apps.py +++ b/tests/test_aiidalab_apps.py @@ -21,7 +21,7 @@ def generate_aiidalab_install_output(aiidalab_exec, nb_user): def _generate_aiidalab_install_output(package_name): nonlocal pkg pkg = package_name - cmd = f"aiidalab install --yes --pre {package_name}" + cmd = f"aiidalab install --yes {package_name}" output = aiidalab_exec(cmd, user=nb_user).strip() output += aiidalab_exec("pip check", user=nb_user).strip() From 8cb2b38fa27efff883a8f479cdc07c8b115de746 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 14 Apr 2026 13:39:16 +0100 Subject: [PATCH 4/4] Combine test jobs --- .github/workflows/main.yml | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08e4b7e8..a4aea1a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,34 +39,22 @@ jobs: runsOn: ubuntu-22.04-arm platforms: linux/arm64 - test-x86: - needs: build-x86 + test: + needs: [build-x86, build-arm] strategy: fail-fast: false matrix: target: ["base", "lab", "base-with-services", "full-stack"] + arch: ["x86", "arm"] uses: ./.github/workflows/test.yml with: - runsOn: ubuntu-22.04 - images: ${{ needs.build-x86.outputs.images }} - target: ${{ matrix.target }} - integration: false - - test-arm: - needs: build-arm - strategy: - fail-fast: false - matrix: - target: ["base", "lab", "base-with-services", "full-stack"] - uses: ./.github/workflows/test.yml - with: - runsOn: ubuntu-22.04-arm - images: ${{ needs.build-arm.outputs.images }} + runsOn: ubuntu-22.04${{ matrix.arch == 'arm' && '-arm' || ''}} + images: ${{ matrix.arch == 'arm' && needs.build-arm.outputs.images || needs.build-x86.outputs.images }} target: ${{ matrix.target }} integration: false build: - needs: [test-x86, test-arm] + needs: test uses: ./.github/workflows/build.yml with: runsOn: ubuntu-22.04 @@ -99,7 +87,7 @@ jobs: if: >- github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref_type == 'tag' || github.ref_name == 'main') - needs: [build, test-x86, test-arm, publish-ghcr] + needs: [build, test, publish-ghcr] uses: ./.github/workflows/publish.yml with: runsOn: ubuntu-22.04